diff --git a/docs/advanced/model-metadata.mdx b/docs/advanced/model-metadata.mdx index d21b0b1d9..51645b38b 100644 --- a/docs/advanced/model-metadata.mdx +++ b/docs/advanced/model-metadata.mdx @@ -13,8 +13,8 @@ strongest first: ```mermaid flowchart LR A[Pricing overrides] -->|pricing fields only| B[config.yaml metadata] - B -->|declared fields win| C[ai-model-list catalog] - C -->|when it knows the model| D[Provider discovery] + B -->|declared fields win| C[Provider discovery] + C -->|fields the provider reports| D[ai-model-list catalog] D -.->|modes still missing| E[ID heuristic] ``` @@ -28,20 +28,25 @@ flowchart LR fields win field-by-field over everything below; omitted fields inherit. This is the escape hatch for local models: declaring `modes: [embedding]` also derives the model's category. -3. **The model catalog** — the +3. **Provider discovery** — some providers report metadata in their own model + listings, and what a provider says about its own deployment wins over the + catalog field by field: Gemini's `supportedGenerationMethods`, Cohere's + per-model `endpoints` and context length, OpenRouter's architecture + modalities and context length, Chutes' context length, max output, + capabilities, and pricing, Ollama's `/api/show` capabilities, and llama.cpp's + context window and modalities (see [llama.cpp](/providers/llamacpp)). This + matters most for self-hosted servers, where the running process is the only + source that can know the real context window. Models declared via configured + model lists skip this step. +4. **The model catalog** — the [`ai-model-list`](https://github.com/ENTERPILOT/ai-model-list) registry, fetched from `MODEL_LIST_URL` (default: the registry's `models.min.json` on GitHub) at startup and on every catalog refresh. It supplies the rich defaults — pricing, context windows, capabilities, modes — for most hosted models, matching IDs directly, through aliases, and with release-date - suffixes stripped. Wrong or missing data is best fixed by contributing to the - registry; use an override for an immediate fix. -4. **Provider discovery** — some providers report capabilities in their own - model listings, and GoModel keeps them for the models it discovers there: - Gemini's `supportedGenerationMethods`, Cohere's per-model `endpoints` and - context length, OpenRouter's architecture modalities and context length, and - Ollama's `/api/show` capabilities. Models declared via configured model - lists skip this step. + suffixes stripped, and it fills in every field the provider above did not + report. Wrong or missing data is best fixed by contributing to the registry; + use an override for an immediate fix. 5. **ID heuristic** — a last-resort name check for models that end up with no modes at all (typical for llama.cpp and LM Studio): IDs containing `embed` or matching well-known embedding families (`bge`, `e5`, `gte`, `minilm`) become diff --git a/docs/providers/llamacpp.mdx b/docs/providers/llamacpp.mdx index f8e61e485..6c197c838 100644 --- a/docs/providers/llamacpp.mdx +++ b/docs/providers/llamacpp.mdx @@ -75,17 +75,41 @@ curl -s http://localhost:8080/v1/embeddings \ -d '{"model": "llamacpp/nomic-embed-text-v1.5", "input": "hello"}' ``` +## Model metadata + +Local GGUFs are not in the upstream model catalog, so GoModel reads what it can +from llama-server itself and reports it on `GET /v1/models`: + +- **Context window** — the per-slot `n_ctx` llama-server reports for the model, + which is the context it is *running* with (`--ctx-size`, divided across + `--parallel` slots) rather than the one the GGUF was trained for. Recent + builds report it as `meta.n_ctx` in the listing itself; older ones are asked + for it via `/props`. If neither answers — LM Studio, or a proxy that hides + `/props` — the model's trained `n_ctx_train` is used instead, which is only an + upper bound and may exceed what the server will accept. +- **Modalities** — a multimodal server (started with `--mmproj`) reports + `vision`, `video`, and `audio` on `/props`; the supported ones become model + capabilities. + +`/props` describes the one model the server has loaded, so it is only consulted +for a single-entry listing. In [router +mode](https://github.com/ggml-org/llama.cpp/tree/master/tools/server), where one +process serves several models, each model carries its own `meta.n_ctx` and needs +no `/props` at all. + +Both are defaults, not decisions: anything you declare under the provider's +model metadata wins — see [Model metadata](/advanced/model-metadata). + ## Model classification -llama-server's `/v1/models` listing carries no capability metadata, so GoModel -classifies its models by ID: names containing `embed` or matching well-known -embedding families (`bge`, `e5`, `gte`, `minilm`) are categorized as embedding -models, and names containing `rerank` as reranking models — namespaced IDs are -checked by their final path segment. For anything that stays unclassified, -declare `modes` (and context window or pricing) under the provider's model -metadata — see [Model metadata](/advanced/model-metadata). Categories only -affect dashboard grouping and failover suggestions; `/v1/embeddings` routes to -any model the provider serves regardless of category. +llama-server reports nothing that separates a chat model from an embedding one, +so GoModel classifies its models by ID: names containing `embed` or matching +well-known embedding families (`bge`, `e5`, `gte`, `minilm`) are categorized as +embedding models, and names containing `rerank` as reranking models — namespaced +IDs are checked by their final path segment. For anything that stays +unclassified, declare `modes` under the provider's model metadata. Categories +only affect dashboard grouping and failover suggestions; `/v1/embeddings` routes +to any model the provider serves regardless of category. ## Beyond chat and embeddings diff --git a/internal/modeldata/enricher.go b/internal/modeldata/enricher.go index 65dc68860..55118ad90 100644 --- a/internal/modeldata/enricher.go +++ b/internal/modeldata/enricher.go @@ -14,6 +14,11 @@ type ModelInfoAccessor interface { GetProviderType(modelID string) string // SetMetadata sets the metadata for a model ID. SetMetadata(modelID string, meta *core.ModelMetadata) + // DiscoveredMetadata returns the metadata the provider itself reported for + // a model, or nil when it reported none. It must keep returning the same + // value across enrichment passes: Enrich merges onto it rather than onto + // its own previous result, which is what keeps repeated passes idempotent. + DiscoveredMetadata(modelID string) *core.ModelMetadata } // EnrichStats summarizes one metadata enrichment pass. @@ -22,8 +27,15 @@ type EnrichStats struct { Total int } -// Enrich iterates all models accessible via the accessor and attaches resolved -// metadata from the model list. Models not found in the list are left unchanged. +// Enrich iterates all models accessible via the accessor and merges resolved +// catalog metadata into each one. Models the catalog does not know are left +// unchanged. +// +// The catalog is the base and the provider's own report the override, field by +// field: a running provider describes its actual deployment (a local server's +// real context window, an API's live capability flags) better than a static +// registry can, while the catalog still supplies everything the provider never +// reports — display names, pricing, rankings. func Enrich(accessor ModelInfoAccessor, list *ModelList) EnrichStats { if list == nil || accessor == nil { return EnrichStats{} @@ -34,11 +46,17 @@ func Enrich(accessor ModelInfoAccessor, list *ModelList) EnrichStats { for _, modelID := range ids { providerType := accessor.GetProviderType(modelID) - meta := Resolve(list, providerType, modelID) - if meta != nil { - accessor.SetMetadata(modelID, meta) - enriched++ + discovered := accessor.DiscoveredMetadata(modelID) + catalog := Resolve(list, providerType, modelID) + if catalog == nil { + // The catalog does not know this model. Fall back to the provider's + // own report rather than leaving whatever a previous pass merged in: + // a refresh that drops an entry must drop its fields too. + accessor.SetMetadata(modelID, discovered.Clone()) + continue } + accessor.SetMetadata(modelID, MergeMetadata(catalog, discovered)) + enriched++ } return EnrichStats{ diff --git a/internal/modeldata/enricher_test.go b/internal/modeldata/enricher_test.go index 1d07d41b4..c6a913f1a 100644 --- a/internal/modeldata/enricher_test.go +++ b/internal/modeldata/enricher_test.go @@ -11,12 +11,14 @@ type mockAccessor struct { ids []string providerTypes map[string]string metadata map[string]*core.ModelMetadata + discovered map[string]*core.ModelMetadata } func newMockAccessor(models map[string]string) *mockAccessor { a := &mockAccessor{ providerTypes: models, metadata: make(map[string]*core.ModelMetadata), + discovered: make(map[string]*core.ModelMetadata), } for id := range models { a.ids = append(a.ids, id) @@ -30,6 +32,10 @@ func (a *mockAccessor) SetMetadata(modelID string, meta *core.ModelMetadata) { a.metadata[modelID] = meta } +func (a *mockAccessor) DiscoveredMetadata(modelID string) *core.ModelMetadata { + return a.discovered[modelID] +} + func TestEnrich_MatchedAndUnmatched(t *testing.T) { list := &ModelList{ Models: map[string]ModelEntry{ @@ -64,14 +70,13 @@ func TestEnrich_MatchedAndUnmatched(t *testing.T) { } } - // unknown-model should NOT be enriched - if _, ok := accessor.metadata["unknown-model"]; ok { - t.Error("expected unknown-model to NOT be enriched") + // Models the catalog does not know keep only what their provider reported, + // which here is nothing. + if meta := accessor.metadata["unknown-model"]; meta != nil { + t.Errorf("expected unknown-model to carry no catalog metadata, got %+v", meta) } - - // custom-finetune should NOT be enriched - if _, ok := accessor.metadata["custom-finetune"]; ok { - t.Error("expected custom-finetune to NOT be enriched") + if meta := accessor.metadata["custom-finetune"]; meta != nil { + t.Errorf("expected custom-finetune to carry no catalog metadata, got %+v", meta) } } @@ -165,3 +170,91 @@ func TestEnrich_ProviderModelOverride(t *testing.T) { t.Errorf("ContextWindow = %d, want 64000 (azure override)", *meta.ContextWindow) } } + +func TestEnrich_ProviderDiscoveryWinsFieldWiseOverCatalog(t *testing.T) { + accessor := newMockAccessor(map[string]string{"gemma-3-4b-it": "llamacpp"}) + // What a local server reported about the model it is actually running. + accessor.discovered["gemma-3-4b-it"] = &core.ModelMetadata{ + ContextWindow: new(4096), + Capabilities: map[string]bool{"vision": true}, + } + // A catalog entry that knows the bare ID, with no llamacpp entry at all. + list := &ModelList{Models: map[string]ModelEntry{ + "gemma-3-4b-it": { + DisplayName: "Gemma 3 4B IT", + ContextWindow: new(131072), + Capabilities: map[string]bool{"tools": true}, + }, + }} + + Enrich(accessor, list) + + got := accessor.metadata["gemma-3-4b-it"] + if got == nil { + t.Fatal("metadata = nil, want merged metadata") + } + if got.ContextWindow == nil || *got.ContextWindow != 4096 { + t.Fatalf("context window = %v, want the running server's 4096", got.ContextWindow) + } + if !got.Capabilities["vision"] { + t.Fatal("discovered capability vision was dropped") + } + // Fields the provider never reports still come from the catalog. + if !got.Capabilities["tools"] { + t.Fatal("catalog capability tools was dropped") + } + if got.DisplayName != "Gemma 3 4B IT" { + t.Fatalf("display name = %q, want the catalog's", got.DisplayName) + } +} + +func TestEnrich_RepeatedPassesTrackCatalogUpdates(t *testing.T) { + accessor := newMockAccessor(map[string]string{"gpt-4o": "openai"}) + list := &ModelList{Models: map[string]ModelEntry{ + "gpt-4o": {DisplayName: "GPT-4o", ContextWindow: new(128000)}, + }} + + Enrich(accessor, list) + if got := accessor.metadata["gpt-4o"]; got.ContextWindow == nil || *got.ContextWindow != 128000 { + t.Fatalf("first pass context window = %v, want 128000", got.ContextWindow) + } + + // A later catalog refresh corrects the value. Because Enrich merges onto the + // provider's pristine report rather than onto its own previous output, the + // new value must win instead of being pinned by the stale one. + list.Models["gpt-4o"] = ModelEntry{DisplayName: "GPT-4o", ContextWindow: new(200000)} + Enrich(accessor, list) + + if got := accessor.metadata["gpt-4o"]; got.ContextWindow == nil || *got.ContextWindow != 200000 { + t.Fatalf("second pass context window = %v, want the refreshed 200000", got.ContextWindow) + } +} + +func TestEnrich_DropsCatalogFieldsWhenEntryDisappears(t *testing.T) { + accessor := newMockAccessor(map[string]string{"gemma-3-4b-it": "llamacpp"}) + accessor.discovered["gemma-3-4b-it"] = &core.ModelMetadata{ContextWindow: new(4096)} + list := &ModelList{Models: map[string]ModelEntry{ + "gemma-3-4b-it": {DisplayName: "Gemma 3 4B IT", ContextWindow: new(131072)}, + }} + + Enrich(accessor, list) + if got := accessor.metadata["gemma-3-4b-it"]; got.DisplayName != "Gemma 3 4B IT" { + t.Fatalf("first pass display name = %q, want the catalog's", got.DisplayName) + } + + // The catalog drops the entry on a later refresh; its fields must go with + // it, leaving only what the provider itself reported. + delete(list.Models, "gemma-3-4b-it") + Enrich(accessor, list) + + got := accessor.metadata["gemma-3-4b-it"] + if got == nil { + t.Fatal("metadata = nil, want the provider's own report to survive") + } + if got.DisplayName != "" { + t.Fatalf("display name = %q, want it dropped with the catalog entry", got.DisplayName) + } + if got.ContextWindow == nil || *got.ContextWindow != 4096 { + t.Fatalf("context window = %v, want the provider's 4096", got.ContextWindow) + } +} diff --git a/internal/providers/configured_models.go b/internal/providers/configured_models.go index 8bb2723f6..1740f2785 100644 --- a/internal/providers/configured_models.go +++ b/internal/providers/configured_models.go @@ -160,12 +160,7 @@ func modelInfoMapFromResponse(resp *core.ModelsResponse, provider core.Provider, continue } model.ID = modelID - out[modelID] = &ModelInfo{ - Model: model, - Provider: provider, - ProviderName: providerName, - ProviderType: providerType, - } + out[modelID] = newModelInfo(model, provider, providerName, providerType) } return out } diff --git a/internal/providers/llamacpp/llamacpp.go b/internal/providers/llamacpp/llamacpp.go index 256aadcdc..a6f6c9be4 100644 --- a/internal/providers/llamacpp/llamacpp.go +++ b/internal/providers/llamacpp/llamacpp.go @@ -33,6 +33,12 @@ var Registration = providers.Registration{ type Provider struct { compatible *openai.CompatibleProvider rootClient *llmclient.Client + // propsClient issues the optional /props metadata call. It is deliberately + // separate from rootClient: /props is best-effort enrichment, so it must not + // spend the retry budget or trip the circuit breaker that native passthrough + // routes share. A server that fails /props with a retryable status would + // otherwise take /health, /rerank and /tokenize down with it. + propsClient *llmclient.Client } var ( @@ -63,9 +69,23 @@ func New(cfg providers.ProviderConfig, opts providers.ProviderOptions) core.Prov }, func(req *http.Request) { setHeaders(req, keys.NextForContext(req.Context())) }), + propsClient: newPropsClient(baseURL, opts.Hooks, func(req *http.Request) { + setHeaders(req, keys.NextForContext(req.Context())) + }), } } +// newPropsClient builds the client used for optional /props enrichment: no +// retries and no circuit breaker, so a failing /props costs one request and +// leaves the shared native-route budget untouched. +func newPropsClient(baseURL string, hooks llmclient.Hooks, setHeader llmclient.HeaderSetter) *llmclient.Client { + return llmclient.New(llmclient.Config{ + ProviderName: "llamacpp", + BaseURL: passthroughBaseURL(baseURL), + Hooks: hooks, + }, setHeader) +} + // NewWithHTTPClient creates a new llama.cpp provider with a custom HTTP client. // If httpClient is nil, http.DefaultClient is used. func NewWithHTTPClient(apiKey string, baseURL string, httpClient *http.Client, hooks llmclient.Hooks) *Provider { @@ -81,6 +101,13 @@ func NewWithHTTPClient(apiKey string, baseURL string, httpClient *http.Client, h rootClient: llmclient.NewWithHTTPClient(httpClient, rootClientCfg, func(req *http.Request) { setHeaders(req, apiKey) }), + propsClient: llmclient.NewWithHTTPClient(httpClient, llmclient.Config{ + ProviderName: "llamacpp", + BaseURL: passthroughBaseURL(resolvedBaseURL), + Hooks: hooks, + }, func(req *http.Request) { + setHeaders(req, apiKey) + }), } } @@ -88,6 +115,7 @@ func NewWithHTTPClient(apiKey string, baseURL string, httpClient *http.Client, h func (p *Provider) SetBaseURL(url string) { p.compatible.SetBaseURL(url) p.rootClient.SetBaseURL(passthroughBaseURL(url)) + p.propsClient.SetBaseURL(passthroughBaseURL(url)) } func setHeaders(req *http.Request, apiKey string) { @@ -108,11 +136,6 @@ func (p *Provider) StreamChatCompletion(ctx context.Context, req *core.ChatReque return p.compatible.StreamChatCompletion(ctx, req) } -// ListModels retrieves the list of available models from llama-server. -func (p *Provider) ListModels(ctx context.Context) (*core.ModelsResponse, error) { - return p.compatible.ListModels(ctx) -} - // Responses sends a Responses API request to llama-server. func (p *Provider) Responses(ctx context.Context, req *core.ResponsesRequest) (*core.ResponsesResponse, error) { return p.compatible.Responses(ctx, req) diff --git a/internal/providers/llamacpp/models.go b/internal/providers/llamacpp/models.go new file mode 100644 index 000000000..30c1d0b5c --- /dev/null +++ b/internal/providers/llamacpp/models.go @@ -0,0 +1,181 @@ +package llamacpp + +import ( + "context" + "net/http" + "strings" + "time" + + "github.com/enterpilot/gomodel/internal/core" + "github.com/enterpilot/gomodel/internal/llmclient" +) + +// modelsResponse mirrors llama-server's /v1/models payload. It restates the +// OpenAI-compatible fields core.Model already carries because the response also +// holds a "meta" object describing the loaded GGUF, which the plain OpenAI +// shape drops — decoding it here keeps the llama.cpp-specific field in this +// package instead of in core. +type modelsResponse struct { + Object string `json:"object"` + Data []modelEntry `json:"data"` +} + +type modelEntry struct { + ID string `json:"id"` + Object string `json:"object"` + OwnedBy string `json:"owned_by"` + Created int64 `json:"created"` + Meta *modelMeta `json:"meta"` +} + +// modelMeta is llama-server's description of the GGUF behind a model entry. +// n_ctx is the context the server is running with and n_ctx_train the one the +// model was trained for; the rest of the object (n_embd, n_params, size, +// n_vocab, vocab_type, ftype) stays undecoded rather than leaking llama.cpp +// internals into the OpenAI-compatible model shape. +type modelMeta struct { + NCtx int `json:"n_ctx"` + NCtxTrain int `json:"n_ctx_train"` +} + +// serverProps is the subset of llama-server's /props response we surface. +// default_generation_settings.n_ctx is the per-slot context the server was +// actually started with — the limit a request is measured against. +// propsTimeout bounds the optional /props enrichment call. +const propsTimeout = 5 * time.Second + +type serverProps struct { + DefaultGenerationSettings struct { + NCtx int `json:"n_ctx"` + } `json:"default_generation_settings"` + Modalities map[string]bool `json:"modalities"` +} + +// ListModels retrieves the list of available models from llama-server, keeping +// the context window and modalities the server reports for the model it has +// loaded so clients can size requests without a model-registry entry. +func (p *Provider) ListModels(ctx context.Context) (*core.ModelsResponse, error) { + var raw modelsResponse + if err := p.compatible.Do(ctx, llmclient.Request{ + Method: http.MethodGet, + Endpoint: "/models", + }, &raw); err != nil { + return nil, err + } + return raw.toCore(p.fetchServerProps(ctx, len(raw.Data))), nil +} + +// fetchServerProps reads the running server's own state, or nil when it is +// unavailable or cannot be attributed. /props describes the single model +// llama-server has loaded, so it is only meaningful for a single-entry listing; +// llama.cpp's router mode serves several models from one process, and each of +// those already carries its own meta.n_ctx. Failures are ignored — /props is +// enrichment, and servers that do not implement it (LM Studio answers 200 with +// an error body) must still list their models. +func (p *Provider) fetchServerProps(ctx context.Context, modelCount int) *serverProps { + if modelCount != 1 { + return nil + } + // Bounded so a server that accepts the connection but never answers cannot + // stall discovery on an optional call. + ctx, cancel := context.WithTimeout(ctx, propsTimeout) + defer cancel() + + var props serverProps + if err := p.propsClient.Do(ctx, llmclient.Request{ + Method: http.MethodGet, + Endpoint: "/props", + }, &props); err != nil { + return nil + } + return &props +} + +func (r *modelsResponse) toCore(props *serverProps) *core.ModelsResponse { + object := strings.TrimSpace(r.Object) + if object == "" { + object = "list" + } + resp := &core.ModelsResponse{Object: object, Data: make([]core.Model, 0, len(r.Data))} + for _, entry := range r.Data { + resp.Data = append(resp.Data, entry.toCore(props)) + } + return resp +} + +func (e modelEntry) toCore(props *serverProps) core.Model { + object := strings.TrimSpace(e.Object) + if object == "" { + object = "model" + } + model := core.Model{ + ID: strings.TrimSpace(e.ID), + Object: object, + OwnedBy: strings.TrimSpace(e.OwnedBy), + Created: e.Created, + } + + // Modes stay unset on purpose. llama-server reports nothing that separates a + // chat model from an embedding one, so leaving them empty keeps the + // registry's ID heuristic free to classify local GGUFs. + metadata := core.ModelMetadata{} + if contextWindow := e.contextWindow(props); contextWindow > 0 { + metadata.ContextWindow = &contextWindow + } + if props != nil { + metadata.Capabilities = modalityCapabilities(props.Modalities) + } + if metadata.ContextWindow == nil && len(metadata.Capabilities) == 0 { + // Nothing was reported; leaving Metadata nil keeps the model catalog and + // the ID heuristic free to supply everything. + return model + } + model.Metadata = &metadata + return model +} + +// contextWindow resolves the limit a request is actually measured against, +// strongest source first: +// +// - meta.n_ctx — the per-slot context the server is running with, reported +// per model, so it stays correct when one process serves several. +// - /props — the same number, for builds whose listing predates meta.n_ctx. +// - meta.n_ctx_train — the GGUF's trained ceiling. Only an upper bound: +// llama-server's --ctx-size default sits far below it for most models, so +// it is a last resort rather than the headline figure. +func (e modelEntry) contextWindow(props *serverProps) int { + if e.Meta != nil && e.Meta.NCtx > 0 { + return e.Meta.NCtx + } + if props != nil && props.DefaultGenerationSettings.NCtx > 0 { + return props.DefaultGenerationSettings.NCtx + } + if e.Meta != nil { + return e.Meta.NCtxTrain + } + return 0 +} + +// modalityCapabilities maps llama-server's multimodal flags onto GoModel +// capability keys. Only the modalities that have an established capability name +// are published — an unrecognized key llama.cpp adds later would otherwise +// become a public capability nobody can interpret. Unsupported modalities are +// omitted rather than recorded as false, so a later metadata layer can still +// claim them. +func modalityCapabilities(modalities map[string]bool) map[string]bool { + capabilities := make(map[string]bool, len(modalities)) + for modality, supported := range modalities { + name := strings.ToLower(strings.TrimSpace(modality)) + if !supported { + continue + } + switch name { + case "vision", "video", "audio": + capabilities[name] = true + } + } + if len(capabilities) == 0 { + return nil + } + return capabilities +} diff --git a/internal/providers/llamacpp/models_test.go b/internal/providers/llamacpp/models_test.go new file mode 100644 index 000000000..9289f7acc --- /dev/null +++ b/internal/providers/llamacpp/models_test.go @@ -0,0 +1,313 @@ +package llamacpp + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/enterpilot/gomodel/config" + "github.com/enterpilot/gomodel/internal/core" + "github.com/enterpilot/gomodel/internal/llmclient" + "github.com/enterpilot/gomodel/internal/providers" +) + +// legacyListing is the /v1/models payload of builds whose meta object predates +// n_ctx, leaving only the GGUF's trained context. +const legacyListing = `{ + "object":"list", + "data":[{ + "id":"Meta-Llama-3.1-8B-Instruct", + "object":"model", + "created":1735142223, + "owned_by":"llamacpp", + "meta":{"vocab_type":2,"n_vocab":128256,"n_ctx_train":131072,"n_embd":4096,"n_params":8030261312,"size":4912898304} + }] +}` + +func TestListModels_SurfacesServerReportedMetadata(t *testing.T) { + // realListing is a verbatim llama-server b10470 payload (gemma-3-270m-it + // started with -c 2048), whose meta carries the running context as n_ctx. + const realListing = `{ + "object":"list", + "data":[{ + "id":"gemma-3-270m-it", + "object":"model", + "created":1787218858, + "owned_by":"llamacpp", + "meta":{"vocab_type":1,"n_vocab":262144,"n_ctx":2048,"n_ctx_train":32768,"n_embd":640,"n_params":268098176,"size":285018624,"ftype":"Q8_0"} + }] + }` + + tests := []struct { + name string + listing string + propsStatus int + props string + wantContextWindow int + wantModelID string + wantCapabilities map[string]bool + wantPropsFetched bool + }{ + { + // meta.n_ctx is per-model, so it must win even over a /props that + // disagrees — in router mode /props cannot be attributed at all. + name: "listing n_ctx wins over props and trained context", + listing: realListing, + propsStatus: http.StatusOK, + props: `{"default_generation_settings":{"n_ctx":9999},"modalities":{"vision":false,"video":false,"audio":false}}`, + wantContextWindow: 2048, + wantModelID: "gemma-3-270m-it", + wantPropsFetched: true, + }, + { + name: "props runtime context wins over trained context on older builds", + listing: legacyListing, + propsStatus: http.StatusOK, + props: `{"default_generation_settings":{"n_ctx":8192},"total_slots":1,"modalities":{"vision":false}}`, + wantContextWindow: 8192, + wantPropsFetched: true, + }, + { + name: "trained context is the fallback when props is unavailable", + listing: legacyListing, + propsStatus: http.StatusNotFound, + props: `{"error":"not found"}`, + wantContextWindow: 131072, + wantPropsFetched: true, + }, + { + // LM Studio answers /props with 200 and an error body rather than a + // 404, so a decodable-but-empty payload must not be mistaken for a + // server reporting a zero context. + name: "props answered with an unrelated 200 payload", + listing: legacyListing, + propsStatus: http.StatusOK, + props: `{"error":"Unexpected endpoint or method. (GET /props)"}`, + wantContextWindow: 131072, + wantPropsFetched: true, + }, + { + // A truncated or non-JSON body must not be read as a zero context. + name: "props answered with malformed json", + listing: legacyListing, + propsStatus: http.StatusOK, + props: `{"default_generation_settings":{"n_ctx":`, + wantContextWindow: 131072, + wantPropsFetched: true, + }, + { + name: "supported modalities become capabilities", + listing: legacyListing, + propsStatus: http.StatusOK, + // "telepathy" stands in for a modality a future llama.cpp adds: it + // must not become a public capability on its own. + props: `{"default_generation_settings":{"n_ctx":4096},"modalities":{"vision":true,"video":true,"audio":false,"telepathy":true}}`, + wantContextWindow: 4096, + wantCapabilities: map[string]bool{"vision": true, "video": true}, + wantPropsFetched: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + propsFetched := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/v1/models": + _, _ = w.Write([]byte(tt.listing)) + case "/props": + propsFetched = true + w.WriteHeader(tt.propsStatus) + _, _ = w.Write([]byte(tt.props)) + default: + t.Errorf("unexpected path %q", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + + provider := NewWithHTTPClient("", server.URL+"/v1", server.Client(), llmclient.Hooks{}) + + resp, err := provider.ListModels(context.Background()) + if err != nil { + t.Fatalf("ListModels() error = %v", err) + } + if len(resp.Data) != 1 { + t.Fatalf("len(resp.Data) = %d, want 1", len(resp.Data)) + } + if propsFetched != tt.wantPropsFetched { + t.Fatalf("props fetched = %v, want %v", propsFetched, tt.wantPropsFetched) + } + + model := resp.Data[0] + wantID := tt.wantModelID + if wantID == "" { + wantID = "Meta-Llama-3.1-8B-Instruct" + } + if model.ID != wantID { + t.Fatalf("model.ID = %q, want %q", model.ID, wantID) + } + if model.Metadata == nil { + t.Fatalf("model.Metadata = nil, want context window %d", tt.wantContextWindow) + } + if model.Metadata.ContextWindow == nil || *model.Metadata.ContextWindow != tt.wantContextWindow { + t.Fatalf("context window = %v, want %d", model.Metadata.ContextWindow, tt.wantContextWindow) + } + if len(model.Metadata.Capabilities) != len(tt.wantCapabilities) { + t.Fatalf("capabilities = %v, want %v", model.Metadata.Capabilities, tt.wantCapabilities) + } + for name, want := range tt.wantCapabilities { + if model.Metadata.Capabilities[name] != want { + t.Fatalf("capability %q = %v, want %v", name, model.Metadata.Capabilities[name], want) + } + } + // Modes must stay empty so the registry's ID heuristic can still + // classify local embedding and reranking GGUFs. + if len(model.Metadata.Modes) != 0 { + t.Fatalf("modes = %v, want none", model.Metadata.Modes) + } + }) + } +} + +func TestListModels_RouterModeKeepsPerModelContextAndSkipsProps(t *testing.T) { + propsFetched := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/v1/models": + _, _ = w.Write([]byte(`{ + "object":"list", + "data":[ + {"id":"gemma-3-4b","object":"model","meta":{"n_ctx":8192,"n_ctx_train":131072}}, + {"id":"qwen3-8b","object":"model","meta":{"n_ctx":32768,"n_ctx_train":262144}} + ] + }`)) + case "/props": + propsFetched = true + _, _ = w.Write([]byte(`{"default_generation_settings":{"n_ctx":512}}`)) + default: + t.Errorf("unexpected path %q", r.URL.Path) + } + })) + defer server.Close() + + provider := NewWithHTTPClient("", server.URL+"/v1", server.Client(), llmclient.Hooks{}) + + resp, err := provider.ListModels(context.Background()) + if err != nil { + t.Fatalf("ListModels() error = %v", err) + } + if propsFetched { + t.Fatal("props was fetched for a multi-model listing; it describes a single loaded model") + } + + want := map[string]int{"gemma-3-4b": 8192, "qwen3-8b": 32768} + for _, model := range resp.Data { + if model.Metadata == nil || model.Metadata.ContextWindow == nil { + t.Fatalf("model %q lost its context window", model.ID) + } + if got := *model.Metadata.ContextWindow; got != want[model.ID] { + t.Fatalf("model %q context window = %d, want %d", model.ID, got, want[model.ID]) + } + } +} + +func TestListModels_LeavesMetadataUnsetWhenServerReportsNothing(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/v1/models": + // LM Studio and other plain OpenAI-compatible servers omit "meta". + _, _ = w.Write([]byte(`{"data":[{"id":"local-model"}]}`)) + case "/props": + _, _ = w.Write([]byte(`{"default_generation_settings":{"n_ctx":0}}`)) + default: + t.Errorf("unexpected path %q", r.URL.Path) + } + })) + defer server.Close() + + provider := NewWithHTTPClient("", server.URL+"/v1", server.Client(), llmclient.Hooks{}) + + resp, err := provider.ListModels(context.Background()) + if err != nil { + t.Fatalf("ListModels() error = %v", err) + } + if len(resp.Data) != 1 { + t.Fatalf("len(resp.Data) = %d, want 1", len(resp.Data)) + } + if resp.Object != "list" { + t.Fatalf("resp.Object = %q, want list", resp.Object) + } + if resp.Data[0].Object != "model" { + t.Fatalf("model.Object = %q, want model", resp.Data[0].Object) + } + if resp.Data[0].Metadata != nil { + t.Fatalf("model.Metadata = %+v, want nil so lower metadata layers still apply", resp.Data[0].Metadata) + } +} + +// TestListModels_FailingPropsLeavesNativeRoutesUsable pins the isolation of the +// optional /props call: it must not retry against the shared native-route +// budget, nor trip the circuit breaker those routes depend on. Sharing +// rootClient here cost four attempts per listing and locked /health out +// entirely after six discovery cycles. +func TestListModels_FailingPropsLeavesNativeRoutesUsable(t *testing.T) { + var propsAttempts, healthUpstream int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/v1/models": + _, _ = w.Write([]byte(`{"object":"list","data":[{"id":"m","object":"model","meta":{"n_ctx_train":8192}}]}`)) + case "/props": + propsAttempts++ + w.WriteHeader(http.StatusServiceUnavailable) // retryable status + _, _ = w.Write([]byte(`{"error":"unavailable"}`)) + case "/health": + healthUpstream++ + _, _ = w.Write([]byte(`{"status":"ok"}`)) + default: + t.Errorf("unexpected path %q", r.URL.Path) + } + })) + defer server.Close() + + retry := config.DefaultRetryConfig() + retry.InitialBackoff = time.Millisecond // the attempt count is what matters + provider := New(providers.ProviderConfig{BaseURL: server.URL + "/v1"}, providers.ProviderOptions{ + Resilience: config.ResilienceConfig{ + Retry: retry, + CircuitBreaker: config.DefaultCircuitBreakerConfig(), + }, + }).(*Provider) + + const listings = 6 + for i := range listings { + resp, err := provider.ListModels(context.Background()) + if err != nil { + t.Fatalf("ListModels() #%d error = %v", i, err) + } + // The listing still succeeds on meta.n_ctx_train despite /props failing. + if resp.Data[0].Metadata == nil || *resp.Data[0].Metadata.ContextWindow != 8192 { + t.Fatalf("listing #%d lost its fallback context window", i) + } + } + if propsAttempts != listings { + t.Fatalf("props attempts = %d, want %d (one per listing, no retries)", propsAttempts, listings) + } + + if _, err := provider.Passthrough(context.Background(), &core.PassthroughRequest{ + Method: http.MethodGet, + Endpoint: "health", + Headers: http.Header{}, + }); err != nil { + t.Fatalf("native /health rejected after failing /props calls: %v", err) + } + if healthUpstream != 1 { + t.Fatalf("health upstream hits = %d, want 1", healthUpstream) + } +} diff --git a/internal/providers/registry.go b/internal/providers/registry.go index 48215ea0a..6507db5ae 100644 --- a/internal/providers/registry.go +++ b/internal/providers/registry.go @@ -23,6 +23,25 @@ type ModelInfo struct { Provider core.Provider ProviderName string ProviderType string + // Discovered is the metadata the provider itself reported for this model, + // held apart from Model.Metadata because enrichment rewrites that field on + // every catalog refresh. Keeping the pristine value lets each refresh + // recompute the merge from the same inputs rather than layering onto its own + // previous output, which would otherwise pin stale catalog data in place. + Discovered *core.ModelMetadata +} + +// newModelInfo registers a model together with the metadata its provider +// reported for it. Always build ModelInfo through this so Discovered is +// captured before enrichment has a chance to overwrite Model.Metadata. +func newModelInfo(model core.Model, provider core.Provider, providerName, providerType string) *ModelInfo { + return &ModelInfo{ + Model: model, + Provider: provider, + ProviderName: providerName, + ProviderType: providerType, + Discovered: model.Metadata.Clone(), + } } // ModelRegistry manages the mapping of models to their providers. diff --git a/internal/providers/registry_cache.go b/internal/providers/registry_cache.go index b12f29696..c83fadee6 100644 --- a/internal/providers/registry_cache.go +++ b/internal/providers/registry_cache.go @@ -69,17 +69,14 @@ func (r *ModelRegistry) LoadFromCache(ctx context.Context) (int, error) { } providerModels := make(map[string]*ModelInfo, len(cachedProv.Models)) for _, cached := range cachedProv.Models { - info := &ModelInfo{ - Model: core.Model{ - ID: cached.ID, - Object: "model", - OwnedBy: cachedProv.OwnedBy, - Created: cached.Created, - }, - Provider: provider, - ProviderName: providerName, - ProviderType: providerType, - } + // The cache stores no metadata, so nothing was discovered yet; the + // next live refresh supplies it. + info := newModelInfo(core.Model{ + ID: cached.ID, + Object: "model", + OwnedBy: cachedProv.OwnedBy, + Created: cached.Created, + }, provider, providerName, providerType) providerModels[cached.ID] = info if _, exists := newModels[cached.ID]; !exists { newModels[cached.ID] = info diff --git a/internal/providers/registry_init.go b/internal/providers/registry_init.go index 570eeae95..6bfba85c8 100644 --- a/internal/providers/registry_init.go +++ b/internal/providers/registry_init.go @@ -260,12 +260,7 @@ func (r *ModelRegistry) fetchAllProviderModels( } for _, model := range resp.Data { - info := &ModelInfo{ - Model: model, - Provider: provider, - ProviderName: providerName, - ProviderType: providerTypes[provider], - } + info := newModelInfo(model, provider, providerName, providerTypes[provider]) out.modelsByProvider[providerName][model.ID] = info if _, exists := out.models[model.ID]; exists { diff --git a/internal/providers/registry_metadata.go b/internal/providers/registry_metadata.go index 5cfd730e1..8e2139db1 100644 --- a/internal/providers/registry_metadata.go +++ b/internal/providers/registry_metadata.go @@ -487,3 +487,12 @@ func (a *registryAccessor) SetMetadata(modelID string, meta *core.ModelMetadata) info.Model.Metadata = meta } } + +// DiscoveredMetadata returns the metadata the provider reported for a model at +// registration, which Enrich merges on top of the catalog's. +func (a *registryAccessor) DiscoveredMetadata(modelID string) *core.ModelMetadata { + if info, ok := a.models[modelID]; ok { + return info.Discovered + } + return nil +} diff --git a/internal/providers/registry_metadata_override_test.go b/internal/providers/registry_metadata_override_test.go index 358d4cfe6..5aaaca764 100644 --- a/internal/providers/registry_metadata_override_test.go +++ b/internal/providers/registry_metadata_override_test.go @@ -455,3 +455,71 @@ func TestSetProviderMetadataOverrides_DeepClonesExternalInput(t *testing.T) { t.Errorf("stored Pricing mutated via caller: %+v", stored.Pricing) } } + +// TestEnrichModels_KeepsProviderDiscoveredMetadataOverCatalog covers the local +// server case: a llama.cpp alias that collides with a catalog ID must not lose +// the context window the running server actually reported. +func TestEnrichModels_KeepsProviderDiscoveredMetadataOverCatalog(t *testing.T) { + registry := NewModelRegistry() + + runtimeContext := 4096 + mock := ®istryMockProvider{ + name: "llamacpp", + modelsResponse: &core.ModelsResponse{ + Object: "list", + Data: []core.Model{ + { + ID: "gemma-3-4b-it", + Object: "model", + OwnedBy: "llamacpp", + Metadata: &core.ModelMetadata{ + ContextWindow: &runtimeContext, + Capabilities: map[string]bool{"vision": true}, + }, + }, + }, + }, + } + registry.RegisterProviderWithType(mock, "llamacpp") + if err := registry.Initialize(context.Background()); err != nil { + t.Fatalf("Initialize() error = %v", err) + } + + // The catalog knows the bare model ID and has no llamacpp entry at all. + raw := []byte(`{ + "version": 1, + "updated_at": "2025-01-01T00:00:00Z", + "models": { + "gemma-3-4b-it": { + "display_name": "Gemma 3 4B IT", + "modes": ["chat"], + "context_window": 131072 + } + } + }`) + list, err := modeldata.Parse(raw) + if err != nil { + t.Fatalf("Parse() error = %v", err) + } + registry.SetModelList(list, raw) + registry.EnrichModels() + + info := registry.GetModel("gemma-3-4b-it") + if info == nil || info.Model.Metadata == nil { + t.Fatal("expected the model to stay registered with metadata") + } + meta := info.Model.Metadata + if meta.ContextWindow == nil || *meta.ContextWindow != runtimeContext { + t.Fatalf("ContextWindow = %v, want the server-reported %d", meta.ContextWindow, runtimeContext) + } + if !meta.Capabilities["vision"] { + t.Fatal("discovered vision capability was dropped by enrichment") + } + // The catalog still fills in what the server never reports. + if meta.DisplayName != "Gemma 3 4B IT" { + t.Fatalf("DisplayName = %q, want the catalog's", meta.DisplayName) + } + if len(meta.Modes) != 1 || meta.Modes[0] != "chat" { + t.Fatalf("Modes = %v, want [chat] from the catalog", meta.Modes) + } +}