diff --git a/docs/llms/model-tuning.md b/docs/llms/model-tuning.md index 1d97135a5..1116cb891 100644 --- a/docs/llms/model-tuning.md +++ b/docs/llms/model-tuning.md @@ -107,11 +107,11 @@ bare `qwen3.8` does not find `qwen3.8:latest`. Write the tag you run. ## The settings -Every one is optional. What you do not state resolves to the compiled default in the last column. +Every one is optional. What you do not state resolves to the compiled default in the last column, with one exception: entry-level `sampling` is read as a complete statement, so a key left out of it does not fall back to the default (for example, `sampling: { temperature: 0 }` omits `topP`, and `sampling: {}` sends neither parameter). | setting | type and bounds | what it decides | default | | --- | --- | --- | --- | -| `sampling` | `{temperature: 0–2, topP: 0–1}` | how every turn of this model is sampled | `{0, 1}` | +| `sampling` | `{temperature?: 0–2, topP?: 0–1}` | how every turn of this model is sampled | `{temperature: 0, topP: 1}` | | `perWorkflow` | the same object, per workflow id | sampling for named surfaces only — the narrowest an override gets | — | | `unreportedCallCeiling` | integer 1–100 | how many calls it may make without reporting before the run is narrowed to the tools that would finish it | `12` | | `reportReminderLimit` | integer 0–5 | how many times a turn with no call and no report may be answered with the report reminder | `1` | @@ -127,7 +127,10 @@ Every one is optional. What you do not state resolves to the compiled default in | `threadContextMaxChars` | integer 200–32000 | how much of a CONVERSATION this model may be handed — the earlier steps' objectives and the most recent step's report, when a follow-up continues a previous run | the product's budget (4000) | Workflow ids for `perWorkflow`: `investigation`, `query-optimization`, `database-assessment`, -`operations`, `data-analysis`. +`operations`, `data-analysis`. Note that `perWorkflow` merges onto the model's entry-level `sampling` +(or the compiled `{temperature: 0, topP: 1}` default if omitted). For endpoints that require omitting +`topP` (such as Anthropic/Claude), state `sampling: { temperature: ... }` at the entry level so `topP` +is not inherited from the default. **`threadContextMaxChars` is the one setting Studio ships NO measurement for**, and that is deliberate rather than an omission: no entry in the shipped document names it, because nobody has diff --git a/src/lib/agent/investigation.ts b/src/lib/agent/investigation.ts index c966c46de..78ff2d793 100644 --- a/src/lib/agent/investigation.ts +++ b/src/lib/agent/investigation.ts @@ -2392,8 +2392,8 @@ async function takeTurn( mode === "agent" ? suppressesAgentReasoning(agentModel.modelId) : suppressesPlanReasoning(agentModel.modelId); const stream = streamText({ model: agentModel.model, - temperature: sampling.temperature, - topP: sampling.topP, + ...(sampling.temperature !== undefined ? { temperature: sampling.temperature } : {}), + ...(sampling.topP !== undefined ? { topP: sampling.topP } : {}), // Constrained decoding, where a shape was asked for. `Output.object` is what makes the // SDK send `response_format`, and it composes here precisely because this branch offers // no tools. diff --git a/src/lib/agent/model-tuning/schema.ts b/src/lib/agent/model-tuning/schema.ts index 6df204fd3..5f178c233 100644 --- a/src/lib/agent/model-tuning/schema.ts +++ b/src/lib/agent/model-tuning/schema.ts @@ -55,8 +55,8 @@ const WORKFLOWS = [ ] as const satisfies readonly AgentRunWorkflowType[]; const samplingSchema = z.strictObject({ - temperature: z.number().min(0).max(2), - topP: z.number().min(0).max(1), + temperature: z.number().min(0).max(2).optional(), + topP: z.number().min(0).max(1).optional(), }); /* diff --git a/src/lib/agent/models/index.ts b/src/lib/agent/models/index.ts index 1aa599b49..6ec2b84c3 100644 --- a/src/lib/agent/models/index.ts +++ b/src/lib/agent/models/index.ts @@ -263,5 +263,6 @@ export function offersRefusalExamples(modelId: string): boolean { export function samplingFor(modelId: string, workflow: AgentRunWorkflowType | undefined): AgentSampling { const own = entryFor(modelId); const ownSurface = workflow === undefined ? undefined : own?.perWorkflow?.[workflow]; - return { ...DEFAULT_SAMPLING, ...own?.sampling, ...ownSurface }; + const base = own?.sampling ?? DEFAULT_SAMPLING; + return ownSurface ? { ...base, ...ownSurface } : base; } diff --git a/src/lib/agent/models/profile.ts b/src/lib/agent/models/profile.ts index 39e93a9b6..3546879a5 100644 --- a/src/lib/agent/models/profile.ts +++ b/src/lib/agent/models/profile.ts @@ -10,8 +10,8 @@ import type { AgentRunWorkflowType } from "../types"; /** How a turn is sampled. Structural output, so the default explores nothing. */ export interface AgentSampling { - readonly temperature: number; - readonly topP: number; + readonly temperature?: number; + readonly topP?: number; } export interface AgentModelProfile { diff --git a/tests/unit/lib/agent/model-profiles.test.ts b/tests/unit/lib/agent/model-profiles.test.ts index 264a8350f..c02582530 100644 --- a/tests/unit/lib/agent/model-profiles.test.ts +++ b/tests/unit/lib/agent/model-profiles.test.ts @@ -1,19 +1,24 @@ import { describe, expect, test } from "bun:test"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { AGENT_THREAD_CONTEXT_MAX_CHARS } from "@/lib/agent/execution-policy"; +import { resetTuning } from "@/lib/agent/model-tuning"; import { - modelProfiles, + answersUnreadStop, ceilingFor, + modelProfiles, + planStatementRetriesFor, presentReminderLimitFor, + reportReminderLimitFor, retriesEmptyTurn, retriesUnreadStop, - answersUnreadStop, - turnTimeoutMsFor, - planStatementRetriesFor, - reportReminderLimitFor, samplingFor, threadContextMaxCharsFor, + turnTimeoutMsFor, verdictHoldLimitFor, } from "@/lib/agent/models"; -import { AGENT_THREAD_CONTEXT_MAX_CHARS } from "@/lib/agent/execution-policy"; +import { DEFAULT_SAMPLING } from "@/lib/agent/models/profile"; import type { AgentRunWorkflowType } from "@/lib/agent/types"; /** @@ -43,6 +48,12 @@ const WORKFLOWS: readonly AgentRunWorkflowType[] = [ "data-analysis", ]; +const writeDocument = (body: unknown): string => { + const path = join(mkdtempSync(join(tmpdir(), "libredb-tuning-")), "models.json"); + writeFileSync(path, typeof body === "string" ? body : JSON.stringify(body)); + return path; +}; + describe("sampling is decided per model, defaulting to deterministic", () => { test("a model nobody has measured gets the default, on every workflow", () => { for (const workflow of WORKFLOWS) { @@ -54,7 +65,9 @@ describe("sampling is decided per model, defaulting to deterministic", () => { // A cell locks only at 5/5, so the bar is a variance test as much as a capability one, // and choosing a tool is a structural task with nothing for a sample to explore. This is // the setting that won five cells. + expect(DEFAULT_SAMPLING).toEqual({ temperature: 0, topP: 1 }); expect(samplingFor("gemma4:26b", "database-assessment")).toEqual({ temperature: 0, topP: 1 }); + expect(samplingFor("qwen3:8b", "database-assessment")).toEqual({ temperature: 0, topP: 1 }); }); test("qwen3:8b is sampled on query-optimization, and nowhere else", async () => { @@ -64,10 +77,41 @@ describe("sampling is decided per model, defaulting to deterministic", () => { the override is scoped to the one cell that needs it rather than to the model. */ expect(samplingFor("qwen3:8b", "query-optimization").temperature).toBeGreaterThan(0); - expect(samplingFor("qwen3:8b", "database-assessment")).toEqual({ temperature: 0, topP: 1 }); expect(samplingFor("qwen3:8b", "investigation")).toEqual({ temperature: 0, topP: 1 }); }); + test("samplingFor over an operator-supplied temperature-only entry resolves without topP", () => { + // Verified against Anthropic/Claude endpoint compatibility: sending both temperature and topP + // causes Claude models to reject with 400. + const path = writeDocument({ + schemaVersion: 1, + models: [ + { id: "claude-haiku-4-5", measured: "temp only", settings: { sampling: { temperature: 0 } } }, + { + id: "claude-custom-workflow", + measured: "workflow with entry sampling", + settings: { sampling: { temperature: 0 }, perWorkflow: { investigation: { temperature: 0.5 } } }, + }, + { + id: "claude-per-workflow-only", + measured: "workflow only inherits default topP", + settings: { perWorkflow: { investigation: { temperature: 0.5 } } }, + }, + ], + }); + process.env.AGENT_MODEL_TUNING_PATH = path; + resetTuning(); + try { + expect(samplingFor("claude-haiku-4-5", "investigation")).toEqual({ temperature: 0 }); + expect(samplingFor("claude-haiku-4-5", undefined)).toEqual({ temperature: 0 }); + expect(samplingFor("claude-custom-workflow", "investigation")).toEqual({ temperature: 0.5 }); + expect(samplingFor("claude-per-workflow-only", "investigation")).toEqual({ temperature: 0.5, topP: 1 }); + } finally { + delete process.env.AGENT_MODEL_TUNING_PATH; + resetTuning(); + } + }); + test("a model id is matched case-insensitively, and its TAG is not stripped", () => { /* Two facts, and the second is the one the old name got wrong. This used to be called "a tag diff --git a/tests/unit/lib/agent/model-tuning.test.ts b/tests/unit/lib/agent/model-tuning.test.ts index ca1416b93..881805fa8 100644 --- a/tests/unit/lib/agent/model-tuning.test.ts +++ b/tests/unit/lib/agent/model-tuning.test.ts @@ -239,6 +239,41 @@ describe("what a document from outside Studio is held to instead", () => { expect(retriesEmptyTurn("gemma4:26b")).toBe(true); }); + test("an entry may state temperature without topP, suitable for Claude/Anthropic endpoints", () => { + const tempOnly = { + models: [{ id: "claude-haiku-4-5", measured: "temperature 0 only", settings: { sampling: { temperature: 0 } } }], + }; + const tuning = parseOperatorTuning(document(tempOnly), "test"); + expect(tuning.models["claude-haiku-4-5"]).toEqual({ + measured: "temperature 0 only", + sampling: { temperature: 0 }, + }); + }); + + test("an entry may state topP without temperature", () => { + const topPOnly = { + models: [{ id: "custom-model:7b", measured: "topP 0.9 only", settings: { sampling: { topP: 0.9 } } }], + }; + const tuning = parseOperatorTuning(document(topPOnly), "test"); + expect(tuning.models["custom-model:7b"]).toEqual({ + measured: "topP 0.9 only", + sampling: { topP: 0.9 }, + }); + }); + + test("an entry may state an empty sampling object for adaptive-thinking models", () => { + const emptySampling = { + models: [ + { id: "claude-sonnet-5", measured: "adaptive thinking; no sampling params", settings: { sampling: {} } }, + ], + }; + const tuning = parseOperatorTuning(document(emptySampling), "test"); + expect(tuning.models["claude-sonnet-5"]).toEqual({ + measured: "adaptive thinking; no sampling params", + sampling: {}, + }); + }); + test("a key this Studio does not implement is reported rather than refusing the document", () => { const misspelled = { models: [{ id: "some-model:9b", measured: "m", settings: { retryEmtpyTurn: true } }],