diff --git a/Cargo.lock b/Cargo.lock index 3057ad9f..982556c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2946,9 +2946,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "log", diff --git a/crates/openplotva-app/src/dialog_runtime.rs b/crates/openplotva-app/src/dialog_runtime.rs index 65339997..bf1e6a38 100644 --- a/crates/openplotva-app/src/dialog_runtime.rs +++ b/crates/openplotva-app/src/dialog_runtime.rs @@ -374,6 +374,8 @@ impl ChatStepProvider for RouterChatProvider { { step_request.input.enable_thinking = Some(enable_thinking); } + step_request.input.sampling = + dialog_sampling_from_overrides(&attempt.overrides); step_request.input.bad_words = openplotva_llm::aifarm::resolve_bad_words( attempt.overrides.extra.get("bad_words"), ); @@ -574,6 +576,24 @@ fn local_reasoner_dialog_config_from_app_config(config: &AppConfig) -> AifarmDia cfg.with_defaults() } +/// Sampling a route sets for its turns, from the merged model and assignment config. +/// A value outside its valid range is ignored instead of failing the request. +fn dialog_sampling_from_overrides( + overrides: &openplotva_llm::router::InferenceOverrides, +) -> openplotva_dialog::DialogSampling { + let extra = |key: &str| overrides.extra.get(key).and_then(serde_json::Value::as_f64); + let within = |value: Option, valid: fn(f64) -> bool| value.filter(|v| valid(*v)); + openplotva_dialog::DialogSampling { + temperature: within(overrides.temperature, |v| (0.0..=2.0).contains(&v)), + top_p: within(extra("top_p"), |v| v > 0.0 && v <= 1.0), + top_k: within(extra("top_k"), |v| v >= 0.0), + min_p: within(extra("min_p"), |v| (0.0..1.0).contains(&v)), + repeat_penalty: within(overrides.repeat_penalty, |v| v > 0.0), + frequency_penalty: within(overrides.frequency_penalty, |v| (-2.0..=2.0).contains(&v)), + presence_penalty: within(overrides.presence_penalty, |v| (-2.0..=2.0).contains(&v)), + } +} + fn aifarm_discovery_provider_name(service: &str, config: &AppConfig) -> String { let service = service.trim(); if service.is_empty() || service == config.llm.dialog.discovery_service_name { @@ -1253,6 +1273,65 @@ mod tests { assert_eq!(inputs[0].max_output_tokens, 123); } + #[tokio::test] + async fn route_sampling_reaches_the_turn_and_ignores_invalid_values() { + let run = |config: serde_json::Value| async move { + let default_provider = Arc::new(SequencedProvider::new("unused", vec![])); + let routed_provider = Arc::new(SequencedProvider::new( + "aifarm", + vec![Ok(DialogOutput { + provider: "aifarm".to_owned(), + answer: "ok".to_owned(), + ..DialogOutput::default() + })], + )); + let routed_provider_dyn: DialogProviderHandle = routed_provider.clone(); + let mut clients = HashMap::new(); + clients.insert("aifarm".to_owned(), routed_provider_dyn); + let mut snapshot = routed_dialog_snapshot(); + snapshot.models[0].config = config; + let provider = router_provider_for_test( + snapshot, + default_provider, + clients, + crate::runtime_routing::RoutingEventReporter::new( + crate::runtime_routing::RoutingEventBuffer::new(8), + None, + None, + ), + ); + provider + .as_chat_step() + .expect("step seam") + .run_chat_step(default_step_request()) + .await + .expect("dialog output"); + routed_provider.inputs()[0].sampling.clone() + }; + + let card = run(json!({ + "temperature": 0.6, "top_p": 0.9, "top_k": 64, "min_p": 0.05, "repeat_penalty": 1.1 + })) + .await; + assert_eq!( + card, + openplotva_dialog::DialogSampling { + temperature: Some(0.6), + top_p: Some(0.9), + top_k: Some(64.0), + min_p: Some(0.05), + repeat_penalty: Some(1.1), + ..openplotva_dialog::DialogSampling::default() + } + ); + assert!(run(json!({})).await.is_empty()); + let invalid = run(json!({ + "temperature": 9.0, "top_p": 0.0, "top_k": -1, "min_p": 1.0, "repeat_penalty": 0.0 + })) + .await; + assert!(invalid.is_empty(), "{invalid:?}"); + } + #[tokio::test] async fn model_config_lets_its_model_execute_textual_tool_calls() { let run = |config: serde_json::Value| async move { diff --git a/crates/openplotva-dialog/src/history.rs b/crates/openplotva-dialog/src/history.rs index 5058f95d..4f7f6d45 100644 --- a/crates/openplotva-dialog/src/history.rs +++ b/crates/openplotva-dialog/src/history.rs @@ -264,6 +264,14 @@ pub struct DialogInput { skip_serializing_if = "std::ops::Not::not" )] pub textual_tool_calls: bool, + /// Sampling the routing config of the serving model sets for this turn; unset + /// fields keep the provider's configured value. + #[serde( + default, + rename = "Sampling", + skip_serializing_if = "DialogSampling::is_empty" + )] + pub sampling: DialogSampling, /// Verdict that rejected the previous sample of this turn, as a stable code /// (`context_leak`, `empty`, …); empty on the first attempt. The message builder /// turns it into a short note after the rendered last message, so a re-sample is @@ -281,6 +289,48 @@ pub struct DialogInput { pub context_capture: Option, } +/// Per-turn sampling chosen by the routing config of the model that serves it. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +pub struct DialogSampling { + #[serde( + default, + rename = "Temperature", + skip_serializing_if = "Option::is_none" + )] + pub temperature: Option, + #[serde(default, rename = "TopP", skip_serializing_if = "Option::is_none")] + pub top_p: Option, + #[serde(default, rename = "TopK", skip_serializing_if = "Option::is_none")] + pub top_k: Option, + #[serde(default, rename = "MinP", skip_serializing_if = "Option::is_none")] + pub min_p: Option, + #[serde( + default, + rename = "RepeatPenalty", + skip_serializing_if = "Option::is_none" + )] + pub repeat_penalty: Option, + #[serde( + default, + rename = "FrequencyPenalty", + skip_serializing_if = "Option::is_none" + )] + pub frequency_penalty: Option, + #[serde( + default, + rename = "PresencePenalty", + skip_serializing_if = "Option::is_none" + )] + pub presence_penalty: Option, +} + +impl DialogSampling { + #[must_use] + pub fn is_empty(&self) -> bool { + *self == Self::default() + } +} + /// One memory card that surfaced during recall for a turn — the scored skeleton /// kept for the admin X-ray, not the full card body. #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] diff --git a/crates/openplotva-dialog/src/lib.rs b/crates/openplotva-dialog/src/lib.rs index 9a28fa33..2b82f569 100644 --- a/crates/openplotva-dialog/src/lib.rs +++ b/crates/openplotva-dialog/src/lib.rs @@ -20,11 +20,11 @@ pub use leak_guard::{DialogLeak, ReplyLeakGuard, ReplyLeakGuardBuilder}; pub use history::{ CapturedMemory, DEFAULT_CONTEXT_HISTORY_LIMIT, DailyPersona, DialogContext, DialogInput, - DialogMessage, DialogOutput, DialogTraceArtifacts, DialogTraceUsage, DialogUser, - HistoryMessage, MESSAGE_KIND_TEXT, MESSAGE_KIND_TOOL_REQUEST, MESSAGE_KIND_TOOL_RESPONSE, - MessageKind, MultimodalImage, Persona, PersonaSnapshot, ROLE_MODEL, ROLE_TOOL, ROLE_USER, - SettingKv, TurnContextArtifact, clone_history_messages, conversation_projection, - history_message_has_context_content, history_meta_has_context_content, + DialogMessage, DialogOutput, DialogSampling, DialogTraceArtifacts, DialogTraceUsage, + DialogUser, HistoryMessage, MESSAGE_KIND_TEXT, MESSAGE_KIND_TOOL_REQUEST, + MESSAGE_KIND_TOOL_RESPONSE, MessageKind, MultimodalImage, Persona, PersonaSnapshot, ROLE_MODEL, + ROLE_TOOL, ROLE_USER, SettingKv, TurnContextArtifact, clone_history_messages, + conversation_projection, history_message_has_context_content, history_meta_has_context_content, normalize_history_message, normalize_history_message_kind, select_history_messages_for_context, select_llm_history_messages_for_context, unique_history_text, }; diff --git a/crates/openplotva-llm/src/aifarm.rs b/crates/openplotva-llm/src/aifarm.rs index 59da1551..90c8ec32 100644 --- a/crates/openplotva-llm/src/aifarm.rs +++ b/crates/openplotva-llm/src/aifarm.rs @@ -20,11 +20,11 @@ use url::Url; use openplotva_core::{ChatAttachment, ChatMessageMeta}; use openplotva_dialog::{ ChatStepOutput, ChatStepRequest, ChatStepToolCall, DEFAULT_CONTEXT_HISTORY_LIMIT, DialogInput, - DialogTraceArtifacts, DialogTraceError, DialogTraceUsage, HistoryMessage, MESSAGE_KIND_TEXT, - MESSAGE_KIND_TOOL_REQUEST, MESSAGE_KIND_TOOL_RESPONSE, NativeToolCall, NativeToolFunction, - PROVIDER_AIFARM, PROVIDER_NVIDIA, PROVIDER_VMLX, ROLE_MODEL, ROLE_USER, ReplyLeakGuard, - STEP_REACT_TO_MESSAGE, STEP_SEND_MESSAGE, SessionMessage, ToolParseDecision, ToolStep, - ToolsMode, clone_history_messages, decode_plotva_final_response_with_salvage, + DialogSampling, DialogTraceArtifacts, DialogTraceError, DialogTraceUsage, HistoryMessage, + MESSAGE_KIND_TEXT, MESSAGE_KIND_TOOL_REQUEST, MESSAGE_KIND_TOOL_RESPONSE, NativeToolCall, + NativeToolFunction, PROVIDER_AIFARM, PROVIDER_NVIDIA, PROVIDER_VMLX, ROLE_MODEL, ROLE_USER, + ReplyLeakGuard, STEP_REACT_TO_MESSAGE, STEP_SEND_MESSAGE, SessionMessage, ToolParseDecision, + ToolStep, ToolsMode, clone_history_messages, decode_plotva_final_response_with_salvage, is_dialog_history_noise_tool_call_name, normalize_history_message, parse_assistant_content, parse_native_tool_step, reply_shape, sanitize_tool_text, select_llm_history_messages_for_context, tool_telemetry, @@ -206,6 +206,9 @@ pub struct ChatCompletionRequest { serialize_with = "serialize_top_k" )] pub top_k: Option, + /// Minimum token probability relative to the most likely token. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub min_p: Option, /// llama.cpp repeat penalty. #[serde(default, skip_serializing_if = "Option::is_none")] pub repeat_penalty: Option, @@ -306,6 +309,7 @@ impl Serialize for RedactedChatCompletionRequest<'_> { if let Some(top_k) = request.top_k { state.serialize_field("top_k", &top_k_wire_value(top_k))?; } + serialize_optional_field(&mut state, "min_p", request.min_p)?; serialize_optional_field(&mut state, "repeat_penalty", request.repeat_penalty)?; serialize_optional_field(&mut state, "repetition_penalty", request.repetition_penalty)?; serialize_optional_field(&mut state, "frequency_penalty", request.frequency_penalty)?; @@ -3300,6 +3304,7 @@ where include_reasoning: self.cfg.include_reasoning, ..ChatCompletionRequest::default() }; + apply_dialog_sampling(&mut request, &input.sampling); if let ToolsMode::Native(tool_values) = tools && !tool_values.is_empty() { @@ -3853,6 +3858,7 @@ fn aifarm_trace_inference_params(request: &ChatCompletionRequest) -> Option Result<(), CompletionError> + { + let cfg = AifarmDialogConfig { + temperature: Some(0.2), + top_p: Some(0.95), + frequency_penalty: Some(0.3), + presence_penalty: Some(0.2), + ..AifarmDialogConfig::default() + }; + let mut bodies = Vec::new(); + for sampling in [ + DialogSampling::default(), + DialogSampling { + temperature: Some(0.6), + top_p: Some(0.9), + top_k: Some(64.0), + min_p: Some(0.05), + repeat_penalty: Some(1.1), + ..DialogSampling::default() + }, + ] { + let (provider, transport, _) = direct_dialog_provider( + json!({ "choices": [{ "message": { "role": "assistant", "content": "ага" } }] }), + cfg.clone(), + ); + let mut input = base_input(); + input.sampling = sampling; + crate::ChatStepProvider::run_chat_step( + &provider, + openplotva_dialog::ChatStepRequest { + required_tool: None, + preferred_target: None, + input, + transcript: Vec::new(), + tools: openplotva_dialog::ToolsMode::Disabled, + iteration: 1, + }, + ) + .await + .expect("dialog step"); + let sent = transport.requests(); + bodies.push(serde_json::from_slice::(&sent[0].body).expect("wire request")); + } + + let [configured, routed] = &bodies[..] else { + unreachable!("two requests were sent"); + }; + assert_eq!(configured["temperature"], json!(0.2)); + assert_eq!(configured["top_p"], json!(0.95)); + assert!(configured.get("min_p").is_none()); + assert!(configured.get("top_k").is_none()); + assert_eq!(routed["temperature"], json!(0.6)); + assert_eq!(routed["top_p"], json!(0.9)); + assert_eq!(routed["top_k"], json!(64)); + assert_eq!(routed["min_p"], json!(0.05)); + assert_eq!(routed["repeat_penalty"], json!(1.1)); + assert_eq!( + routed["frequency_penalty"], + json!(0.3), + "unset fields keep the configured value" + ); + assert_eq!(routed["presence_penalty"], json!(0.2)); + Ok(()) + } + #[tokio::test] async fn dialog_request_omits_bad_words_when_the_model_has_none() -> Result<(), CompletionError> { diff --git a/crates/openplotva-storage/src/lib.rs b/crates/openplotva-storage/src/lib.rs index 7ffe738a..f389966c 100644 --- a/crates/openplotva-storage/src/lib.rs +++ b/crates/openplotva-storage/src/lib.rs @@ -10256,6 +10256,44 @@ mod tests { assert!(DOWN.contains("WHERE config ? 'backend_lanes_v1_previous_name'")); } + #[test] + fn gemma_card_sampling_is_scoped_to_the_dialog_assignment_and_reversible() { + const UP: &str = include_str!("../../../migrations/208_gemma_card_dialog_sampling.up.sql"); + const DOWN: &str = + include_str!("../../../migrations/208_gemma_card_dialog_sampling.down.sql"); + + for value in [ + "UPDATE workflow_assignments AS assignment", + "assignment.workflow_key = 'dialog'", + "provider.name = 'aifarm-vllm-gpu0'", + "model.model_name = 'Gemma 4 26B Heretic'", + r#"'{"temperature": 0.6, "top_p": 0.9, "top_k": 64, "min_p": 0.05, "repeat_penalty": 1.1}'::jsonb"#, + "NOT assignment.inference_overrides ? 'gemma_card_sampling_v1_previous'", + ] { + assert!( + UP.contains(value), + "missing Gemma sampling contract {value}" + ); + } + assert!(!UP.contains("provider_models SET")); + assert!(!UP.contains("INSERT INTO")); + assert!(!UP.contains("DELETE FROM")); + for key in [ + "'temperature'", + "'top_p'", + "'top_k'", + "'min_p'", + "'repeat_penalty'", + ] { + assert!(DOWN.contains(key), "down migration must restore {key}"); + } + assert!( + DOWN.contains( + "WHERE assignment.inference_overrides ? 'gemma_card_sampling_v1_previous'" + ) + ); + } + #[test] fn translation_workflow_routes_like_dialog_and_is_reversible() { const UP: &str = include_str!("../../../migrations/207_translation_workflow.up.sql"); diff --git a/migrations/208_gemma_card_dialog_sampling.down.sql b/migrations/208_gemma_card_dialog_sampling.down.sql new file mode 100644 index 00000000..9f7c6dda --- /dev/null +++ b/migrations/208_gemma_card_dialog_sampling.down.sql @@ -0,0 +1,16 @@ +-- Restore each sampling key from the marker the up migration saved: a key that was absent +-- before is removed, a key that had a value gets it back. +UPDATE workflow_assignments AS assignment +SET inference_overrides = ( + SELECT COALESCE(jsonb_object_agg(key, value) FILTER (WHERE value <> 'null'::jsonb), '{}'::jsonb) + FROM jsonb_each(assignment.inference_overrides->'gemma_card_sampling_v1_previous') +) || ( + assignment.inference_overrides + - 'gemma_card_sampling_v1_previous' + - 'temperature' + - 'top_p' + - 'top_k' + - 'min_p' + - 'repeat_penalty' +) +WHERE assignment.inference_overrides ? 'gemma_card_sampling_v1_previous'; diff --git a/migrations/208_gemma_card_dialog_sampling.up.sql b/migrations/208_gemma_card_dialog_sampling.up.sql new file mode 100644 index 00000000..bf075c04 --- /dev/null +++ b/migrations/208_gemma_card_dialog_sampling.up.sql @@ -0,0 +1,27 @@ +-- Dialog turns served by the GPU0 Gemma take the sampling its model card recommends: +-- temperature 0.6, top_k 64, top_p 0.9, min_p 0.05, repeat_penalty 1.1. The values sit on the +-- dialog assignment of that model, so the Qwen and GPT fallbacks, the history, optimizer and +-- other Gemma workflows keep their own sampling. NInfer clamps top_k to 20 candidates and the +-- farm proxy drops repeat_penalty; both stay in the config as the card states them. +-- Each replaced key is saved under one marker so the down migration restores it exactly. +UPDATE workflow_assignments AS assignment +SET inference_overrides = assignment.inference_overrides + || jsonb_build_object( + 'gemma_card_sampling_v1_previous', + jsonb_build_object( + 'temperature', COALESCE(assignment.inference_overrides->'temperature', 'null'::jsonb), + 'top_p', COALESCE(assignment.inference_overrides->'top_p', 'null'::jsonb), + 'top_k', COALESCE(assignment.inference_overrides->'top_k', 'null'::jsonb), + 'min_p', COALESCE(assignment.inference_overrides->'min_p', 'null'::jsonb), + 'repeat_penalty', COALESCE(assignment.inference_overrides->'repeat_penalty', 'null'::jsonb) + ) + ) + || '{"temperature": 0.6, "top_p": 0.9, "top_k": 64, "min_p": 0.05, "repeat_penalty": 1.1}'::jsonb +FROM provider_models AS model, + llm_providers AS provider +WHERE model.id = assignment.provider_model_id + AND provider.id = model.provider_id + AND assignment.workflow_key = 'dialog' + AND provider.name = 'aifarm-vllm-gpu0' + AND model.model_name = 'Gemma 4 26B Heretic' + AND NOT assignment.inference_overrides ? 'gemma_card_sampling_v1_previous';