From ba6fc3210c37a53ba9f48a7471d7e6e14c6c7070 Mon Sep 17 00:00:00 2001 From: Valeriy Selitskiy <239034+iamwavecut@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:18:35 +0200 Subject: [PATCH 1/2] Sample Gemma dialog turns as the model card recommends A route can now set the sampling of its dialog turns: temperature, top_p, top_k, min_p, repeat_penalty and the two OpenAI penalties come from the merged model and assignment config, replace the provider's configured value field by field, and are ignored when out of range. The chat request gains min_p. Migration 208 sets the card values (temperature 0.6, top_p 0.9, top_k 64, min_p 0.05, repeat_penalty 1.1) on the dialog assignment of the GPU0 Gemma model only. The fallbacks and the other Gemma workflows keep their own sampling. NInfer clamps top_k to 20 candidates and the serving proxy drops repeat_penalty. The down migration restores the previous keys exactly. --- crates/openplotva-app/src/dialog_runtime.rs | 79 +++++++++++++++ crates/openplotva-dialog/src/history.rs | 50 ++++++++++ crates/openplotva-dialog/src/lib.rs | 10 +- crates/openplotva-llm/src/aifarm.rs | 95 ++++++++++++++++++- crates/openplotva-storage/src/lib.rs | 38 ++++++++ .../208_gemma_card_dialog_sampling.down.sql | 16 ++++ .../208_gemma_card_dialog_sampling.up.sql | 27 ++++++ 7 files changed, 305 insertions(+), 10 deletions(-) create mode 100644 migrations/208_gemma_card_dialog_sampling.down.sql create mode 100644 migrations/208_gemma_card_dialog_sampling.up.sql diff --git a/crates/openplotva-app/src/dialog_runtime.rs b/crates/openplotva-app/src/dialog_runtime.rs index 65339997..bf1e6a38 100644 --- a/crates/openplotva-app/src/dialog_runtime.rs +++ b/crates/openplotva-app/src/dialog_runtime.rs @@ -374,6 +374,8 @@ impl ChatStepProvider for RouterChatProvider { { step_request.input.enable_thinking = Some(enable_thinking); } + step_request.input.sampling = + dialog_sampling_from_overrides(&attempt.overrides); step_request.input.bad_words = openplotva_llm::aifarm::resolve_bad_words( attempt.overrides.extra.get("bad_words"), ); @@ -574,6 +576,24 @@ fn local_reasoner_dialog_config_from_app_config(config: &AppConfig) -> AifarmDia cfg.with_defaults() } +/// Sampling a route sets for its turns, from the merged model and assignment config. +/// A value outside its valid range is ignored instead of failing the request. +fn dialog_sampling_from_overrides( + overrides: &openplotva_llm::router::InferenceOverrides, +) -> openplotva_dialog::DialogSampling { + let extra = |key: &str| overrides.extra.get(key).and_then(serde_json::Value::as_f64); + let within = |value: Option, valid: fn(f64) -> bool| value.filter(|v| valid(*v)); + openplotva_dialog::DialogSampling { + temperature: within(overrides.temperature, |v| (0.0..=2.0).contains(&v)), + top_p: within(extra("top_p"), |v| v > 0.0 && v <= 1.0), + top_k: within(extra("top_k"), |v| v >= 0.0), + min_p: within(extra("min_p"), |v| (0.0..1.0).contains(&v)), + repeat_penalty: within(overrides.repeat_penalty, |v| v > 0.0), + frequency_penalty: within(overrides.frequency_penalty, |v| (-2.0..=2.0).contains(&v)), + presence_penalty: within(overrides.presence_penalty, |v| (-2.0..=2.0).contains(&v)), + } +} + fn aifarm_discovery_provider_name(service: &str, config: &AppConfig) -> String { let service = service.trim(); if service.is_empty() || service == config.llm.dialog.discovery_service_name { @@ -1253,6 +1273,65 @@ mod tests { assert_eq!(inputs[0].max_output_tokens, 123); } + #[tokio::test] + async fn route_sampling_reaches_the_turn_and_ignores_invalid_values() { + let run = |config: serde_json::Value| async move { + let default_provider = Arc::new(SequencedProvider::new("unused", vec![])); + let routed_provider = Arc::new(SequencedProvider::new( + "aifarm", + vec![Ok(DialogOutput { + provider: "aifarm".to_owned(), + answer: "ok".to_owned(), + ..DialogOutput::default() + })], + )); + let routed_provider_dyn: DialogProviderHandle = routed_provider.clone(); + let mut clients = HashMap::new(); + clients.insert("aifarm".to_owned(), routed_provider_dyn); + let mut snapshot = routed_dialog_snapshot(); + snapshot.models[0].config = config; + let provider = router_provider_for_test( + snapshot, + default_provider, + clients, + crate::runtime_routing::RoutingEventReporter::new( + crate::runtime_routing::RoutingEventBuffer::new(8), + None, + None, + ), + ); + provider + .as_chat_step() + .expect("step seam") + .run_chat_step(default_step_request()) + .await + .expect("dialog output"); + routed_provider.inputs()[0].sampling.clone() + }; + + let card = run(json!({ + "temperature": 0.6, "top_p": 0.9, "top_k": 64, "min_p": 0.05, "repeat_penalty": 1.1 + })) + .await; + assert_eq!( + card, + openplotva_dialog::DialogSampling { + temperature: Some(0.6), + top_p: Some(0.9), + top_k: Some(64.0), + min_p: Some(0.05), + repeat_penalty: Some(1.1), + ..openplotva_dialog::DialogSampling::default() + } + ); + assert!(run(json!({})).await.is_empty()); + let invalid = run(json!({ + "temperature": 9.0, "top_p": 0.0, "top_k": -1, "min_p": 1.0, "repeat_penalty": 0.0 + })) + .await; + assert!(invalid.is_empty(), "{invalid:?}"); + } + #[tokio::test] async fn model_config_lets_its_model_execute_textual_tool_calls() { let run = |config: serde_json::Value| async move { diff --git a/crates/openplotva-dialog/src/history.rs b/crates/openplotva-dialog/src/history.rs index 5058f95d..4f7f6d45 100644 --- a/crates/openplotva-dialog/src/history.rs +++ b/crates/openplotva-dialog/src/history.rs @@ -264,6 +264,14 @@ pub struct DialogInput { skip_serializing_if = "std::ops::Not::not" )] pub textual_tool_calls: bool, + /// Sampling the routing config of the serving model sets for this turn; unset + /// fields keep the provider's configured value. + #[serde( + default, + rename = "Sampling", + skip_serializing_if = "DialogSampling::is_empty" + )] + pub sampling: DialogSampling, /// Verdict that rejected the previous sample of this turn, as a stable code /// (`context_leak`, `empty`, …); empty on the first attempt. The message builder /// turns it into a short note after the rendered last message, so a re-sample is @@ -281,6 +289,48 @@ pub struct DialogInput { pub context_capture: Option, } +/// Per-turn sampling chosen by the routing config of the model that serves it. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +pub struct DialogSampling { + #[serde( + default, + rename = "Temperature", + skip_serializing_if = "Option::is_none" + )] + pub temperature: Option, + #[serde(default, rename = "TopP", skip_serializing_if = "Option::is_none")] + pub top_p: Option, + #[serde(default, rename = "TopK", skip_serializing_if = "Option::is_none")] + pub top_k: Option, + #[serde(default, rename = "MinP", skip_serializing_if = "Option::is_none")] + pub min_p: Option, + #[serde( + default, + rename = "RepeatPenalty", + skip_serializing_if = "Option::is_none" + )] + pub repeat_penalty: Option, + #[serde( + default, + rename = "FrequencyPenalty", + skip_serializing_if = "Option::is_none" + )] + pub frequency_penalty: Option, + #[serde( + default, + rename = "PresencePenalty", + skip_serializing_if = "Option::is_none" + )] + pub presence_penalty: Option, +} + +impl DialogSampling { + #[must_use] + pub fn is_empty(&self) -> bool { + *self == Self::default() + } +} + /// One memory card that surfaced during recall for a turn — the scored skeleton /// kept for the admin X-ray, not the full card body. #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] diff --git a/crates/openplotva-dialog/src/lib.rs b/crates/openplotva-dialog/src/lib.rs index 9a28fa33..2b82f569 100644 --- a/crates/openplotva-dialog/src/lib.rs +++ b/crates/openplotva-dialog/src/lib.rs @@ -20,11 +20,11 @@ pub use leak_guard::{DialogLeak, ReplyLeakGuard, ReplyLeakGuardBuilder}; pub use history::{ CapturedMemory, DEFAULT_CONTEXT_HISTORY_LIMIT, DailyPersona, DialogContext, DialogInput, - DialogMessage, DialogOutput, DialogTraceArtifacts, DialogTraceUsage, DialogUser, - HistoryMessage, MESSAGE_KIND_TEXT, MESSAGE_KIND_TOOL_REQUEST, MESSAGE_KIND_TOOL_RESPONSE, - MessageKind, MultimodalImage, Persona, PersonaSnapshot, ROLE_MODEL, ROLE_TOOL, ROLE_USER, - SettingKv, TurnContextArtifact, clone_history_messages, conversation_projection, - history_message_has_context_content, history_meta_has_context_content, + DialogMessage, DialogOutput, DialogSampling, DialogTraceArtifacts, DialogTraceUsage, + DialogUser, HistoryMessage, MESSAGE_KIND_TEXT, MESSAGE_KIND_TOOL_REQUEST, + MESSAGE_KIND_TOOL_RESPONSE, MessageKind, MultimodalImage, Persona, PersonaSnapshot, ROLE_MODEL, + ROLE_TOOL, ROLE_USER, SettingKv, TurnContextArtifact, clone_history_messages, + conversation_projection, history_message_has_context_content, history_meta_has_context_content, normalize_history_message, normalize_history_message_kind, select_history_messages_for_context, select_llm_history_messages_for_context, unique_history_text, }; diff --git a/crates/openplotva-llm/src/aifarm.rs b/crates/openplotva-llm/src/aifarm.rs index 59da1551..90c8ec32 100644 --- a/crates/openplotva-llm/src/aifarm.rs +++ b/crates/openplotva-llm/src/aifarm.rs @@ -20,11 +20,11 @@ use url::Url; use openplotva_core::{ChatAttachment, ChatMessageMeta}; use openplotva_dialog::{ ChatStepOutput, ChatStepRequest, ChatStepToolCall, DEFAULT_CONTEXT_HISTORY_LIMIT, DialogInput, - DialogTraceArtifacts, DialogTraceError, DialogTraceUsage, HistoryMessage, MESSAGE_KIND_TEXT, - MESSAGE_KIND_TOOL_REQUEST, MESSAGE_KIND_TOOL_RESPONSE, NativeToolCall, NativeToolFunction, - PROVIDER_AIFARM, PROVIDER_NVIDIA, PROVIDER_VMLX, ROLE_MODEL, ROLE_USER, ReplyLeakGuard, - STEP_REACT_TO_MESSAGE, STEP_SEND_MESSAGE, SessionMessage, ToolParseDecision, ToolStep, - ToolsMode, clone_history_messages, decode_plotva_final_response_with_salvage, + DialogSampling, DialogTraceArtifacts, DialogTraceError, DialogTraceUsage, HistoryMessage, + MESSAGE_KIND_TEXT, MESSAGE_KIND_TOOL_REQUEST, MESSAGE_KIND_TOOL_RESPONSE, NativeToolCall, + NativeToolFunction, PROVIDER_AIFARM, PROVIDER_NVIDIA, PROVIDER_VMLX, ROLE_MODEL, ROLE_USER, + ReplyLeakGuard, STEP_REACT_TO_MESSAGE, STEP_SEND_MESSAGE, SessionMessage, ToolParseDecision, + ToolStep, ToolsMode, clone_history_messages, decode_plotva_final_response_with_salvage, is_dialog_history_noise_tool_call_name, normalize_history_message, parse_assistant_content, parse_native_tool_step, reply_shape, sanitize_tool_text, select_llm_history_messages_for_context, tool_telemetry, @@ -206,6 +206,9 @@ pub struct ChatCompletionRequest { serialize_with = "serialize_top_k" )] pub top_k: Option, + /// Minimum token probability relative to the most likely token. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub min_p: Option, /// llama.cpp repeat penalty. #[serde(default, skip_serializing_if = "Option::is_none")] pub repeat_penalty: Option, @@ -306,6 +309,7 @@ impl Serialize for RedactedChatCompletionRequest<'_> { if let Some(top_k) = request.top_k { state.serialize_field("top_k", &top_k_wire_value(top_k))?; } + serialize_optional_field(&mut state, "min_p", request.min_p)?; serialize_optional_field(&mut state, "repeat_penalty", request.repeat_penalty)?; serialize_optional_field(&mut state, "repetition_penalty", request.repetition_penalty)?; serialize_optional_field(&mut state, "frequency_penalty", request.frequency_penalty)?; @@ -3300,6 +3304,7 @@ where include_reasoning: self.cfg.include_reasoning, ..ChatCompletionRequest::default() }; + apply_dialog_sampling(&mut request, &input.sampling); if let ToolsMode::Native(tool_values) = tools && !tool_values.is_empty() { @@ -3853,6 +3858,7 @@ fn aifarm_trace_inference_params(request: &ChatCompletionRequest) -> Option Result<(), CompletionError> + { + let cfg = AifarmDialogConfig { + temperature: Some(0.2), + top_p: Some(0.95), + frequency_penalty: Some(0.3), + presence_penalty: Some(0.2), + ..AifarmDialogConfig::default() + }; + let mut bodies = Vec::new(); + for sampling in [ + DialogSampling::default(), + DialogSampling { + temperature: Some(0.6), + top_p: Some(0.9), + top_k: Some(64.0), + min_p: Some(0.05), + repeat_penalty: Some(1.1), + ..DialogSampling::default() + }, + ] { + let (provider, transport, _) = direct_dialog_provider( + json!({ "choices": [{ "message": { "role": "assistant", "content": "ага" } }] }), + cfg.clone(), + ); + let mut input = base_input(); + input.sampling = sampling; + crate::ChatStepProvider::run_chat_step( + &provider, + openplotva_dialog::ChatStepRequest { + required_tool: None, + preferred_target: None, + input, + transcript: Vec::new(), + tools: openplotva_dialog::ToolsMode::Disabled, + iteration: 1, + }, + ) + .await + .expect("dialog step"); + let sent = transport.requests(); + bodies.push(serde_json::from_slice::(&sent[0].body).expect("wire request")); + } + + let [configured, routed] = &bodies[..] else { + unreachable!("two requests were sent"); + }; + assert_eq!(configured["temperature"], json!(0.2)); + assert_eq!(configured["top_p"], json!(0.95)); + assert!(configured.get("min_p").is_none()); + assert!(configured.get("top_k").is_none()); + assert_eq!(routed["temperature"], json!(0.6)); + assert_eq!(routed["top_p"], json!(0.9)); + assert_eq!(routed["top_k"], json!(64)); + assert_eq!(routed["min_p"], json!(0.05)); + assert_eq!(routed["repeat_penalty"], json!(1.1)); + assert_eq!( + routed["frequency_penalty"], + json!(0.3), + "unset fields keep the configured value" + ); + assert_eq!(routed["presence_penalty"], json!(0.2)); + Ok(()) + } + #[tokio::test] async fn dialog_request_omits_bad_words_when_the_model_has_none() -> Result<(), CompletionError> { diff --git a/crates/openplotva-storage/src/lib.rs b/crates/openplotva-storage/src/lib.rs index 7ffe738a..f389966c 100644 --- a/crates/openplotva-storage/src/lib.rs +++ b/crates/openplotva-storage/src/lib.rs @@ -10256,6 +10256,44 @@ mod tests { assert!(DOWN.contains("WHERE config ? 'backend_lanes_v1_previous_name'")); } + #[test] + fn gemma_card_sampling_is_scoped_to_the_dialog_assignment_and_reversible() { + const UP: &str = include_str!("../../../migrations/208_gemma_card_dialog_sampling.up.sql"); + const DOWN: &str = + include_str!("../../../migrations/208_gemma_card_dialog_sampling.down.sql"); + + for value in [ + "UPDATE workflow_assignments AS assignment", + "assignment.workflow_key = 'dialog'", + "provider.name = 'aifarm-vllm-gpu0'", + "model.model_name = 'Gemma 4 26B Heretic'", + r#"'{"temperature": 0.6, "top_p": 0.9, "top_k": 64, "min_p": 0.05, "repeat_penalty": 1.1}'::jsonb"#, + "NOT assignment.inference_overrides ? 'gemma_card_sampling_v1_previous'", + ] { + assert!( + UP.contains(value), + "missing Gemma sampling contract {value}" + ); + } + assert!(!UP.contains("provider_models SET")); + assert!(!UP.contains("INSERT INTO")); + assert!(!UP.contains("DELETE FROM")); + for key in [ + "'temperature'", + "'top_p'", + "'top_k'", + "'min_p'", + "'repeat_penalty'", + ] { + assert!(DOWN.contains(key), "down migration must restore {key}"); + } + assert!( + DOWN.contains( + "WHERE assignment.inference_overrides ? 'gemma_card_sampling_v1_previous'" + ) + ); + } + #[test] fn translation_workflow_routes_like_dialog_and_is_reversible() { const UP: &str = include_str!("../../../migrations/207_translation_workflow.up.sql"); diff --git a/migrations/208_gemma_card_dialog_sampling.down.sql b/migrations/208_gemma_card_dialog_sampling.down.sql new file mode 100644 index 00000000..9f7c6dda --- /dev/null +++ b/migrations/208_gemma_card_dialog_sampling.down.sql @@ -0,0 +1,16 @@ +-- Restore each sampling key from the marker the up migration saved: a key that was absent +-- before is removed, a key that had a value gets it back. +UPDATE workflow_assignments AS assignment +SET inference_overrides = ( + SELECT COALESCE(jsonb_object_agg(key, value) FILTER (WHERE value <> 'null'::jsonb), '{}'::jsonb) + FROM jsonb_each(assignment.inference_overrides->'gemma_card_sampling_v1_previous') +) || ( + assignment.inference_overrides + - 'gemma_card_sampling_v1_previous' + - 'temperature' + - 'top_p' + - 'top_k' + - 'min_p' + - 'repeat_penalty' +) +WHERE assignment.inference_overrides ? 'gemma_card_sampling_v1_previous'; diff --git a/migrations/208_gemma_card_dialog_sampling.up.sql b/migrations/208_gemma_card_dialog_sampling.up.sql new file mode 100644 index 00000000..bf075c04 --- /dev/null +++ b/migrations/208_gemma_card_dialog_sampling.up.sql @@ -0,0 +1,27 @@ +-- Dialog turns served by the GPU0 Gemma take the sampling its model card recommends: +-- temperature 0.6, top_k 64, top_p 0.9, min_p 0.05, repeat_penalty 1.1. The values sit on the +-- dialog assignment of that model, so the Qwen and GPT fallbacks, the history, optimizer and +-- other Gemma workflows keep their own sampling. NInfer clamps top_k to 20 candidates and the +-- farm proxy drops repeat_penalty; both stay in the config as the card states them. +-- Each replaced key is saved under one marker so the down migration restores it exactly. +UPDATE workflow_assignments AS assignment +SET inference_overrides = assignment.inference_overrides + || jsonb_build_object( + 'gemma_card_sampling_v1_previous', + jsonb_build_object( + 'temperature', COALESCE(assignment.inference_overrides->'temperature', 'null'::jsonb), + 'top_p', COALESCE(assignment.inference_overrides->'top_p', 'null'::jsonb), + 'top_k', COALESCE(assignment.inference_overrides->'top_k', 'null'::jsonb), + 'min_p', COALESCE(assignment.inference_overrides->'min_p', 'null'::jsonb), + 'repeat_penalty', COALESCE(assignment.inference_overrides->'repeat_penalty', 'null'::jsonb) + ) + ) + || '{"temperature": 0.6, "top_p": 0.9, "top_k": 64, "min_p": 0.05, "repeat_penalty": 1.1}'::jsonb +FROM provider_models AS model, + llm_providers AS provider +WHERE model.id = assignment.provider_model_id + AND provider.id = model.provider_id + AND assignment.workflow_key = 'dialog' + AND provider.name = 'aifarm-vllm-gpu0' + AND model.model_name = 'Gemma 4 26B Heretic' + AND NOT assignment.inference_overrides ? 'gemma_card_sampling_v1_previous'; From af7ed5d5b96f5fdf33026c61a050ca544b0e3a71 Mon Sep 17 00:00:00 2001 From: Valeriy Selitskiy <239034+iamwavecut@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:26:04 +0200 Subject: [PATCH 2/2] Bump rustls to 0.23.46 for RUSTSEC-2026-0346 to 0349 The advisories published today fail the dependency gate for 0.23.45 (ECH rejection data leak, unbounded Brotli certificate allocation, ServerHello key-exchange downgrade, TLS secrets in Debug output). The lock file moves to the patched release; no manifest change is needed. --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3057ad9f..982556c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2946,9 +2946,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "log",