From 66e23528f55c933ac777ffebf57f60f413fa8e98 Mon Sep 17 00:00:00 2001 From: fszontagh Date: Thu, 27 Aug 2026 12:19:54 +0200 Subject: [PATCH] fix: validate vision projector output dim against LLM hidden size --- ggml | 2 +- src/model/te/llm.hpp | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/ggml b/ggml index 8e800cef2..032b6997d 160000 --- a/ggml +++ b/ggml @@ -1 +1 @@ -Subproject commit 8e800cef2948046cc47f9db6090491c6128ca42c +Subproject commit 032b6997db4c9c75dc85d8d2bb2beec77b1231b0 diff --git a/src/model/te/llm.hpp b/src/model/te/llm.hpp index 092c68e48..a8e614f56 100644 --- a/src/model/te/llm.hpp +++ b/src/model/te/llm.hpp @@ -73,6 +73,7 @@ namespace LLM { int num_heads = 16; int64_t in_channels = 3; int64_t out_hidden_size = 3584; + bool out_hidden_size_detected = false; int temporal_patch_size = 2; int patch_size = 14; int spatial_merge_size = 2; @@ -238,7 +239,8 @@ namespace LLM { } if (contains(name, "visual.merger.linear_fc2.weight") || contains(name, "visual.merger.mlp.2.weight")) { - config.vision.out_hidden_size = tensor_storage.ne[1]; + config.vision.out_hidden_size = tensor_storage.ne[1]; + config.vision.out_hidden_size_detected = true; } continue; } @@ -1768,6 +1770,15 @@ namespace LLM { LOG_WARN("no vision weights detected, vision disabled"); enable_vision = false; } + // The default would reject valid models, so only compare a detected dim. + if (enable_vision && config.vision.out_hidden_size_detected && + config.vision.out_hidden_size != config.hidden_size) { + LOG_ERROR("vision projector output size (%" PRId64 ") does not match LLM hidden size (%" PRId64 "), " + "the vision weights (mmproj) likely belong to a different LLM variant, vision disabled", + config.vision.out_hidden_size, + config.hidden_size); + enable_vision = false; + } if (enable_vision) { LOG_DEBUG("enable llm vision"); if (config.llama_cpp_style) {