From 98bdf0b70f9c9552908e3446a1d5038dff1cb8d3 Mon Sep 17 00:00:00 2001 From: mudler <2420543+mudler@users.noreply.github.com> Date: Sat, 15 Aug 2026 20:06:13 +0000 Subject: [PATCH 1/4] :arrow_up: Update mudler/vllm.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- backend/go/vllm-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go/vllm-cpp/Makefile b/backend/go/vllm-cpp/Makefile index e1437be33b46..f01b9a77f163 100644 --- a/backend/go/vllm-cpp/Makefile +++ b/backend/go/vllm-cpp/Makefile @@ -11,7 +11,7 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e # vllm.cpp version VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp -VLLM_CPP_VERSION?=9fd9e8f34408d5dd21d7f9385e96fc755708950b +VLLM_CPP_VERSION?=4880c5715f36445a30bd39d3349a06dc96085a11 # MLX GEMM provider (darwin/metal only; see the metal branch below for why). # Consumed as the prebuilt pip wheel: building MLX from source needs `xcrun From 60730425d0da2beaf2d0504f394e8d7305449068 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Sat, 15 Aug 2026 04:07:09 +0000 Subject: [PATCH 2/4] fix(vllm-cpp): mirror ABI v20 layouts The dependency bump advances the engine ABI from v17 to v20. The old binding stops every backend build at the ABI guard and undersizes structures used at runtime. Mirror the appended model and video fields so every platform uses the pinned engine layout. Assisted-by: Codex:gpt-5 --- backend/go/vllm-cpp/README.md | 2 +- backend/go/vllm-cpp/govllmcpp.go | 49 ++++++++++++++++++----------- backend/go/vllm-cpp/video_test.go | 11 +++++-- backend/go/vllm-cpp/vllmcpp_test.go | 19 ++++++----- 4 files changed, 52 insertions(+), 29 deletions(-) diff --git a/backend/go/vllm-cpp/README.md b/backend/go/vllm-cpp/README.md index 7bfb9224138a..49804ffb447a 100644 --- a/backend/go/vllm-cpp/README.md +++ b/backend/go/vllm-cpp/README.md @@ -9,7 +9,7 @@ It serves two things: text generation, and MiniMax-H3 joint video+audio generation. The backend dlopens the engine's stable C ABI (`libvllm`, `include/vllm.h`, -ABI v16) through purego: +ABI v20) through purego: - `Load` -> `vllm_engine_load`: accepts a `.gguf` file or a HF-style model directory (`config.json` + safetensors). `context_size` maps to diff --git a/backend/go/vllm-cpp/govllmcpp.go b/backend/go/vllm-cpp/govllmcpp.go index 5de1c0e6456f..318b2d375493 100644 --- a/backend/go/vllm-cpp/govllmcpp.go +++ b/backend/go/vllm-cpp/govllmcpp.go @@ -1,6 +1,6 @@ package main -// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v16). +// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v20). // // The structs below are hand-mirrored PODs of the C declarations, with // explicit padding so the Go layout matches the C layout on linux/darwin @@ -21,7 +21,7 @@ import ( // the header of the VLLM_CPP_VERSION pinned in the Makefile: the build checks // the two against each other, because a mismatch is only caught at runtime by // registerLib, where it takes the backend down on every load (issue #11379). -const abiVersion = 17 +const abiVersion = 20 // The ABI's tri-state toggles (enable_prefix_caching ABI v7, // enable_jump_forward ABI v10) share one encoding: 0 is NOT "off", it is @@ -69,6 +69,7 @@ type cModelParams struct { MaxNumBatchedTokens int32 // <= 0 = per-arch default (ABI v9) SchedulingPolicy uintptr // const char*; NULL = "fcfs" (ABI v9) KVTransferConfig uintptr // const char* JSON; NULL = no connector (ABI v9) + OffloadConfig uintptr // const char* JSON; NULL = no weight offload EnableJumpForward int32 // tri-state 0/1/2 (ABI v10) // v14/v16 tail. LocalAI sets none of these (0 is "auto" for the device and // "unset" for both sizing knobs, i.e. the pre-v14 engine byte for byte), but @@ -79,6 +80,9 @@ type cModelParams struct { Device int32 // 0 auto, 1 cpu, 2 cuda (ABI v14) GPUMemoryUtil float64 // 0 => 0.92 (ABI v16) KVCacheMemoryBytes int64 // 0 => unset (ABI v16) + LanguageModelOnly int32 // 0 = multimodal inputs enabled (ABI v19) + _ [4]byte + LimitMMPerPrompt uintptr // const char* JSON; NULL = default limits (ABI v19) } // cSamplingParams mirrors vllm_sampling_params (structured fields included). @@ -147,6 +151,11 @@ type cVideoModelParams struct { Device int32 // 0 cpu, 1 cuda DequantBf16 int32 // 0 keep-quant, 1 dequant/stream bf16 Fp4Resident int32 // NVFP4+cuda: keep FP4 packed, Marlin W4A16 + _ [4]byte + Family uintptr // const char*; NULL = detect (ABI v18) + ExtraKeys uintptr // const char* const* (ABI v18) + ExtraValues uintptr // const char* const* (ABI v18) + NExtras int32 // 0 = none (ABI v18) _ [4]byte // trailing pad to the struct's 8-byte alignment } @@ -154,22 +163,26 @@ type cVideoModelParams struct { // `steps` pair up into 8-byte slots; the uint64 seed forces the alignment after // them, and the float noise_aug leaves a pad before output_dir. type cVideoParams struct { - Prompt uintptr // const char* - Width int32 - Height int32 - NumFrames int32 // <= 1 => per-task default (124 for t2va/fl2va) - Steps int32 // <= 0 => the H3 default (50) - Seed uint64 - HasSeed int32 - _ [4]byte - FirstFrame uintptr // const char*; fl2va keyframe, binary PPM (P6) - LastFrame uintptr // const char* - RefImage uintptr // const char*; ref2va only - RefVideo uintptr // const char*; ref2va only, a frame_%06d.ppm DIRECTORY - RefAudio uintptr // const char*; ref2va only, 16-bit PCM WAV - NoiseAug float32 // <= 0 => 1.0 - _ [4]byte - OutputDir uintptr // const char*; REQUIRED + Prompt uintptr // const char* + Width int32 + Height int32 + NumFrames int32 // <= 1 => per-task default (124 for t2va/fl2va) + Steps int32 // <= 0 => the H3 default (50) + Seed uint64 + HasSeed int32 + _ [4]byte + FirstFrame uintptr // const char*; fl2va keyframe, binary PPM (P6) + LastFrame uintptr // const char* + RefImage uintptr // const char*; ref2va only + RefVideo uintptr // const char*; ref2va only, a frame_%06d.ppm DIRECTORY + RefAudio uintptr // const char*; ref2va only, 16-bit PCM WAV + NoiseAug float32 // <= 0 => 1.0 + _ [4]byte + OutputDir uintptr // const char*; REQUIRED + ExtraKeys uintptr // const char* const* (ABI v18) + ExtraValues uintptr // const char* const* (ABI v18) + NExtras int32 // 0 = none (ABI v18) + _ [4]byte } // cVideoResult mirrors vllm_video_result. Every member is library-allocated and diff --git a/backend/go/vllm-cpp/video_test.go b/backend/go/vllm-cpp/video_test.go index 538e2f0c723c..5470005423fc 100644 --- a/backend/go/vllm-cpp/video_test.go +++ b/backend/go/vllm-cpp/video_test.go @@ -28,7 +28,11 @@ var _ = Describe("C ABI video struct mirrors", func() { Expect(unsafe.Offsetof(p.Device)).To(Equal(uintptr(72))) Expect(unsafe.Offsetof(p.DequantBf16)).To(Equal(uintptr(76))) Expect(unsafe.Offsetof(p.Fp4Resident)).To(Equal(uintptr(80))) - Expect(unsafe.Sizeof(p)).To(Equal(uintptr(88))) + Expect(unsafe.Offsetof(p.Family)).To(Equal(uintptr(88))) + Expect(unsafe.Offsetof(p.ExtraKeys)).To(Equal(uintptr(96))) + Expect(unsafe.Offsetof(p.ExtraValues)).To(Equal(uintptr(104))) + Expect(unsafe.Offsetof(p.NExtras)).To(Equal(uintptr(112))) + Expect(unsafe.Sizeof(p)).To(Equal(uintptr(120))) }) It("cVideoParams matches vllm_video_params", func() { @@ -47,7 +51,10 @@ var _ = Describe("C ABI video struct mirrors", func() { Expect(unsafe.Offsetof(p.RefAudio)).To(Equal(uintptr(72))) Expect(unsafe.Offsetof(p.NoiseAug)).To(Equal(uintptr(80))) Expect(unsafe.Offsetof(p.OutputDir)).To(Equal(uintptr(88))) - Expect(unsafe.Sizeof(p)).To(Equal(uintptr(96))) + Expect(unsafe.Offsetof(p.ExtraKeys)).To(Equal(uintptr(96))) + Expect(unsafe.Offsetof(p.ExtraValues)).To(Equal(uintptr(104))) + Expect(unsafe.Offsetof(p.NExtras)).To(Equal(uintptr(112))) + Expect(unsafe.Sizeof(p)).To(Equal(uintptr(120))) }) It("cVideoResult matches vllm_video_result", func() { diff --git a/backend/go/vllm-cpp/vllmcpp_test.go b/backend/go/vllm-cpp/vllmcpp_test.go index 614e746b130b..9fd8c321c322 100644 --- a/backend/go/vllm-cpp/vllmcpp_test.go +++ b/backend/go/vllm-cpp/vllmcpp_test.go @@ -16,7 +16,7 @@ func TestVllmCpp(t *testing.T) { RunSpecs(t, "vllm-cpp suite") } -// The Go POD mirrors must match the C struct layout of vllm.h (ABI v16) +// The Go POD mirrors must match the C struct layout of vllm.h (ABI v20) // byte-for-byte: these offsets are the C offsets on LP64 (linux/darwin // amd64+arm64). A failure here means govllmcpp.go drifted from vllm.h. var _ = Describe("C ABI struct mirrors", func() { @@ -24,7 +24,7 @@ var _ = Describe("C ABI struct mirrors", func() { // VLLM_ABI_VERSION in the vllm.h of VLLM_CPP_VERSION (Makefile). // Moving the pin past this without growing the mirrors below ships a // backend that refuses every load at startup (issue #11379). - Expect(abiVersion).To(Equal(16)) + Expect(abiVersion).To(Equal(20)) }) It("cModelParams matches vllm_model_params", func() { @@ -42,13 +42,16 @@ var _ = Describe("C ABI struct mirrors", func() { Expect(unsafe.Offsetof(p.MaxNumBatchedTokens)).To(Equal(uintptr(60))) Expect(unsafe.Offsetof(p.SchedulingPolicy)).To(Equal(uintptr(64))) Expect(unsafe.Offsetof(p.KVTransferConfig)).To(Equal(uintptr(72))) - Expect(unsafe.Offsetof(p.EnableJumpForward)).To(Equal(uintptr(80))) - Expect(unsafe.Offsetof(p.Device)).To(Equal(uintptr(84))) - // 88, not 92: gpu_memory_utilization is a double, so it takes the next + Expect(unsafe.Offsetof(p.OffloadConfig)).To(Equal(uintptr(80))) + Expect(unsafe.Offsetof(p.EnableJumpForward)).To(Equal(uintptr(88))) + Expect(unsafe.Offsetof(p.Device)).To(Equal(uintptr(92))) + // 96: gpu_memory_utilization is a double, so it takes the next // 8-aligned slot after the int32 pair. Go pads identically. - Expect(unsafe.Offsetof(p.GPUMemoryUtil)).To(Equal(uintptr(88))) - Expect(unsafe.Offsetof(p.KVCacheMemoryBytes)).To(Equal(uintptr(96))) - Expect(unsafe.Sizeof(p)).To(Equal(uintptr(104))) + Expect(unsafe.Offsetof(p.GPUMemoryUtil)).To(Equal(uintptr(96))) + Expect(unsafe.Offsetof(p.KVCacheMemoryBytes)).To(Equal(uintptr(104))) + Expect(unsafe.Offsetof(p.LanguageModelOnly)).To(Equal(uintptr(112))) + Expect(unsafe.Offsetof(p.LimitMMPerPrompt)).To(Equal(uintptr(120))) + Expect(unsafe.Sizeof(p)).To(Equal(uintptr(128))) }) It("cSamplingParams matches vllm_sampling_params (ABI v8)", func() { From 98318c8dbf6620b3152789b53a44ad37cb6f3719 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Sun, 16 Aug 2026 07:31:12 +0000 Subject: [PATCH 3/4] fix(vllm-cpp): satisfy Apple Clang The new engine pin captures a namespace-scope help string in a lambda. Apple Clang rejects the redundant capture because upstream enables -Werror. Carry the one-line source patch until the engine pin includes the fix. Assisted-by: Codex:gpt-5 --- ...001-fix-qwen35-unused-lambda-capture.patch | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 backend/go/vllm-cpp/patches/0001-fix-qwen35-unused-lambda-capture.patch diff --git a/backend/go/vllm-cpp/patches/0001-fix-qwen35-unused-lambda-capture.patch b/backend/go/vllm-cpp/patches/0001-fix-qwen35-unused-lambda-capture.patch new file mode 100644 index 000000000000..b53f6c73fd15 --- /dev/null +++ b/backend/go/vllm-cpp/patches/0001-fix-qwen35-unused-lambda-capture.patch @@ -0,0 +1,26 @@ +From: LocalAI maintainers +Subject: [PATCH] fix(qwen3.5): drop redundant kRequired capture + +Apple Clang diagnoses the capture of this namespace-scope reference as unused, +and vllm.cpp promotes that warning to an error. The lambda can access the name +without a capture. + +Retire this patch when the pinned vllm.cpp revision no longer captures +kRequired in the qwen3.5 refusal lambda. +--- + src/vllm/model_executor/models/qwen3_5_weights.cpp | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/vllm/model_executor/models/qwen3_5_weights.cpp b/src/vllm/model_executor/models/qwen3_5_weights.cpp +index 149157c..113da72 100644 +--- a/src/vllm/model_executor/models/qwen3_5_weights.cpp ++++ b/src/vllm/model_executor/models/qwen3_5_weights.cpp +@@ -950,7 +950,7 @@ void Qwen35Weights::validateMoeLayout(const TensorSet& tensors) const { + // ...and the three NON-routed components, refused by the dtype the probe + // RESOLVED rather than discovered as a complaint from inside a reader (#490). + // Each of these already failed before #864; naming it is the whole change. +- const auto refuse = [&kRequired](const char* what, MoeProjDtype got, ++ const auto refuse = [](const char* what, MoeProjDtype got, + const char* supported) { + VT_CHECK(false, std::string("qwen3_5 weights: a ") + + MoeProjDtypeName(got) + " " + what + From a354f3c45abd5b443043aa6b71037e928aa70e76 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Sun, 16 Aug 2026 10:06:50 +0000 Subject: [PATCH 4/4] fix(vllm-cpp): align the carry patch The Apple Clang patch used context from another source revision. Source preparation rejected it before every backend build. Align the patch with the pinned engine revision. Assisted-by: Codex:gpt-5 [monitoring-prs] --- .../patches/0001-fix-qwen35-unused-lambda-capture.patch | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/go/vllm-cpp/patches/0001-fix-qwen35-unused-lambda-capture.patch b/backend/go/vllm-cpp/patches/0001-fix-qwen35-unused-lambda-capture.patch index b53f6c73fd15..cb3e970536de 100644 --- a/backend/go/vllm-cpp/patches/0001-fix-qwen35-unused-lambda-capture.patch +++ b/backend/go/vllm-cpp/patches/0001-fix-qwen35-unused-lambda-capture.patch @@ -12,10 +12,10 @@ kRequired in the qwen3.5 refusal lambda. 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vllm/model_executor/models/qwen3_5_weights.cpp b/src/vllm/model_executor/models/qwen3_5_weights.cpp -index 149157c..113da72 100644 +index 7150443..7de1b9d 100644 --- a/src/vllm/model_executor/models/qwen3_5_weights.cpp +++ b/src/vllm/model_executor/models/qwen3_5_weights.cpp -@@ -950,7 +950,7 @@ void Qwen35Weights::validateMoeLayout(const TensorSet& tensors) const { +@@ -950,7 +950,7 @@ void CheckMoeQuantLayoutSupported(const std::vector& names, // ...and the three NON-routed components, refused by the dtype the probe // RESOLVED rather than discovered as a complaint from inside a reader (#490). // Each of these already failed before #864; naming it is the whole change. @@ -23,4 +23,4 @@ index 149157c..113da72 100644 + const auto refuse = [](const char* what, MoeProjDtype got, const char* supported) { VT_CHECK(false, std::string("qwen3_5 weights: a ") + - MoeProjDtypeName(got) + " " + what + + MoeProjDtypeName(got) + " " + what +