diff --git a/backend/go/vllm-cpp/Makefile b/backend/go/vllm-cpp/Makefile index e1437be33b46..f01b9a77f163 100644 --- a/backend/go/vllm-cpp/Makefile +++ b/backend/go/vllm-cpp/Makefile @@ -11,7 +11,7 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e # vllm.cpp version VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp -VLLM_CPP_VERSION?=9fd9e8f34408d5dd21d7f9385e96fc755708950b +VLLM_CPP_VERSION?=4880c5715f36445a30bd39d3349a06dc96085a11 # MLX GEMM provider (darwin/metal only; see the metal branch below for why). # Consumed as the prebuilt pip wheel: building MLX from source needs `xcrun diff --git a/backend/go/vllm-cpp/README.md b/backend/go/vllm-cpp/README.md index 7bfb9224138a..49804ffb447a 100644 --- a/backend/go/vllm-cpp/README.md +++ b/backend/go/vllm-cpp/README.md @@ -9,7 +9,7 @@ It serves two things: text generation, and MiniMax-H3 joint video+audio generation. The backend dlopens the engine's stable C ABI (`libvllm`, `include/vllm.h`, -ABI v16) through purego: +ABI v20) through purego: - `Load` -> `vllm_engine_load`: accepts a `.gguf` file or a HF-style model directory (`config.json` + safetensors). `context_size` maps to diff --git a/backend/go/vllm-cpp/govllmcpp.go b/backend/go/vllm-cpp/govllmcpp.go index 5de1c0e6456f..318b2d375493 100644 --- a/backend/go/vllm-cpp/govllmcpp.go +++ b/backend/go/vllm-cpp/govllmcpp.go @@ -1,6 +1,6 @@ package main -// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v16). +// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v20). // // The structs below are hand-mirrored PODs of the C declarations, with // explicit padding so the Go layout matches the C layout on linux/darwin @@ -21,7 +21,7 @@ import ( // the header of the VLLM_CPP_VERSION pinned in the Makefile: the build checks // the two against each other, because a mismatch is only caught at runtime by // registerLib, where it takes the backend down on every load (issue #11379). -const abiVersion = 17 +const abiVersion = 20 // The ABI's tri-state toggles (enable_prefix_caching ABI v7, // enable_jump_forward ABI v10) share one encoding: 0 is NOT "off", it is @@ -69,6 +69,7 @@ type cModelParams struct { MaxNumBatchedTokens int32 // <= 0 = per-arch default (ABI v9) SchedulingPolicy uintptr // const char*; NULL = "fcfs" (ABI v9) KVTransferConfig uintptr // const char* JSON; NULL = no connector (ABI v9) + OffloadConfig uintptr // const char* JSON; NULL = no weight offload EnableJumpForward int32 // tri-state 0/1/2 (ABI v10) // v14/v16 tail. LocalAI sets none of these (0 is "auto" for the device and // "unset" for both sizing knobs, i.e. the pre-v14 engine byte for byte), but @@ -79,6 +80,9 @@ type cModelParams struct { Device int32 // 0 auto, 1 cpu, 2 cuda (ABI v14) GPUMemoryUtil float64 // 0 => 0.92 (ABI v16) KVCacheMemoryBytes int64 // 0 => unset (ABI v16) + LanguageModelOnly int32 // 0 = multimodal inputs enabled (ABI v19) + _ [4]byte + LimitMMPerPrompt uintptr // const char* JSON; NULL = default limits (ABI v19) } // cSamplingParams mirrors vllm_sampling_params (structured fields included). @@ -147,6 +151,11 @@ type cVideoModelParams struct { Device int32 // 0 cpu, 1 cuda DequantBf16 int32 // 0 keep-quant, 1 dequant/stream bf16 Fp4Resident int32 // NVFP4+cuda: keep FP4 packed, Marlin W4A16 + _ [4]byte + Family uintptr // const char*; NULL = detect (ABI v18) + ExtraKeys uintptr // const char* const* (ABI v18) + ExtraValues uintptr // const char* const* (ABI v18) + NExtras int32 // 0 = none (ABI v18) _ [4]byte // trailing pad to the struct's 8-byte alignment } @@ -154,22 +163,26 @@ type cVideoModelParams struct { // `steps` pair up into 8-byte slots; the uint64 seed forces the alignment after // them, and the float noise_aug leaves a pad before output_dir. type cVideoParams struct { - Prompt uintptr // const char* - Width int32 - Height int32 - NumFrames int32 // <= 1 => per-task default (124 for t2va/fl2va) - Steps int32 // <= 0 => the H3 default (50) - Seed uint64 - HasSeed int32 - _ [4]byte - FirstFrame uintptr // const char*; fl2va keyframe, binary PPM (P6) - LastFrame uintptr // const char* - RefImage uintptr // const char*; ref2va only - RefVideo uintptr // const char*; ref2va only, a frame_%06d.ppm DIRECTORY - RefAudio uintptr // const char*; ref2va only, 16-bit PCM WAV - NoiseAug float32 // <= 0 => 1.0 - _ [4]byte - OutputDir uintptr // const char*; REQUIRED + Prompt uintptr // const char* + Width int32 + Height int32 + NumFrames int32 // <= 1 => per-task default (124 for t2va/fl2va) + Steps int32 // <= 0 => the H3 default (50) + Seed uint64 + HasSeed int32 + _ [4]byte + FirstFrame uintptr // const char*; fl2va keyframe, binary PPM (P6) + LastFrame uintptr // const char* + RefImage uintptr // const char*; ref2va only + RefVideo uintptr // const char*; ref2va only, a frame_%06d.ppm DIRECTORY + RefAudio uintptr // const char*; ref2va only, 16-bit PCM WAV + NoiseAug float32 // <= 0 => 1.0 + _ [4]byte + OutputDir uintptr // const char*; REQUIRED + ExtraKeys uintptr // const char* const* (ABI v18) + ExtraValues uintptr // const char* const* (ABI v18) + NExtras int32 // 0 = none (ABI v18) + _ [4]byte } // cVideoResult mirrors vllm_video_result. Every member is library-allocated and diff --git a/backend/go/vllm-cpp/patches/0001-fix-qwen35-unused-lambda-capture.patch b/backend/go/vllm-cpp/patches/0001-fix-qwen35-unused-lambda-capture.patch new file mode 100644 index 000000000000..cb3e970536de --- /dev/null +++ b/backend/go/vllm-cpp/patches/0001-fix-qwen35-unused-lambda-capture.patch @@ -0,0 +1,26 @@ +From: LocalAI maintainers +Subject: [PATCH] fix(qwen3.5): drop redundant kRequired capture + +Apple Clang diagnoses the capture of this namespace-scope reference as unused, +and vllm.cpp promotes that warning to an error. The lambda can access the name +without a capture. + +Retire this patch when the pinned vllm.cpp revision no longer captures +kRequired in the qwen3.5 refusal lambda. +--- + src/vllm/model_executor/models/qwen3_5_weights.cpp | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/vllm/model_executor/models/qwen3_5_weights.cpp b/src/vllm/model_executor/models/qwen3_5_weights.cpp +index 7150443..7de1b9d 100644 +--- a/src/vllm/model_executor/models/qwen3_5_weights.cpp ++++ b/src/vllm/model_executor/models/qwen3_5_weights.cpp +@@ -950,7 +950,7 @@ void CheckMoeQuantLayoutSupported(const std::vector& names, + // ...and the three NON-routed components, refused by the dtype the probe + // RESOLVED rather than discovered as a complaint from inside a reader (#490). + // Each of these already failed before #864; naming it is the whole change. +- const auto refuse = [&kRequired](const char* what, MoeProjDtype got, ++ const auto refuse = [](const char* what, MoeProjDtype got, + const char* supported) { + VT_CHECK(false, std::string("qwen3_5 weights: a ") + + MoeProjDtypeName(got) + " " + what + diff --git a/backend/go/vllm-cpp/video_test.go b/backend/go/vllm-cpp/video_test.go index 538e2f0c723c..5470005423fc 100644 --- a/backend/go/vllm-cpp/video_test.go +++ b/backend/go/vllm-cpp/video_test.go @@ -28,7 +28,11 @@ var _ = Describe("C ABI video struct mirrors", func() { Expect(unsafe.Offsetof(p.Device)).To(Equal(uintptr(72))) Expect(unsafe.Offsetof(p.DequantBf16)).To(Equal(uintptr(76))) Expect(unsafe.Offsetof(p.Fp4Resident)).To(Equal(uintptr(80))) - Expect(unsafe.Sizeof(p)).To(Equal(uintptr(88))) + Expect(unsafe.Offsetof(p.Family)).To(Equal(uintptr(88))) + Expect(unsafe.Offsetof(p.ExtraKeys)).To(Equal(uintptr(96))) + Expect(unsafe.Offsetof(p.ExtraValues)).To(Equal(uintptr(104))) + Expect(unsafe.Offsetof(p.NExtras)).To(Equal(uintptr(112))) + Expect(unsafe.Sizeof(p)).To(Equal(uintptr(120))) }) It("cVideoParams matches vllm_video_params", func() { @@ -47,7 +51,10 @@ var _ = Describe("C ABI video struct mirrors", func() { Expect(unsafe.Offsetof(p.RefAudio)).To(Equal(uintptr(72))) Expect(unsafe.Offsetof(p.NoiseAug)).To(Equal(uintptr(80))) Expect(unsafe.Offsetof(p.OutputDir)).To(Equal(uintptr(88))) - Expect(unsafe.Sizeof(p)).To(Equal(uintptr(96))) + Expect(unsafe.Offsetof(p.ExtraKeys)).To(Equal(uintptr(96))) + Expect(unsafe.Offsetof(p.ExtraValues)).To(Equal(uintptr(104))) + Expect(unsafe.Offsetof(p.NExtras)).To(Equal(uintptr(112))) + Expect(unsafe.Sizeof(p)).To(Equal(uintptr(120))) }) It("cVideoResult matches vllm_video_result", func() { diff --git a/backend/go/vllm-cpp/vllmcpp_test.go b/backend/go/vllm-cpp/vllmcpp_test.go index 614e746b130b..9fd8c321c322 100644 --- a/backend/go/vllm-cpp/vllmcpp_test.go +++ b/backend/go/vllm-cpp/vllmcpp_test.go @@ -16,7 +16,7 @@ func TestVllmCpp(t *testing.T) { RunSpecs(t, "vllm-cpp suite") } -// The Go POD mirrors must match the C struct layout of vllm.h (ABI v16) +// The Go POD mirrors must match the C struct layout of vllm.h (ABI v20) // byte-for-byte: these offsets are the C offsets on LP64 (linux/darwin // amd64+arm64). A failure here means govllmcpp.go drifted from vllm.h. var _ = Describe("C ABI struct mirrors", func() { @@ -24,7 +24,7 @@ var _ = Describe("C ABI struct mirrors", func() { // VLLM_ABI_VERSION in the vllm.h of VLLM_CPP_VERSION (Makefile). // Moving the pin past this without growing the mirrors below ships a // backend that refuses every load at startup (issue #11379). - Expect(abiVersion).To(Equal(16)) + Expect(abiVersion).To(Equal(20)) }) It("cModelParams matches vllm_model_params", func() { @@ -42,13 +42,16 @@ var _ = Describe("C ABI struct mirrors", func() { Expect(unsafe.Offsetof(p.MaxNumBatchedTokens)).To(Equal(uintptr(60))) Expect(unsafe.Offsetof(p.SchedulingPolicy)).To(Equal(uintptr(64))) Expect(unsafe.Offsetof(p.KVTransferConfig)).To(Equal(uintptr(72))) - Expect(unsafe.Offsetof(p.EnableJumpForward)).To(Equal(uintptr(80))) - Expect(unsafe.Offsetof(p.Device)).To(Equal(uintptr(84))) - // 88, not 92: gpu_memory_utilization is a double, so it takes the next + Expect(unsafe.Offsetof(p.OffloadConfig)).To(Equal(uintptr(80))) + Expect(unsafe.Offsetof(p.EnableJumpForward)).To(Equal(uintptr(88))) + Expect(unsafe.Offsetof(p.Device)).To(Equal(uintptr(92))) + // 96: gpu_memory_utilization is a double, so it takes the next // 8-aligned slot after the int32 pair. Go pads identically. - Expect(unsafe.Offsetof(p.GPUMemoryUtil)).To(Equal(uintptr(88))) - Expect(unsafe.Offsetof(p.KVCacheMemoryBytes)).To(Equal(uintptr(96))) - Expect(unsafe.Sizeof(p)).To(Equal(uintptr(104))) + Expect(unsafe.Offsetof(p.GPUMemoryUtil)).To(Equal(uintptr(96))) + Expect(unsafe.Offsetof(p.KVCacheMemoryBytes)).To(Equal(uintptr(104))) + Expect(unsafe.Offsetof(p.LanguageModelOnly)).To(Equal(uintptr(112))) + Expect(unsafe.Offsetof(p.LimitMMPerPrompt)).To(Equal(uintptr(120))) + Expect(unsafe.Sizeof(p)).To(Equal(uintptr(128))) }) It("cSamplingParams matches vllm_sampling_params (ABI v8)", func() {