From eb9d7510c049f526c9d1a5b2731977c3eb99813d Mon Sep 17 00:00:00 2001 From: Brooks Date: Thu, 1 Oct 2026 10:15:09 +0800 Subject: [PATCH 1/3] Expose llama_model_params.devices as IModelParams.Devices LLamaModelParams.devices was private with a todo, so the only way to pick a device from managed code was main_gpu. llama.cpp's default device selection (llama_prepare_model_devices) only considers integrated GPUs when no discrete GPU is present, and main_gpu indexes into that filtered list, so an iGPU next to a dGPU could never be selected. llama.cpp's escape hatch for this is the devices list (--device on the CLI). Add IModelParams.Devices (device names as returned by ggml_backend_dev_name, e.g. "Vulkan1"). ToLlamaModelParams resolves the names against the available ggml backend devices, builds a NULL-terminated array pinned in the existing GroupDisposable and assigns it to devices. Empty list or no matching names keeps devices NULL, i.e. the current behaviour. Also adds the ggml_backend_dev_name P/Invoke, implements the member in LLama.Web's ModelOptions, and covers the property in the ModelParams JSON round-trip test. Co-Authored-By: Claude Fable 5.1 --- LLama.Unittest/ModelsParamsTests.cs | 6 +++ LLama.Web/Common/ModelOptions.cs | 3 ++ LLama/Abstractions/IModelParams.cs | 11 ++++++ LLama/Common/ModelParams.cs | 3 ++ LLama/Extensions/IModelParamsExtensions.cs | 45 ++++++++++++++++++++++ LLama/Native/LLamaModelParams.cs | 6 +-- LLama/Native/NativeApi.cs | 8 ++++ 7 files changed, 79 insertions(+), 3 deletions(-) diff --git a/LLama.Unittest/ModelsParamsTests.cs b/LLama.Unittest/ModelsParamsTests.cs index 59cf70bf5..99f3155ef 100644 --- a/LLama.Unittest/ModelsParamsTests.cs +++ b/LLama.Unittest/ModelsParamsTests.cs @@ -20,6 +20,7 @@ public void SerializeRoundTripSystemTextJson() ContextSize = 42, GpuLayerCount = 111, TensorSplits = { [0] = 3 }, + Devices = { "Vulkan1", "CPU" }, MetadataOverrides = { new MetadataOverride("hello", true), @@ -46,6 +47,11 @@ public void SerializeRoundTripSystemTextJson() actual.TensorBufferOverrides = null!; expected.TensorBufferOverrides = null!; + // Same deal + Assert.True(expected.Devices.SequenceEqual(actual.Devices)); + actual.Devices = null!; + expected.Devices = null!; + // Check encoding is the same var b1 = expected.Encoding.GetBytes("Hello"); var b2 = actual.Encoding.GetBytes("Hello"); diff --git a/LLama.Web/Common/ModelOptions.cs b/LLama.Web/Common/ModelOptions.cs index 5f0bb64d2..593e27078 100644 --- a/LLama.Web/Common/ModelOptions.cs +++ b/LLama.Web/Common/ModelOptions.cs @@ -31,6 +31,9 @@ public class ModelOptions /// public List TensorBufferOverrides { get; set; } = new(); + /// + public List Devices { get; set; } = new(); + /// public int GpuLayerCount { get; set; } = 20; diff --git a/LLama/Abstractions/IModelParams.cs b/LLama/Abstractions/IModelParams.cs index 6cce4372c..8e632c378 100644 --- a/LLama/Abstractions/IModelParams.cs +++ b/LLama/Abstractions/IModelParams.cs @@ -44,6 +44,17 @@ public interface IModelParams /// List TensorBufferOverrides { get; } + /// + /// Names of the backend devices the model may use, in priority order, e.g. "Vulkan1" or "CUDA0" (see ). + /// Equivalent to --device on the llama.cpp command line or devices in llama_model_params. + /// + /// + /// When empty, llama.cpp picks the devices itself: all discrete GPUs, or the first integrated GPU when there is no discrete GPU. + /// Setting this list explicitly bypasses that selection, which is the only way to run on an integrated GPU in a machine that also has a discrete GPU. + /// is an index into this list when it is non-empty. Names that do not match an available device are ignored. + /// + List Devices { get; } + /// /// Number of layers to run in VRAM / GPU memory (n_gpu_layers) /// diff --git a/LLama/Common/ModelParams.cs b/LLama/Common/ModelParams.cs index 55e7c4300..0c03757de 100644 --- a/LLama/Common/ModelParams.cs +++ b/LLama/Common/ModelParams.cs @@ -28,6 +28,9 @@ public record ModelParams /// public List TensorBufferOverrides { get; set; } = new(); + /// + public List Devices { get; set; } = new(); + /// public int GpuLayerCount { get; set; } = 20; diff --git a/LLama/Extensions/IModelParamsExtensions.cs b/LLama/Extensions/IModelParamsExtensions.cs index 5ed152c7e..8c3497b4a 100644 --- a/LLama/Extensions/IModelParamsExtensions.cs +++ b/LLama/Extensions/IModelParamsExtensions.cs @@ -56,6 +56,12 @@ public static IDisposable ToLlamaModelParams(this IModelParams @params, out LLam result.tensor_buft_overrides = ConvertOverrides(@params.TensorBufferOverrides, disposer); } + // Add device list + unsafe + { + result.devices = ConvertDevices(@params.Devices, disposer); + } + // Add metadata overrides if (@params.MetadataOverrides.Count == 0) { @@ -129,6 +135,45 @@ private static IReadOnlyDictionary GetAvailableBufferTypes() return result; } + private static unsafe IntPtr* ConvertDevices(List deviceNames, GroupDisposable disposer) + { + // Early out if no devices were requested (llama.cpp will choose its own) + if (deviceNames.Count == 0) + return null; + + // Map device name -> ggml_backend_dev_t + var devicesByName = new Dictionary(); + var deviceCount = NativeApi.ggml_backend_dev_count(); + for (nuint i = 0; i < deviceCount; i++) + { + var device = NativeApi.ggml_backend_dev_get(i); + if (device == IntPtr.Zero) + continue; + + var name = NativeApi.ggml_backend_dev_name(device).PtrToString(); + if (!string.IsNullOrEmpty(name)) + devicesByName[name!] = device; + } + + // Resolve the requested names in order. One extra slot for the null terminator. + var devicesCount = 0; + var devicesArray = new IntPtr[deviceNames.Count + 1]; + foreach (var name in deviceNames) + { + if (!string.IsNullOrEmpty(name) && devicesByName.TryGetValue(name, out var device)) + devicesArray[devicesCount++] = device; + } + + // Early out if none of the names matched + if (devicesCount == 0) + return null; + + // Pin the array so it can be passed to native code + var pin = devicesArray.AsMemory().Pin(); + disposer.Add(pin); + return (IntPtr*)pin.Pointer; + } + private static unsafe LLamaModelTensorBufferOverride* ConvertOverrides(List overrides, GroupDisposable disposer) { // Early out if there are no overrides diff --git a/LLama/Native/LLamaModelParams.cs b/LLama/Native/LLamaModelParams.cs index ce6ab617a..796ba60e1 100644 --- a/LLama/Native/LLamaModelParams.cs +++ b/LLama/Native/LLamaModelParams.cs @@ -9,10 +9,10 @@ namespace LLama.Native public unsafe struct LLamaModelParams { /// - /// NULL-terminated list of devices to use for offloading (if NULL, all available devices are used) - /// todo: add support for llama_model_params.devices + /// NULL-terminated list of devices to use for offloading (if NULL, all available devices are used). + /// Each element is a ggml_backend_dev_t as returned by . /// - private IntPtr devices; + public IntPtr* devices; /// /// NULL-terminated list of buffer types to use for tensors that match a pattern diff --git a/LLama/Native/NativeApi.cs b/LLama/Native/NativeApi.cs index 2e06f83ee..26b3c9dcd 100644 --- a/LLama/Native/NativeApi.cs +++ b/LLama/Native/NativeApi.cs @@ -448,6 +448,14 @@ public static string llama_split_prefix(string splitPath, int splitNo, int split [DllImport(ggmlLibraryName, CallingConvention = CallingConvention.Cdecl)] public static extern IntPtr ggml_backend_dev_get(nuint i); + /// + /// Get the name of a backend device (e.g. "CPU", "Vulkan0", "CUDA1") + /// + /// Backend device pointer + /// Pointer to a null terminated UTF-8 string, owned by the device + [DllImport(ggmlBaseLibraryName, CallingConvention = CallingConvention.Cdecl)] + public static extern IntPtr ggml_backend_dev_name(IntPtr dev); + /// /// Get the buffer type for a backend device /// From d7047512de00f0e09448f1e3a450ef0d3252cb3a Mon Sep 17 00:00:00 2001 From: Brooks Date: Mon, 5 Oct 2026 10:36:16 +0800 Subject: [PATCH 2/3] Throw UnknownDeviceException for unknown names in IModelParams.Devices Review feedback: a device name that does not match any available device was silently dropped. ConvertDevices now resolves each name with an exact match first, then a case insensitive match, and throws UnknownDeviceException (new, in LLama/Exceptions) if neither matches. The exception carries the requested name and the list of available device names, and both appear in the message. Adds unit tests for the unknown-name and case-insensitive cases. Co-Authored-By: Claude Fable 5.1 --- LLama.Unittest/ModelsParamsTests.cs | 33 ++++++++++++++++ LLama/Abstractions/IModelParams.cs | 2 +- LLama/Exceptions/UnknownDeviceException.cs | 33 ++++++++++++++++ LLama/Extensions/IModelParamsExtensions.cs | 45 +++++++++++++++------- 4 files changed, 99 insertions(+), 14 deletions(-) create mode 100644 LLama/Exceptions/UnknownDeviceException.cs diff --git a/LLama.Unittest/ModelsParamsTests.cs b/LLama.Unittest/ModelsParamsTests.cs index 99f3155ef..ff6980370 100644 --- a/LLama.Unittest/ModelsParamsTests.cs +++ b/LLama.Unittest/ModelsParamsTests.cs @@ -1,6 +1,8 @@ using LLama.Common; using System.Text.Json; using LLama.Abstractions; +using LLama.Exceptions; +using LLama.Extensions; namespace LLama.Unittest { @@ -59,5 +61,36 @@ public void SerializeRoundTripSystemTextJson() Assert.Equal(expected, actual); } + + [Fact] + public void UnknownDeviceThrows() + { + var @params = new ModelParams("abc/123") + { + Devices = { "NoSuchDevice" }, + }; + + var ex = Assert.Throws(() => @params.ToLlamaModelParams(out _)); + + Assert.Equal("NoSuchDevice", ex.RequestedDevice); + Assert.Contains("CPU", ex.AvailableDevices); + Assert.Contains("NoSuchDevice", ex.Message); + Assert.Contains("CPU", ex.Message); + } + + [Fact] + public unsafe void DeviceNameMatchIsCaseInsensitive() + { + var @params = new ModelParams("abc/123") + { + Devices = { "cpu" }, + }; + + using var disposer = @params.ToLlamaModelParams(out var result); + + Assert.True(result.devices != null); + Assert.NotEqual(IntPtr.Zero, result.devices[0]); + Assert.Equal(IntPtr.Zero, result.devices[1]); + } } } diff --git a/LLama/Abstractions/IModelParams.cs b/LLama/Abstractions/IModelParams.cs index 8e632c378..cc2145cf4 100644 --- a/LLama/Abstractions/IModelParams.cs +++ b/LLama/Abstractions/IModelParams.cs @@ -51,7 +51,7 @@ public interface IModelParams /// /// When empty, llama.cpp picks the devices itself: all discrete GPUs, or the first integrated GPU when there is no discrete GPU. /// Setting this list explicitly bypasses that selection, which is the only way to run on an integrated GPU in a machine that also has a discrete GPU. - /// is an index into this list when it is non-empty. Names that do not match an available device are ignored. + /// is an index into this list when it is non-empty. Names are matched case insensitively; a name that does not match any available device throws when the model is loaded. /// List Devices { get; } diff --git a/LLama/Exceptions/UnknownDeviceException.cs b/LLama/Exceptions/UnknownDeviceException.cs new file mode 100644 index 000000000..b8dd6784d --- /dev/null +++ b/LLama/Exceptions/UnknownDeviceException.cs @@ -0,0 +1,33 @@ +using System; +using System.Collections.Generic; + +namespace LLama.Exceptions; + +/// +/// Thrown when a device name in does not match any available ggml backend device +/// +public class UnknownDeviceException + : Exception +{ + /// + /// The device name which was requested but could not be found + /// + public string RequestedDevice { get; } + + /// + /// Names of all devices available on this machine, as returned by + /// + public IReadOnlyList AvailableDevices { get; } + + /// + /// Create a new UnknownDeviceException + /// + /// The device name which was requested but could not be found + /// Names of all devices available on this machine + public UnknownDeviceException(string requestedDevice, IReadOnlyList availableDevices) + : base($"Unknown device '{requestedDevice}'. Available devices: {string.Join(", ", availableDevices)}") + { + RequestedDevice = requestedDevice; + AvailableDevices = availableDevices; + } +} diff --git a/LLama/Extensions/IModelParamsExtensions.cs b/LLama/Extensions/IModelParamsExtensions.cs index 8c3497b4a..f5753b389 100644 --- a/LLama/Extensions/IModelParamsExtensions.cs +++ b/LLama/Extensions/IModelParamsExtensions.cs @@ -2,6 +2,7 @@ using System; using System.Text; using LLama.Abstractions; +using LLama.Exceptions; using LLama.Native; using System.Collections.Generic; @@ -20,6 +21,7 @@ public static class IModelParamsExtensions /// /// /// + /// Thrown if a name in does not match any available device public static IDisposable ToLlamaModelParams(this IModelParams @params, out LLamaModelParams result) { var supportsMmap = NativeApi.llama_supports_mmap(); @@ -141,8 +143,9 @@ private static IReadOnlyDictionary GetAvailableBufferTypes() if (deviceNames.Count == 0) return null; - // Map device name -> ggml_backend_dev_t + // Map device name -> ggml_backend_dev_t, keeping the names in native order for error messages var devicesByName = new Dictionary(); + var availableNames = new List(); var deviceCount = NativeApi.ggml_backend_dev_count(); for (nuint i = 0; i < deviceCount; i++) { @@ -151,22 +154,17 @@ private static IReadOnlyDictionary GetAvailableBufferTypes() continue; var name = NativeApi.ggml_backend_dev_name(device).PtrToString(); - if (!string.IsNullOrEmpty(name)) - devicesByName[name!] = device; + if (string.IsNullOrEmpty(name)) + continue; + + devicesByName[name!] = device; + availableNames.Add(name!); } // Resolve the requested names in order. One extra slot for the null terminator. - var devicesCount = 0; var devicesArray = new IntPtr[deviceNames.Count + 1]; - foreach (var name in deviceNames) - { - if (!string.IsNullOrEmpty(name) && devicesByName.TryGetValue(name, out var device)) - devicesArray[devicesCount++] = device; - } - - // Early out if none of the names matched - if (devicesCount == 0) - return null; + for (var i = 0; i < deviceNames.Count; i++) + devicesArray[i] = ResolveDevice(deviceNames[i], devicesByName, availableNames); // Pin the array so it can be passed to native code var pin = devicesArray.AsMemory().Pin(); @@ -174,6 +172,27 @@ private static IReadOnlyDictionary GetAvailableBufferTypes() return (IntPtr*)pin.Pointer; } + /// + /// Find the device with the given name. Tries an exact match first, then a case insensitive match. + /// + /// Thrown if no device matches the name + private static IntPtr ResolveDevice(string name, Dictionary devicesByName, List availableNames) + { + if (!string.IsNullOrEmpty(name)) + { + if (devicesByName.TryGetValue(name, out var device)) + return device; + + foreach (var pair in devicesByName) + { + if (string.Equals(pair.Key, name, StringComparison.OrdinalIgnoreCase)) + return pair.Value; + } + } + + throw new UnknownDeviceException(name ?? "", availableNames); + } + private static unsafe LLamaModelTensorBufferOverride* ConvertOverrides(List overrides, GroupDisposable disposer) { // Early out if there are no overrides From 6789992b6b8c339e93a0f5ec238e0fa737a5e26e Mon Sep 17 00:00:00 2001 From: Brooks Date: Mon, 5 Oct 2026 10:44:56 +0800 Subject: [PATCH 3/3] Resolve ggml/ggml-base P/Invokes to the dependency libraries already loaded The DllImport resolver only handled "llama" and "mtmd" and returned IntPtr.Zero for "ggml" and "ggml-base", leaving them to the default runtime probing. NativeLibraryUtils loads both by full path as dependencies of llama, which the default probing finds again by name on Windows and Linux but not on macOS (dlopen of a bare "libggml.dylib" does not match the @rpath install name of the already-loaded library), so any P/Invoke into ggml failed there with DllNotFoundException. This broke the new Devices tests on the macOS CI job and also affects TensorBufferOverrides, which uses the same imports. Keep the ggml and ggml-base handles when they are loaded as dependencies, and have the resolver return them (loading llama first if needed). Co-Authored-By: Claude Fable 5.1 --- LLama/Native/Load/NativeLibraryUtils.cs | 24 ++++++++++++++++++++++-- LLama/Native/NativeApi.Load.cs | 11 +++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/LLama/Native/Load/NativeLibraryUtils.cs b/LLama/Native/Load/NativeLibraryUtils.cs index 03d436447..b90ff93dc 100644 --- a/LLama/Native/Load/NativeLibraryUtils.cs +++ b/LLama/Native/Load/NativeLibraryUtils.cs @@ -8,6 +8,18 @@ namespace LLama.Native { internal static class NativeLibraryUtils { + /// + /// Handle of the ggml library loaded as a dependency by , or IntPtr.Zero if it has not been loaded. + /// Used by the DllImport resolver so that P/Invokes into ggml reuse this library instead of relying on the system search path. + /// + internal static IntPtr LoadedGgmlHandle; + + /// + /// Handle of the ggml-base library loaded as a dependency by , or IntPtr.Zero if it has not been loaded. + /// Used by the DllImport resolver so that P/Invokes into ggml-base reuse this library instead of relying on the system search path. + /// + internal static IntPtr LoadedGgmlBaseHandle; + /// /// Try to load libllama/mtmd, using CPU feature detection to try and load a more specialised DLL if possible /// @@ -64,7 +76,8 @@ internal static IntPtr TryLoadLibrary(NativeLibraryConfig config, out INativeLib var dependencyPaths = new List(); // We should always load ggml-base from the current runtime directory - dependencyPaths.Add(Path.Combine(currentRuntimeDirectory, $"{libPrefix}ggml-base{ext}")); + var ggmlBasePath = Path.Combine(currentRuntimeDirectory, $"{libPrefix}ggml-base{ext}"); + dependencyPaths.Add(ggmlBasePath); // If the library has metadata, we can check if we need to load additional dependencies if (library.Metadata != null) @@ -114,13 +127,20 @@ internal static IntPtr TryLoadLibrary(NativeLibraryConfig config, out INativeLib } // And finally, we can add ggml - dependencyPaths.Add(Path.Combine(currentRuntimeDirectory, $"{libPrefix}ggml{ext}")); + var ggmlPath = Path.Combine(currentRuntimeDirectory, $"{libPrefix}ggml{ext}"); + dependencyPaths.Add(ggmlPath); // Now, we will loop through our dependencyPaths and try to load them one by one foreach (var dependencyPath in dependencyPaths) { // Try to load the dependency var dependencyResult = TryLoad(dependencyPath, description.SearchDirectories, config.LogCallback); + + // Keep the ggml/ggml-base handles so the DllImport resolver can hand them out for P/Invokes into those libraries + if (dependencyPath == ggmlBasePath) + LoadedGgmlBaseHandle = dependencyResult; + else if (dependencyPath == ggmlPath) + LoadedGgmlHandle = dependencyResult; // If we successfully loaded the library, log it if (dependencyResult != IntPtr.Zero) diff --git a/LLama/Native/NativeApi.Load.cs b/LLama/Native/NativeApi.Load.cs index 57bb2d146..2a0052b7d 100644 --- a/LLama/Native/NativeApi.Load.cs +++ b/LLama/Native/NativeApi.Load.cs @@ -72,6 +72,17 @@ private static void SetDllImportResolver() return _loadedLlamaHandle; } + if (name == ggmlLibraryName || name == ggmlBaseLibraryName) + { + // ggml and ggml-base are loaded by full path as dependencies of llama, so the default runtime + // resolution cannot be relied on to find them by name (it fails on macOS). Make sure llama (and + // therefore its dependencies) has been loaded, then return the dependency handle. + if (_loadedLlamaHandle == IntPtr.Zero) + _loadedLlamaHandle = NativeLibraryUtils.TryLoadLibrary(NativeLibraryConfig.LLama, out _loadedLLamaLibrary); + + return name == ggmlLibraryName ? NativeLibraryUtils.LoadedGgmlHandle : NativeLibraryUtils.LoadedGgmlBaseHandle; + } + if (name == "mtmd") { // If we've already loaded Mtmd return the handle that was loaded last time.