From aeb217692a86bb4ad71fe9c8a6dd0c6bd554a92e Mon Sep 17 00:00:00 2001 From: Chandan Kumar Date: Fri, 18 Sep 2026 12:53:21 +0530 Subject: [PATCH 1/2] Add CUDA 13.0.0 jobs to the native compile workflow. Extend the cublas matrix and Gather Binaries layout so a Compile run can produce cu13.0.0 artifacts next to cu12.4.0 without changing the managed loader or NuGet packages yet. --- .github/workflows/compile.yml | 37 +++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index 51691dfc4..1f42b1b41 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -405,7 +405,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-22.04, windows-2022] - cuda: ['12.4.0'] + cuda: ['12.4.0', '13.0.0'] runs-on: ${{ matrix.os }} steps: - name: Clone @@ -417,7 +417,7 @@ jobs: ref: '${{ github.event.inputs.llama_cpp_commit }}' - uses: Jimver/cuda-toolkit@v0.2.15 - if: runner.os == 'Windows' + if: runner.os == 'Windows' && matrix.cuda == '12.4.0' id: cuda-toolkit-windows with: cuda: ${{ matrix.cuda }} @@ -425,13 +425,30 @@ jobs: sub-packages: '["nvcc", "cudart", "cublas", "cublas_dev", "thrust", "visual_studio_integration"]' - uses: Jimver/cuda-toolkit@v0.2.15 - if: runner.os == 'Linux' + if: runner.os == 'Linux' && matrix.cuda == '12.4.0' id: cuda-toolkit-linux with: cuda: ${{ matrix.cuda }} method: 'network' linux-local-args: '["--toolkit"]' + - uses: Jimver/cuda-toolkit@v0.2.36 + if: runner.os == 'Windows' && matrix.cuda != '12.4.0' + id: cuda-toolkit-windows-13 + with: + cuda: ${{ matrix.cuda }} + method: 'network' + sub-packages: '["nvcc", "cudart", "cublas", "cublas_dev", "thrust", "visual_studio_integration"]' + use-github-cache: false + + - uses: Jimver/cuda-toolkit@v0.2.36 + if: runner.os == 'Linux' && matrix.cuda != '12.4.0' + id: cuda-toolkit-linux-13 + with: + cuda: ${{ matrix.cuda }} + method: 'network' + linux-local-args: '["--toolkit"]' + - name: Build (Windows) if: runner.os == 'Windows' id: cmake_build_windows @@ -688,7 +705,7 @@ jobs: - name: Rearrange Files run: | # Make all directories at once - mkdir --parents deps/{noavx,avx,avx2,avx512,linux-arm64,musl-noavx,musl-avx,musl-avx2,musl-avx512,osx-arm64,osx-x64,osx-x64-rosetta2,cu12.4.0,vulkan,android-arm64-v8a,android-x86,android-x86_64,win-arm64} + mkdir --parents deps/{noavx,avx,avx2,avx512,linux-arm64,musl-noavx,musl-avx,musl-avx2,musl-avx512,osx-arm64,osx-x64,osx-x64-rosetta2,cu12.4.0,cu13.0.0,vulkan,android-arm64-v8a,android-x86,android-x86_64,win-arm64} # Linux cp artifacts/ggml-bin-linux-noavx-x64.so/libggml.so deps/noavx/libggml.so @@ -821,6 +838,12 @@ jobs: cp artifacts/llama-bin-win-cublas-cu12.4.0-x64.dll/llama.dll deps/cu12.4.0/llama.dll cp artifacts/mtmd-bin-win-cublas-cu12.4.0-x64.dll/mtmd.dll deps/cu12.4.0/mtmd.dll + cp artifacts/ggml-bin-win-cublas-cu13.0.0-x64.dll/ggml.dll deps/cu13.0.0/ggml.dll + cp artifacts/ggml-base-bin-win-cublas-cu13.0.0-x64.dll/ggml-base.dll deps/cu13.0.0/ggml-base.dll + cp artifacts/ggml-cuda-bin-win-cublas-cu13.0.0-x64.dll/ggml-cuda.dll deps/cu13.0.0/ggml-cuda.dll + cp artifacts/llama-bin-win-cublas-cu13.0.0-x64.dll/llama.dll deps/cu13.0.0/llama.dll + cp artifacts/mtmd-bin-win-cublas-cu13.0.0-x64.dll/mtmd.dll deps/cu13.0.0/mtmd.dll + # Linux CUDA cp artifacts/ggml-bin-linux-cublas-cu12.4.0-x64.so/libggml.so deps/cu12.4.0/libggml.so cp artifacts/ggml-base-bin-linux-cublas-cu12.4.0-x64.so/libggml-base.so deps/cu12.4.0/libggml-base.so @@ -828,6 +851,12 @@ jobs: cp artifacts/llama-bin-linux-cublas-cu12.4.0-x64.so/libllama.so deps/cu12.4.0/libllama.so cp artifacts/mtmd-bin-linux-cublas-cu12.4.0-x64.so/libmtmd.so deps/cu12.4.0/libmtmd.so + cp artifacts/ggml-bin-linux-cublas-cu13.0.0-x64.so/libggml.so deps/cu13.0.0/libggml.so + cp artifacts/ggml-base-bin-linux-cublas-cu13.0.0-x64.so/libggml-base.so deps/cu13.0.0/libggml-base.so + cp artifacts/ggml-cuda-bin-linux-cublas-cu13.0.0-x64.so/libggml-cuda.so deps/cu13.0.0/libggml-cuda.so + cp artifacts/llama-bin-linux-cublas-cu13.0.0-x64.so/libllama.so deps/cu13.0.0/libllama.so + cp artifacts/mtmd-bin-linux-cublas-cu13.0.0-x64.so/libmtmd.so deps/cu13.0.0/libmtmd.so + # Windows Vulkan cp artifacts/ggml-bin-win-vulkan-x64.dll/ggml.dll deps/vulkan/ggml.dll cp artifacts/ggml-base-bin-win-vulkan-x64.dll/ggml-base.dll deps/vulkan/ggml-base.dll From ac99b14334ca283b4f841f589ae5ab1a382bf55d Mon Sep 17 00:00:00 2001 From: Chandan Kumar Date: Fri, 18 Sep 2026 12:53:21 +0530 Subject: [PATCH 2/2] Probe cuda13 native libraries before cuda12. Teach NativeLibraryWithCuda to try major 13 when SkipCheck is on or the detected major is 13, and cover the path order with unit tests. Binaries and NuGet packages stay out of this change. --- LLama.Unittest/NativeLibraryWithCudaTests.cs | 61 ++++++++++++++++++++ LLama/Native/Load/NativeLibraryWithCuda.cs | 7 ++- 2 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 LLama.Unittest/NativeLibraryWithCudaTests.cs diff --git a/LLama.Unittest/NativeLibraryWithCudaTests.cs b/LLama.Unittest/NativeLibraryWithCudaTests.cs new file mode 100644 index 000000000..a1ae21643 --- /dev/null +++ b/LLama.Unittest/NativeLibraryWithCudaTests.cs @@ -0,0 +1,61 @@ +using LLama.Native; +using System.Runtime.InteropServices; + +namespace LLama.Unittest +{ + public class NativeLibraryWithCudaTests + { + [Fact] + public void SkipCheckUnknownMajorOnLinuxYieldsCuda13Then12Then11() + { + var library = new NativeLibraryWithCuda(-1, NativeLibraryName.LLama, AvxLevel.Avx2, true); + var systemInfo = new SystemInfo(OSPlatform.Linux, -1, null); + + var paths = library.Prepare(systemInfo, null).ToArray(); + + Assert.Equal(3, paths.Length); + Assert.Contains("cuda13", paths[0]); + Assert.Contains("cuda12", paths[1]); + Assert.Contains("cuda11", paths[2]); + } + + [Fact] + public void DetectedMajor13OnWindowsYieldsCuda13Only() + { + var library = new NativeLibraryWithCuda(13, NativeLibraryName.LLama, AvxLevel.Avx2, false); + var systemInfo = new SystemInfo(OSPlatform.Windows, 13, null); + + var paths = library.Prepare(systemInfo, null).ToArray(); + + Assert.Single(paths); + Assert.Contains("cuda13", paths[0]); + Assert.DoesNotContain("cuda12", paths[0]); + Assert.DoesNotContain("cuda11", paths[0]); + } + + [Fact] + public void DetectedMajor12YieldsCuda12Only() + { + var library = new NativeLibraryWithCuda(12, NativeLibraryName.LLama, AvxLevel.Avx2, false); + var systemInfo = new SystemInfo(OSPlatform.Linux, 12, null); + + var paths = library.Prepare(systemInfo, null).ToArray(); + + Assert.Single(paths); + Assert.Contains("cuda12", paths[0]); + Assert.DoesNotContain("cuda13", paths[0]); + Assert.DoesNotContain("cuda11", paths[0]); + } + + [Fact] + public void SkipCheckFalseUnknownMajorYieldsEmpty() + { + var library = new NativeLibraryWithCuda(-1, NativeLibraryName.LLama, AvxLevel.Avx2, false); + var systemInfo = new SystemInfo(OSPlatform.Linux, -1, null); + + var paths = library.Prepare(systemInfo, null).ToArray(); + + Assert.Empty(paths); + } + } +} diff --git a/LLama/Native/Load/NativeLibraryWithCuda.cs b/LLama/Native/Load/NativeLibraryWithCuda.cs index 36dc4ca81..569d2f930 100644 --- a/LLama/Native/Load/NativeLibraryWithCuda.cs +++ b/LLama/Native/Load/NativeLibraryWithCuda.cs @@ -46,7 +46,12 @@ public IEnumerable Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaL { if (_majorCudaVersion == -1 && _skipCheck) { - // Currently only 11 and 12 are supported. + // Currently 11, 12, and 13 are supported. + var cuda13LibraryPath = GetCudaPath(systemInfo, 13, logCallback); + if (cuda13LibraryPath is not null) + { + yield return cuda13LibraryPath; + } var cuda12LibraryPath = GetCudaPath(systemInfo, 12, logCallback); if (cuda12LibraryPath is not null) {