feat(cuda): add LLamaSharp.Backend.Cuda13 packages

.github/workflows/compile.yml

@@ -405,7 +405,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-22.04, windows-2022]
-        cuda: ['12.4.0']
+        cuda: ['12.4.0', '13.0.0']
     runs-on: ${{ matrix.os }}
     steps:
       - name: Clone
@@ -417,21 +417,38 @@ jobs:
           ref: '${{ github.event.inputs.llama_cpp_commit }}'
 
       - uses: Jimver/cuda-toolkit@v0.2.15
-        if: runner.os == 'Windows'
+        if: runner.os == 'Windows' && matrix.cuda == '12.4.0'
         id: cuda-toolkit-windows
         with:
           cuda: ${{ matrix.cuda }}
           method: 'network'
           sub-packages: '["nvcc", "cudart", "cublas", "cublas_dev", "thrust", "visual_studio_integration"]'
 
       - uses: Jimver/cuda-toolkit@v0.2.15
-        if: runner.os == 'Linux'
+        if: runner.os == 'Linux' && matrix.cuda == '12.4.0'
         id: cuda-toolkit-linux
         with:
           cuda: ${{ matrix.cuda }}
           method: 'network'
           linux-local-args: '["--toolkit"]'
 
+      - uses: Jimver/cuda-toolkit@v0.2.36
+        if: runner.os == 'Windows' && matrix.cuda != '12.4.0'
+        id: cuda-toolkit-windows-13
+        with:
+          cuda: ${{ matrix.cuda }}
+          method: 'network'
+          sub-packages: '["nvcc", "cudart", "cublas", "cublas_dev", "thrust", "visual_studio_integration"]'
+          use-github-cache: false
+
+      - uses: Jimver/cuda-toolkit@v0.2.36
+        if: runner.os == 'Linux' && matrix.cuda != '12.4.0'
+        id: cuda-toolkit-linux-13
+        with:
+          cuda: ${{ matrix.cuda }}
+          method: 'network'
+          linux-local-args: '["--toolkit"]'
+
       - name: Build (Windows)
         if: runner.os == 'Windows'
         id: cmake_build_windows
@@ -688,7 +705,7 @@ jobs:
       - name: Rearrange Files
         run: |
           # Make all directories at once
-          mkdir --parents deps/{noavx,avx,avx2,avx512,linux-arm64,musl-noavx,musl-avx,musl-avx2,musl-avx512,osx-arm64,osx-x64,osx-x64-rosetta2,cu12.4.0,vulkan,android-arm64-v8a,android-x86,android-x86_64,win-arm64}
+          mkdir --parents deps/{noavx,avx,avx2,avx512,linux-arm64,musl-noavx,musl-avx,musl-avx2,musl-avx512,osx-arm64,osx-x64,osx-x64-rosetta2,cu12.4.0,cu13.0.0,vulkan,android-arm64-v8a,android-x86,android-x86_64,win-arm64}
 
           # Linux
           cp artifacts/ggml-bin-linux-noavx-x64.so/libggml.so           deps/noavx/libggml.so
@@ -821,13 +838,25 @@ jobs:
           cp artifacts/llama-bin-win-cublas-cu12.4.0-x64.dll/llama.dll          deps/cu12.4.0/llama.dll
           cp artifacts/mtmd-bin-win-cublas-cu12.4.0-x64.dll/mtmd.dll            deps/cu12.4.0/mtmd.dll
 
+          cp artifacts/ggml-bin-win-cublas-cu13.0.0-x64.dll/ggml.dll            deps/cu13.0.0/ggml.dll
+          cp artifacts/ggml-base-bin-win-cublas-cu13.0.0-x64.dll/ggml-base.dll  deps/cu13.0.0/ggml-base.dll
+          cp artifacts/ggml-cuda-bin-win-cublas-cu13.0.0-x64.dll/ggml-cuda.dll  deps/cu13.0.0/ggml-cuda.dll
+          cp artifacts/llama-bin-win-cublas-cu13.0.0-x64.dll/llama.dll          deps/cu13.0.0/llama.dll
+          cp artifacts/mtmd-bin-win-cublas-cu13.0.0-x64.dll/mtmd.dll            deps/cu13.0.0/mtmd.dll
+
           # Linux CUDA
           cp artifacts/ggml-bin-linux-cublas-cu12.4.0-x64.so/libggml.so              deps/cu12.4.0/libggml.so
           cp artifacts/ggml-base-bin-linux-cublas-cu12.4.0-x64.so/libggml-base.so    deps/cu12.4.0/libggml-base.so
           cp artifacts/ggml-cuda-bin-linux-cublas-cu12.4.0-x64.so/libggml-cuda.so    deps/cu12.4.0/libggml-cuda.so
           cp artifacts/llama-bin-linux-cublas-cu12.4.0-x64.so/libllama.so            deps/cu12.4.0/libllama.so
           cp artifacts/mtmd-bin-linux-cublas-cu12.4.0-x64.so/libmtmd.so              deps/cu12.4.0/libmtmd.so
 
+          cp artifacts/ggml-bin-linux-cublas-cu13.0.0-x64.so/libggml.so              deps/cu13.0.0/libggml.so
+          cp artifacts/ggml-base-bin-linux-cublas-cu13.0.0-x64.so/libggml-base.so    deps/cu13.0.0/libggml-base.so
+          cp artifacts/ggml-cuda-bin-linux-cublas-cu13.0.0-x64.so/libggml-cuda.so    deps/cu13.0.0/libggml-cuda.so
+          cp artifacts/llama-bin-linux-cublas-cu13.0.0-x64.so/libllama.so            deps/cu13.0.0/libllama.so
+          cp artifacts/mtmd-bin-linux-cublas-cu13.0.0-x64.so/libmtmd.so              deps/cu13.0.0/libmtmd.so
+
           # Windows Vulkan
           cp artifacts/ggml-bin-win-vulkan-x64.dll/ggml.dll               deps/vulkan/ggml.dll
           cp artifacts/ggml-base-bin-win-vulkan-x64.dll/ggml-base.dll     deps/vulkan/ggml-base.dll

LLama.Unittest/NativeLibraryWithCudaTests.cs

@@ -0,0 +1,61 @@
+using LLama.Native;
+using System.Runtime.InteropServices;
+
+namespace LLama.Unittest
+{
+    public class NativeLibraryWithCudaTests
+    {
+        [Fact]
+        public void SkipCheckUnknownMajorOnLinuxYieldsCuda13Then12Then11()
+        {
+            var library = new NativeLibraryWithCuda(-1, NativeLibraryName.LLama, AvxLevel.Avx2, true);
+            var systemInfo = new SystemInfo(OSPlatform.Linux, -1, null);
+
+            var paths = library.Prepare(systemInfo, null).ToArray();
+
+            Assert.Equal(3, paths.Length);
+            Assert.Contains("cuda13", paths[0]);
+            Assert.Contains("cuda12", paths[1]);
+            Assert.Contains("cuda11", paths[2]);
+        }
+
+        [Fact]
+        public void DetectedMajor13OnWindowsYieldsCuda13Only()
+        {
+            var library = new NativeLibraryWithCuda(13, NativeLibraryName.LLama, AvxLevel.Avx2, false);
+            var systemInfo = new SystemInfo(OSPlatform.Windows, 13, null);
+
+            var paths = library.Prepare(systemInfo, null).ToArray();
+
+            Assert.Single(paths);
+            Assert.Contains("cuda13", paths[0]);
+            Assert.DoesNotContain("cuda12", paths[0]);
+            Assert.DoesNotContain("cuda11", paths[0]);
+        }
+
+        [Fact]
+        public void DetectedMajor12YieldsCuda12Only()
+        {
+            var library = new NativeLibraryWithCuda(12, NativeLibraryName.LLama, AvxLevel.Avx2, false);
+            var systemInfo = new SystemInfo(OSPlatform.Linux, 12, null);
+
+            var paths = library.Prepare(systemInfo, null).ToArray();
+
+            Assert.Single(paths);
+            Assert.Contains("cuda12", paths[0]);
+            Assert.DoesNotContain("cuda13", paths[0]);
+            Assert.DoesNotContain("cuda11", paths[0]);
+        }
+
+        [Fact]
+        public void SkipCheckFalseUnknownMajorYieldsEmpty()
+        {
+            var library = new NativeLibraryWithCuda(-1, NativeLibraryName.LLama, AvxLevel.Avx2, false);
+            var systemInfo = new SystemInfo(OSPlatform.Linux, -1, null);
+
+            var paths = library.Prepare(systemInfo, null).ToArray();
+
+            Assert.Empty(paths);
+        }
+    }
+}

LLama/LLamaSharp.Runtime.targets

@@ -94,6 +94,24 @@
       </None>
 
 
+      <None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu13.0.0/llama.dll" Condition="Exists('$(MSBuildThisFileDirectory)runtimes/deps/cu13.0.0/llama.dll')">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+        <Link>runtimes/win-x64/native/cuda13/llama.dll</Link>
+      </None>
+      <None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu13.0.0/ggml-base.dll" Condition="Exists('$(MSBuildThisFileDirectory)runtimes/deps/cu13.0.0/ggml-base.dll')">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+        <Link>runtimes/win-x64/native/cuda13/ggml-base.dll</Link>
+      </None>
+      <None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu13.0.0/ggml.dll" Condition="Exists('$(MSBuildThisFileDirectory)runtimes/deps/cu13.0.0/ggml.dll')">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+        <Link>runtimes/win-x64/native/cuda13/ggml.dll</Link>
+      </None>
+      <None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu13.0.0/ggml-cuda.dll" Condition="Exists('$(MSBuildThisFileDirectory)runtimes/deps/cu13.0.0/ggml-cuda.dll')">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+        <Link>runtimes/win-x64/native/cuda13/ggml-cuda.dll</Link>
+      </None>
+
+
       <None Include="$(MSBuildThisFileDirectory)runtimes/deps/vulkan/llama.dll">
         <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
         <Link>runtimes/win-x64/native/vulkan/llama.dll</Link>
@@ -243,6 +261,24 @@
       </None>
 
 
+      <None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu13.0.0/libllama.so" Condition="Exists('$(MSBuildThisFileDirectory)runtimes/deps/cu13.0.0/libllama.so')">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+        <Link>runtimes/linux-x64/native/cuda13/libllama.so</Link>
+      </None>
+      <None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu13.0.0/libggml.so" Condition="Exists('$(MSBuildThisFileDirectory)runtimes/deps/cu13.0.0/libggml.so')">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+        <Link>runtimes/linux-x64/native/cuda13/libggml.so</Link>
+      </None>
+      <None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu13.0.0/libggml-base.so" Condition="Exists('$(MSBuildThisFileDirectory)runtimes/deps/cu13.0.0/libggml-base.so')">
+          <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+          <Link>runtimes/linux-x64/native/cuda13/libggml-base.so</Link>
+      </None>
+      <None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu13.0.0/libggml-cuda.so" Condition="Exists('$(MSBuildThisFileDirectory)runtimes/deps/cu13.0.0/libggml-cuda.so')">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+        <Link>runtimes/linux-x64/native/cuda13/libggml-cuda.so</Link>
+      </None>
+
+
       <None Include="$(MSBuildThisFileDirectory)runtimes/deps/vulkan/libllama.so">
         <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
         <Link>runtimes/linux-x64/native/vulkan/libllama.so</Link>
@@ -428,6 +464,10 @@
         <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
         <Link>runtimes/win-x64/native/cuda12/libmtmd.dll</Link>
       </None>
+      <None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu13.0.0/mtmd.dll" Condition="Exists('$(MSBuildThisFileDirectory)runtimes/deps/cu13.0.0/mtmd.dll')">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+        <Link>runtimes/win-x64/native/cuda13/libmtmd.dll</Link>
+      </None>
       <None Include="$(MSBuildThisFileDirectory)runtimes/deps/vulkan/mtmd.dll">
         <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
         <Link>runtimes/win-x64/native/vulkan/mtmd.dll</Link>
@@ -457,6 +497,10 @@
         <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
         <Link>runtimes/linux-x64/native/cuda12/libmtmd.so</Link>
       </None>
+      <None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu13.0.0/libmtmd.so" Condition="Exists('$(MSBuildThisFileDirectory)runtimes/deps/cu13.0.0/libmtmd.so')">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+        <Link>runtimes/linux-x64/native/cuda13/libmtmd.so</Link>
+      </None>
       <None Include="$(MSBuildThisFileDirectory)runtimes/deps/vulkan/libmtmd.so">
         <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
         <Link>runtimes/linux-x64/native/vulkan/libmtmd.so</Link>

LLama/Native/Load/NativeLibraryWithCuda.cs

@@ -46,7 +46,12 @@ public IEnumerable<string> Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaL
             {
                 if (_majorCudaVersion == -1 && _skipCheck)
                 {
-                    // Currently only 11 and 12 are supported.
+                    // Currently 11, 12, and 13 are supported.
+                    var cuda13LibraryPath = GetCudaPath(systemInfo, 13, logCallback);
+                    if (cuda13LibraryPath is not null)
+                    {
+                        yield return cuda13LibraryPath;
+                    }
                     var cuda12LibraryPath = GetCudaPath(systemInfo, 12, logCallback);
                     if (cuda12LibraryPath is not null)
                     {

LLama/runtimes/build/LLamaSharp.Backend.Cuda13.Linux.nuspec

@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="utf-8"?>
+<package >
+    <metadata>
+        <id>LLamaSharp.Backend.Cuda13.Linux</id>
+        <version>$version$</version>
+        <title>LLamaSharp.Backend.Cuda13.Linux</title>
+        <authors>llama.cpp Authors</authors>
+        <requireLicenseAcceptance>false</requireLicenseAcceptance>
+        <license type="expression">MIT</license>
+        <icon>icon512.png</icon>
+        <projectUrl>https://github.com/SciSharp/LLamaSharp</projectUrl>
+        <description>LLamaSharp.Backend.Cuda13.Linux contains the Linux binaries for LLamaSharp with Cuda13 support.</description>
+        <releaseNotes></releaseNotes>
+        <copyright>Copyright 2023 The llama.cpp Authors. All rights reserved.</copyright>
+        <tags>LLamaSharp LLama LLM GPT AI ChatBot SciSharp</tags>
+
+        <dependencies>
+            <dependency id="LLamaSharp.Backend.Cpu" version="$version$" />
+        </dependencies>
+    </metadata>
+
+    <files>
+        <file src="LLamaSharpBackend.props" target="build/netstandard2.0/LLamaSharp.Backend.Cuda13.Linux.props" />
+
+        <file src="runtimes/deps/cu13.0.0/libggml.so" target="LLamaSharpRuntimes/linux-x64/native/cuda13/libggml.so" />
+        <file src="runtimes/deps/cu13.0.0/libggml-base.so" target="LLamaSharpRuntimes/linux-x64/native/cuda13/libggml-base.so" />
+        <file src="runtimes/deps/cu13.0.0/libggml-cuda.so" target="LLamaSharpRuntimes/linux-x64/native/cuda13/libggml-cuda.so" />
+
+        <file src="runtimes/deps/cu13.0.0/libllama.so" target="LLamaSharpRuntimes/linux-x64/native/cuda13/libllama.so" />
+        <file src="runtimes/deps/cu13.0.0/libmtmd.so" target="LLamaSharpRuntimes/linux-x64/native/cuda13/libmtmd.so" />
+
+        <file src="icon512.png" target="icon512.png" />
+    </files>
+</package>

LLama/runtimes/build/LLamaSharp.Backend.Cuda13.Windows.nuspec

@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="utf-8"?>
+<package >
+    <metadata>
+        <id>LLamaSharp.Backend.Cuda13.Windows</id>
+        <version>$version$</version>
+        <title>LLamaSharp.Backend.Cuda13.Windows</title>
+        <authors>llama.cpp Authors</authors>
+        <requireLicenseAcceptance>false</requireLicenseAcceptance>
+        <license type="expression">MIT</license>
+        <icon>icon512.png</icon>
+        <projectUrl>https://github.com/SciSharp/LLamaSharp</projectUrl>
+        <description>LLamaSharp.Backend.Cuda13.Windows contains the Windows binaries for LLamaSharp with Cuda13 support.</description>
+        <releaseNotes></releaseNotes>
+        <copyright>Copyright 2023 The llama.cpp Authors. All rights reserved.</copyright>
+        <tags>LLamaSharp LLama LLM GPT AI ChatBot SciSharp</tags>
+
+        <dependencies>
+            <dependency id="LLamaSharp.Backend.Cpu" version="$version$" />
+        </dependencies>
+    </metadata>
+
+    <files>
+        <file src="LLamaSharpBackend.props" target="build/netstandard2.0/LLamaSharp.Backend.Cuda13.Windows.props" />
+
+        <file src="runtimes/deps/cu13.0.0/ggml.dll" target="LLamaSharpRuntimes\win-x64\native\cuda13\ggml.dll" />
+        <file src="runtimes/deps/cu13.0.0/ggml-base.dll" target="LLamaSharpRuntimes\win-x64\native\cuda13\ggml-base.dll" />
+        <file src="runtimes/deps/cu13.0.0/ggml-cuda.dll" target="LLamaSharpRuntimes\win-x64\native\cuda13\ggml-cuda.dll" />
+
+        <file src="runtimes/deps/cu13.0.0/llama.dll" target="LLamaSharpRuntimes\win-x64\native\cuda13\llama.dll" />
+        <file src="runtimes/deps/cu13.0.0/mtmd.dll" target="LLamaSharpRuntimes\win-x64\native\cuda13\mtmd.dll" />
+
+        <file src="icon512.png" target="icon512.png" />
+    </files>
+</package>

LLama/runtimes/build/LLamaSharp.Backend.Cuda13.nuspec

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="utf-8"?>
+<package >
+    <metadata>
+        <id>LLamaSharp.Backend.Cuda13</id>
+        <version>$version$</version>
+        <title>LLamaSharp.Backend.Cuda13, the backend for LLamaSharp</title>
+        <authors>llama.cpp Authors</authors>
+        <requireLicenseAcceptance>false</requireLicenseAcceptance>
+        <license type="expression">MIT</license>
+        <icon>icon512.png</icon>
+        <projectUrl>https://github.com/SciSharp/LLamaSharp</projectUrl>
+        <description>LLamaSharp.Backend.Cuda13 is a backend for LLamaSharp to use with Cuda13.</description>
+        <releaseNotes></releaseNotes>
+        <copyright>Copyright 2023 The llama.cpp Authors. All rights reserved.</copyright>
+        <tags>LLamaSharp LLama LLM GPT AI ChatBot SciSharp</tags>
+
+        <!-- Dependencies on platform-specific packages -->
+        <dependencies>
+            <dependency id="LLamaSharp.Backend.Cuda13.Windows" version="$version$" />
+            <dependency id="LLamaSharp.Backend.Cuda13.Linux" version="$version$" />
+        </dependencies>
+    </metadata>
+
+    <files>
+        <file src="LLamaSharpBackend.props" target="build/netstandard2.0/LLamaSharp.Backend.Cuda13.props" />
+        <file src="icon512.png" target="icon512.png" />
+    </files>
+</package>

README.md

@@ -5,6 +5,7 @@
 [![LLamaSharp Badge](https://img.shields.io/nuget/v/LLamaSharp?label=LLamaSharp)](https://www.nuget.org/packages/LLamaSharp)
 [![LLamaSharp Badge](https://img.shields.io/nuget/v/LLamaSharp.Backend.Cpu?label=LLamaSharp.Backend.Cpu)](https://www.nuget.org/packages/LLamaSharp.Backend.Cpu)
 [![LLamaSharp Badge](https://img.shields.io/nuget/v/LLamaSharp.Backend.Cuda12?label=LLamaSharp.Backend.Cuda12)](https://www.nuget.org/packages/LLamaSharp.Backend.Cuda12)
+[![LLamaSharp Badge](https://img.shields.io/nuget/v/LLamaSharp.Backend.Cuda13?label=LLamaSharp.Backend.Cuda13)](https://www.nuget.org/packages/LLamaSharp.Backend.Cuda13)
 [![LLamaSharp Badge](https://img.shields.io/nuget/v/LLamaSharp.Backend.Vulkan?label=LLamaSharp.Backend.Vulkan)](https://www.nuget.org/packages/LLamaSharp.Backend.Vulkan)
 
 
@@ -96,6 +97,7 @@ PM> Install-Package LLamaSharp
    - [`LLamaSharp.Backend.Cpu`](https://www.nuget.org/packages/LLamaSharp.Backend.Cpu): Pure CPU for Windows, Linux & Mac. Metal (GPU) support for Mac.
    - [`LLamaSharp.Backend.Cuda11`](https://www.nuget.org/packages/LLamaSharp.Backend.Cuda11): CUDA 11 for Windows & Linux.
    - [`LLamaSharp.Backend.Cuda12`](https://www.nuget.org/packages/LLamaSharp.Backend.Cuda12): CUDA 12 for Windows & Linux.
+   - [`LLamaSharp.Backend.Cuda13`](https://www.nuget.org/packages/LLamaSharp.Backend.Cuda13): CUDA 13 for Windows & Linux.
    - [`LLamaSharp.Backend.Vulkan`](https://www.nuget.org/packages/LLamaSharp.Backend.Vulkan): Vulkan for Windows & Linux.
 
 ### Model preparation

docs/QuickStart.md

@@ -17,6 +17,7 @@ PM> Install-Package LLamaSharp
    - [`LLamaSharp.Backend.Cpu`](https://www.nuget.org/packages/LLamaSharp.Backend.Cpu): Pure CPU for Windows & Linux & MAC. Metal (GPU) support for MAC.
    - [`LLamaSharp.Backend.Cuda11`](https://www.nuget.org/packages/LLamaSharp.Backend.Cuda11): CUDA11 for Windows & Linux.
    - [`LLamaSharp.Backend.Cuda12`](https://www.nuget.org/packages/LLamaSharp.Backend.Cuda12): CUDA 12 for Windows & Linux.
+   - [`LLamaSharp.Backend.Cuda13`](https://www.nuget.org/packages/LLamaSharp.Backend.Cuda13): CUDA 13 for Windows & Linux.
    - [`LLamaSharp.Backend.OpenCL`](https://www.nuget.org/packages/LLamaSharp.Backend.OpenCL): OpenCL for Windows & Linux.
 
 ## Model preparation