feat: complete hardware-aware GGUF downloads

Author: voidstackloop

app/src/download-queue.test.ts

@@ -1,6 +1,6 @@
 import { describe, it, expect, vi } from "vitest";
 import { createJob, updateJob, listJobs, type DownloadShard, type DownloadJobState } from "./download-jobs-store";
-import { init, broadcast, resumeInterruptedJobs } from "./download-queue";
+import { init, broadcast, configure, resumeInterruptedJobs } from "./download-queue";
 
 function shard(overrides: Partial<DownloadShard> = {}): DownloadShard {
     return {
@@ -28,6 +28,10 @@ function makeJob(state: DownloadJobState) {
 }
 
 describe("download-queue: broadcast plumbing", () => {
+    it("normalizes controls without crashing when a plain dev build has no native addon", () => {
+        expect(configure({ concurrency: 99, bandwidthMbps: -5 })).toEqual({ concurrency: 8, bandwidthMbps: 0 });
+    });
+
     it("does nothing when no window has been registered via init()", () => {
         expect(() => broadcast()).not.toThrow();
     });

app/src/download-queue.ts

@@ -5,6 +5,7 @@ import * as downloadWorker from "./download-worker";
 import { createJob, deleteJob, flushJobs, getJob, listJobs, updateJob, type DownloadJob } from "./download-jobs-store";
 import { getDownloadManager } from "./native-downloader";
 import { detectModelFormat, getSpecs, resolveAutomaticRuntime } from "./system-specs";
+import { logger } from "./logger";
 
 export interface DownloadControls { concurrency: number; bandwidthMbps: number }
 export interface RecoveryStatus { recoveredJobs: number; recoveredAt: string | null }
@@ -41,9 +42,17 @@ export function configure(controls: DownloadControls): DownloadControls {
     const concurrency = Number.isFinite(rawConcurrency) ? Math.max(1, Math.min(8, Math.floor(rawConcurrency))) : 2;
     const rawBandwidth = Number(controls?.bandwidthMbps);
     const bandwidthMbps = Number.isFinite(rawBandwidth) ? Math.max(0, Math.min(100_000, rawBandwidth)) : 0;
-    const manager = getDownloadManager();
-    manager.setGlobalConcurrency(concurrency);
-    manager.setBandwidthLimit(bandwidthMbps > 0 ? bandwidthMbps * 1024 * 1024 / 8 : undefined);
+    try {
+        const manager = getDownloadManager();
+        manager.setGlobalConcurrency(concurrency);
+        manager.setBandwidthLimit(bandwidthMbps > 0 ? bandwidthMbps * 1024 * 1024 / 8 : undefined);
+    } catch (error) {
+        // Plain TypeScript/dev/E2E builds intentionally do not compile the
+        // Rust addon. Keep the rest of app startup healthy; attempting an
+        // actual download will still surface the missing-addon error. The
+        // packaged build always runs build:native and includes app/native.
+        logger.warn(`Native download controls unavailable: ${(error as Error).message}`);
+    }
     return { concurrency, bandwidthMbps };
 }
 

app/src/ipc/system-handlers.ts

@@ -21,6 +21,21 @@ export function registerSystemIpc(): void {
         const settings = settingsStore.getSettings();
         return systemSpecs.recommendModelsWithML(specs, { contextLength: settings.contextLength, quantization: "Q4_K_M", runtime: settings.preferredRuntime ?? "automatic", goal: settings.recommendationGoal ?? "balanced" }, getBenchmarkObservations());
     });
+    ipcMain.handle("system:assessGgufFiles", async (_event, rawInputs: unknown) => {
+        if (!Array.isArray(rawInputs) || rawInputs.length > 200) throw new Error("Expected at most 200 GGUF files.");
+        const inputs = rawInputs.map((value) => {
+            if (!value || typeof value !== "object") throw new Error("Invalid GGUF assessment input.");
+            const input = value as Record<string, unknown>;
+            const modelId = requireString(input.modelId, "Hugging Face model id");
+            const filename = requireString(input.filename, "GGUF filename");
+            const sizeBytes = input.sizeBytes === null ? null : Number(input.sizeBytes);
+            if (sizeBytes !== null && (!Number.isFinite(sizeBytes) || sizeBytes < 0)) throw new Error("Invalid GGUF file size.");
+            return { modelId, filename, sizeBytes };
+        });
+        const specs = await systemSpecs.getSpecs();
+        const settings = settingsStore.getSettings();
+        return systemSpecs.assessGgufFiles(specs, inputs, settings.contextLength);
+    });
     ipcMain.handle("system:getActivity", async () => {
         const ollamaRunning = await ollama.isRunning();
         const ollamaLoadedModels = ollamaRunning

app/src/llamacpp-manager.test.ts

@@ -2,7 +2,19 @@ import { describe, it, expect, beforeEach } from "vitest";
 import * as fs from "node:fs";
 import * as os from "node:os";
 import * as path from "node:path";
-import { deleteModel, groupShardedModels, listModels, normalizeLlamaCppRuntimeConfig } from "./llamacpp-manager";
+import { deleteModel, groupShardedModels, listModels, normalizeLlamaCppRuntimeConfig, resolveGpuLayers } from "./llamacpp-manager";
+
+describe("resolveGpuLayers", () => {
+    it("reserves VRAM for the user's requested context in automatic mode", () => {
+        expect(resolveGpuLayers("auto", undefined, 32_768)).toEqual({ fitContext: { contextSize: 32_768 } });
+    });
+
+    it("keeps CPU, all-layer, and manual modes distinct", () => {
+        expect(resolveGpuLayers("cpu", undefined, 8_192)).toBe(0);
+        expect(resolveGpuLayers("max", undefined, 8_192)).toBe("max");
+        expect(resolveGpuLayers("manual", 24, 8_192)).toBe(24);
+    });
+});
 
 describe("groupShardedModels", () => {
     it("leaves a normal single-file model untouched", () => {

app/src/llamacpp-manager.ts

@@ -146,13 +146,21 @@ export function normalizeLlamaCppRuntimeConfig(input: LlamaCppRuntimeConfig = {}
     return { maxThreads: integer(input.maxThreads, 512), vramReserveBytes: bytes(input.vramReserveBytes), ramReserveBytes: bytes(input.ramReserveBytes), numa };
 }
 
-function resolveGpuLayers(mode: GpuLayerMode | undefined, manualLayers: number | undefined): LlamaModelOptions["gpuLayers"] {
+export function resolveGpuLayers(mode: GpuLayerMode | undefined, manualLayers: number | undefined, contextLength?: number): LlamaModelOptions["gpuLayers"] {
     const resolvedMode = mode ?? (manualLayers === undefined ? "auto" : manualLayers === 0 ? "cpu" : "manual");
     if (resolvedMode === "cpu") return 0;
-    // node-llama-cpp's "auto" is already maximum memory-safe placement and
-    // preserves the Llama instance's VRAM padding. It is therefore also the
-    // correct implementation of the user-facing "Maximum safe offload".
-    if (resolvedMode === "auto" || resolvedMode === "max") return "auto";
+    // Explicitly include the requested context in node-llama-cpp's model
+    // placement calculation. Plain "auto" only reserves space for an
+    // automatically-sized context, which can over-offload weights and then
+    // fail when the user requests a larger context.
+    if (resolvedMode === "auto") {
+        return contextLength && Number.isFinite(contextLength)
+            ? { fitContext: { contextSize: Math.max(512, Math.floor(contextLength)) } }
+            : "auto";
+    }
+    // Advanced escape hatch: request every layer and let node-llama-cpp fail
+    // clearly when current VRAM cannot hold it.
+    if (resolvedMode === "max") return "max";
     if (!Number.isInteger(manualLayers) || manualLayers! < 0) throw new Error("Manual GPU layer mode requires a non-negative integer layer count.");
     return manualLayers;
 }
@@ -614,7 +622,7 @@ export async function chat(
     }
     if (lastUserIndex === -1) throw new Error("No user message to respond to.");
 
-    const resolvedGpuLayers = resolveGpuLayers(options?.gpuLayerMode, options?.gpuLayers);
+    const resolvedGpuLayers = resolveGpuLayers(options?.gpuLayerMode, options?.gpuLayers, options?.contextLength);
     const cacheKey = modelCacheKey(modelPath, resolvedGpuLayers);
     const model = await loadModel(modelPath, resolvedGpuLayers);
     // A transition can be announced while this call is awaiting a queued or

app/src/preload.ts

@@ -145,7 +145,17 @@ export const api = {
             ipcRenderer.on(channel, listener);
             const promise = ipcRenderer
                 .invoke("chat:send", { requestId, provider, model, messages, options, agentMode })
-                .finally(() => ipcRenderer.removeListener(channel, listener));
+                // The invoke reply and a final webContents.send() chunk use
+                // separate IPC routes. The reply can arrive first even when
+                // main sent the chunk first; removing this listener
+                // immediately dropped single-chunk tool calls and final usage
+                // metadata. Keep it through one short renderer turn, and
+                // delay promise resolution so Chat flushes those chunks before
+                // it inspects the completed assistant message.
+                .finally(() => new Promise<void>((resolve) => setTimeout(() => {
+                    ipcRenderer.removeListener(channel, listener);
+                    resolve();
+                }, 25)));
             return { requestId, promise };
         },
 
@@ -155,6 +165,7 @@ export const api = {
     system: {
         getSpecs: () => ipcRenderer.invoke("system:getSpecs"),
         getRecommendations: () => ipcRenderer.invoke("system:getRecommendations"),
+        assessGgufFiles: (files: import("./system-specs").GgufAssessmentInput[]): Promise<import("./system-specs").GgufAssessment[]> => ipcRenderer.invoke("system:assessGgufFiles", files),
         getActivity: () => ipcRenderer.invoke("system:getActivity"),
     },
 

app/src/providers/types.ts

@@ -63,13 +63,12 @@ export interface ChatOptions {
     maxTokens?: number;
     frequencyPenalty?: number;
     presencePenalty?: number;
-    // Ollama-only: how much conversation history the model actually processes
-    // (num_ctx). Cloud providers fix this per-model and don't expose it via API.
+    // Local GGUF context window: Ollama maps this to num_ctx and the built-in
+    // llama.cpp runtime maps it to createContext({ contextSize }). Cloud
+    // providers fix this per model and don't expose it via API.
     contextLength?: number;
-    // Ollama-only: how many model layers to offload to GPU (num_gpu).
-    // undefined = let Ollama auto-decide, 0 = force CPU-only, a positive
-    // number = offload that many layers (useful for tuning multi-GPU setups
-    // or freeing VRAM for something else running alongside Ollama).
+    // Local GGUF GPU placement. Ollama maps a manual count to num_gpu; the
+    // built-in llama.cpp runtime additionally supports gpuLayerMode below.
     gpuLayers?: number;
     // Built-in node-llama-cpp controls. Other providers ignore these fields.
     gpuLayerMode?: GpuLayerMode;

app/src/system-specs.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect } from "vitest";
-import { classifyGpuVendor, computeGpuTopology, detectModelFormat, parseMigInstancesFromNvidiaSmiL, recommendModels, recommendModelsWithML, resolveAutomaticRuntime, type GpuInfo, type GpuTopology, type SystemSpecs } from "./system-specs";
+import { assessGgufFiles, classifyGpuVendor, computeGpuTopology, detectGgufQuantization, detectModelFormat, parseMigInstancesFromNvidiaSmiL, recommendModels, recommendModelsWithML, resolveAutomaticRuntime, type GpuInfo, type GpuTopology, type SystemSpecs } from "./system-specs";
 
 function baseTopology(overrides: Partial<GpuTopology> = {}): GpuTopology {
     return {
@@ -253,6 +253,41 @@ describe("detectModelFormat", () => {
     });
 });
 
+describe("assessGgufFiles", () => {
+    it("extracts common GGUF quantizations without confusing model names", () => {
+        expect(detectGgufQuantization("Kimi-Q3_K_L.gguf")).toEqual({ label: "Q3_K_L", bits: 3.69 });
+        expect(detectGgufQuantization("model-IQ2_XXS.gguf").label).toBe("IQ2_XXS");
+        expect(detectGgufQuantization("model-UD-Q4_K_XL.gguf").label).toBe("UD-Q4_K_XL");
+        expect(detectGgufQuantization("model.gguf").label).toBe("GGUF");
+    });
+
+    it("reports a full-GPU fit and a non-zero speed estimate", () => {
+        const specs = baseSpecs({
+            freeRAMGB: 32,
+            gpu: { name: "RTX 4090", vramGB: 24, vendor: "nvidia" },
+            gpus: [{ name: "RTX 4090", vramGB: 24, vendor: "nvidia" }],
+            totalVramGB: 24,
+            largestGpuVramGB: 24,
+        });
+        const [result] = assessGgufFiles(specs, [{ modelId: "org/model", filename: "model-Q4_K_M.gguf", sizeBytes: 8_000_000_000 }]);
+        expect(result.outcome).toBe("Runs fully on GPU");
+        expect(result.fits).toBe(true);
+        expect(result.totalRequiredGB).toBeGreaterThan(8);
+        expect(result.estimatedTokensPerSecond).toBeGreaterThan(0);
+    });
+
+    it("marks an oversized file unsafe and handles missing sizes honestly", () => {
+        const [oversized, unknown] = assessGgufFiles(baseSpecs({ totalRAMGB: 16, freeRAMGB: 8 }), [
+            { modelId: "org/model", filename: "huge-Q8_0.gguf", sizeBytes: 80_000_000_000 },
+            { modelId: "org/model", filename: "unknown.gguf", sizeBytes: null },
+        ]);
+        expect(oversized.outcome).toBe("Likely out of memory");
+        expect(oversized.estimatedTokensPerSecond).toBe(0);
+        expect(unknown.canAssess).toBe(false);
+        expect(unknown.fits).toBeNull();
+    });
+});
+
 describe("resolveAutomaticRuntime", () => {
     const linuxNvidia = { platform: "linux" as const, arch: "x64", gpus: [{ name: "RTX 4090", vramGB: 24, vendor: "nvidia" as const }] };
     const linuxAmd = { platform: "linux" as const, arch: "x64", gpus: [{ name: "Radeon RX 7900", vramGB: 24, vendor: "amd" as const }] };