Add hardware-aware model recommendations and an Automatic model runtime

recommendModels() now estimates per-model memory/KV-cache/offload fit and recommends a concrete runtime per outcome (vLLM for tensor-parallel fits, llama.cpp for partial/CPU-only). New resolveAutomaticRuntime()/detectModelFormat() in system-specs.ts implement the Automatic runtime policy: GGUF -> llama.cpp, safetensors + NVIDIA/AMD -> vLLM, MLX format + Apple Silicon -> MLX, Ollama-library models -> Ollama (MLX on Apple Silicon), anything else -> Transformers.

Author: voidstackloop

app/src/huggingface.test.ts

@@ -1,117 +1,76 @@
-import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
-import * as fs from "node:fs";
-import * as os from "node:os";
-import * as path from "node:path";
+import { describe, it, expect, vi, beforeEach } from "vitest";
 import { downloadGgufFile } from "./huggingface";
-
-function fakeResponse(opts: { status: number; headers?: Record<string, string>; chunks?: string[] }) {
-    const chunks = (opts.chunks ?? []).map((c) => new TextEncoder().encode(c));
-    let i = 0;
-    return {
-        ok: opts.status >= 200 && opts.status < 300,
-        status: opts.status,
-        headers: { get: (name: string) => opts.headers?.[name.toLowerCase()] ?? null },
-        body: {
-            getReader: () => ({
-                read: async () => (i < chunks.length ? { done: false, value: chunks[i++] } : { done: true, value: undefined }),
-            }),
-        },
-    } as unknown as Response;
-}
-
-describe("downloadGgufFile", () => {
-    let dir: string;
-    let destPath: string;
-
+import { downloadGgufFileNative } from "./native-downloader";
+
+// The actual HTTP/resume/verify logic now lives in the Rust native addon
+// (lib/), covered by its own test suite there (mock-HTTP-server-based, see
+// lib/src/download/tests.rs) — a JS-side `vi.stubGlobal("fetch", ...)` can't
+// reach code that no longer runs in JS. What's left to verify from the TS
+// side is just this file's adapter: does it call the native function with
+// the right arguments, and does it translate the native progress/error
+// shapes into what callers (main.ts's hf:downloadFile handler) expect.
+vi.mock("./native-downloader", () => ({
+    downloadGgufFileNative: vi.fn(),
+}));
+
+const nativeMock = vi.mocked(downloadGgufFileNative);
+
+describe("downloadGgufFile (native adapter)", () => {
     beforeEach(() => {
-        dir = fs.mkdtempSync(path.join(os.tmpdir(), "hf-download-test-"));
-        destPath = path.join(dir, "model.gguf");
-    });
-
-    afterEach(() => {
-        vi.unstubAllGlobals();
+        nativeMock.mockReset();
     });
 
-    it("downloads fresh, reports progress, and renames the .part file to the final name on success", async () => {
-        const progress: { receivedBytes: number; totalBytes: number | null }[] = [];
-        vi.stubGlobal(
-            "fetch",
-            vi.fn(async () => fakeResponse({ status: 200, headers: { "content-length": "10" }, chunks: ["hello", "world"] }))
-        );
+    it("calls the native addon with modelId/filename/destPath and no token when none is given", async () => {
+        nativeMock.mockResolvedValue(undefined);
 
-        await downloadGgufFile("org/model", "model.gguf", destPath, (p) => progress.push(p));
+        await downloadGgufFile("org/model", "model.gguf", "/tmp/model.gguf", () => {});
 
-        expect(fs.existsSync(destPath)).toBe(true);
-        expect(fs.existsSync(destPath + ".part")).toBe(false);
-        expect(fs.readFileSync(destPath, "utf8")).toBe("helloworld");
-        expect(progress.at(-1)).toEqual({ receivedBytes: 10, totalBytes: 10 });
+        expect(nativeMock).toHaveBeenCalledWith(
+            "org/model",
+            "model.gguf",
+            "/tmp/model.gguf",
+            undefined,
+            expect.any(Function)
+        );
     });
 
-    it("resumes from an existing .part file using a Range request instead of starting over", async () => {
-        fs.writeFileSync(destPath + ".part", "hello");
-        const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => {
-            expect((init?.headers as Record<string, string>).Range).toBe("bytes=5-");
-            return fakeResponse({ status: 206, headers: { "content-range": "bytes 5-9/10" }, chunks: ["world"] });
-        });
-        vi.stubGlobal("fetch", fetchMock);
+    it("passes a null token through as undefined, matching the native addon's Option<String> signature", async () => {
+        nativeMock.mockResolvedValue(undefined);
 
-        await downloadGgufFile("org/model", "model.gguf", destPath, () => {});
+        await downloadGgufFile("org/model", "model.gguf", "/tmp/model.gguf", () => {}, null);
 
-        expect(fs.readFileSync(destPath, "utf8")).toBe("helloworld");
+        expect(nativeMock.mock.calls[0][3]).toBeUndefined();
     });
 
-    it("discards a stale partial and starts over when the server ignores the Range request", async () => {
-        fs.writeFileSync(destPath + ".part", "OLD-STALE-DATA");
-        vi.stubGlobal(
-            "fetch",
-            vi.fn(async () => fakeResponse({ status: 200, headers: { "content-length": "5" }, chunks: ["fresh"] }))
-        );
+    it("passes a real token through unchanged", async () => {
+        nativeMock.mockResolvedValue(undefined);
 
-        await downloadGgufFile("org/model", "model.gguf", destPath, () => {});
+        await downloadGgufFile("org/model", "model.gguf", "/tmp/model.gguf", () => {}, "secret-token");
 
-        expect(fs.readFileSync(destPath, "utf8")).toBe("fresh");
+        expect(nativeMock.mock.calls[0][3]).toBe("secret-token");
     });
 
-    it("keeps the .part file (doesn't delete it) when the connection drops mid-stream, so a retry can resume", async () => {
-        vi.stubGlobal("fetch", vi.fn(async () => ({
-            ok: true,
-            status: 200,
-            headers: { get: () => "20" },
-            body: {
-                getReader: () => ({
-                    read: async () => {
-                        throw new Error("connection reset");
-                    },
-                }),
-            },
-        } as unknown as Response)));
-
-        await expect(downloadGgufFile("org/model", "model.gguf", destPath, () => {})).rejects.toThrow("connection reset");
-        expect(fs.existsSync(destPath + ".part")).toBe(true);
-        expect(fs.existsSync(destPath)).toBe(false);
-    });
+    it("adapts native progress events, mapping a missing totalBytes to null", async () => {
+        const progress: { receivedBytes: number; totalBytes: number | null }[] = [];
+        nativeMock.mockImplementation(async (_modelId, _filename, _dest, _token, onProgress) => {
+            onProgress(null, { receivedBytes: 5, totalBytes: undefined });
+            onProgress(null, { receivedBytes: 10, totalBytes: 10 });
+        });
 
-    it("keeps the .part file and throws when the stream ends short of the expected size", async () => {
-        vi.stubGlobal(
-            "fetch",
-            vi.fn(async () => fakeResponse({ status: 200, headers: { "content-length": "100" }, chunks: ["short"] }))
-        );
+        await downloadGgufFile("org/model", "model.gguf", "/tmp/model.gguf", (p) => progress.push(p));
 
-        await expect(downloadGgufFile("org/model", "model.gguf", destPath, () => {})).rejects.toThrow(/incomplete/);
-        expect(fs.existsSync(destPath + ".part")).toBe(true);
+        expect(progress).toEqual([
+            { receivedBytes: 5, totalBytes: null },
+            { receivedBytes: 10, totalBytes: 10 },
+        ]);
     });
 
-    it("discards a partial and retries once when the server responds 416 (range no longer satisfiable)", async () => {
-        fs.writeFileSync(destPath + ".part", "stale");
-        const fetchMock = vi
-            .fn()
-            .mockResolvedValueOnce(fakeResponse({ status: 416 }))
-            .mockResolvedValueOnce(fakeResponse({ status: 200, headers: { "content-length": "5" }, chunks: ["fresh"] }));
-        vi.stubGlobal("fetch", fetchMock);
-
-        await downloadGgufFile("org/model", "model.gguf", destPath, () => {});
+    it("propagates the native call's rejection unchanged, so callers' error logging keeps working", async () => {
+        const nativeError = new Error(
+            'Download of "model.gguf" was incomplete (got 5 of 10 bytes) — try downloading it again to resume.'
+        );
+        nativeMock.mockRejectedValue(nativeError);
 
-        expect(fs.readFileSync(destPath, "utf8")).toBe("fresh");
-        expect(fetchMock).toHaveBeenCalledTimes(2);
+        await expect(downloadGgufFile("org/model", "model.gguf", "/tmp/model.gguf", () => {})).rejects.toBe(nativeError);
     });
 });

app/src/huggingface.ts

@@ -47,98 +47,22 @@ export interface DownloadProgress {
     totalBytes: number | null;
 }
 
-// Suffix for in-progress downloads. Writing directly to the final .gguf name
-// would let a truncated file — from a network drop, a crash, or the app
-// being force-quit mid-download, none of which run our error-path cleanup —
-// sit there indistinguishable from a real model, so listModels() would offer
-// it and loading it would fail with a confusing "corrupt GGUF" error instead
-// of the app just not knowing about it.
-export const PARTIAL_DOWNLOAD_SUFFIX = ".part";
-
-function parseContentRangeTotal(headerValue: string | null): number | null {
-    // "bytes 12345-67890/98765" — the part after the slash is the full size.
-    const match = headerValue?.match(/\/(\d+)$/);
-    return match ? Number(match[1]) : null;
-}
-
+// The actual HTTP + resume/verify logic lives in the native Rust addon
+// (lib/) for speed — a plain single-stream port wouldn't outperform this,
+// but splitting large files across several parallel Range-request
+// connections meaningfully does. This wrapper keeps the exact signature the
+// rest of the app (main.ts's hf:downloadFile handler) already calls, so
+// nothing downstream needed to change.
 export async function downloadGgufFile(
     modelId: string,
     filename: string,
     destPath: string,
     onProgress: (progress: DownloadProgress) => void,
     token?: string | null
 ): Promise<void> {
-    const fs = await import("node:fs");
-    const url = `https://huggingface.co/${modelId}/resolve/main/${encodeURIComponent(filename)}`;
-    const partPath = destPath + PARTIAL_DOWNLOAD_SUFFIX;
-
-    // A .part file left over from a dropped connection or a force-quit is
-    // real progress, not garbage — resume it with a Range request instead of
-    // re-downloading from byte zero every time.
-    let existingBytes = fs.existsSync(partPath) ? fs.statSync(partPath).size : 0;
-    const headers: Record<string, string> = {};
-    if (token) headers.Authorization = `Bearer ${token}`;
-    if (existingBytes > 0) headers.Range = `bytes=${existingBytes}-`;
-
-    let res: Response;
-    try {
-        res = await fetch(url, { headers });
-    } catch (err) {
-        throw new Error(`Couldn't reach Hugging Face: ${(err as Error).message}`);
-    }
-
-    if (existingBytes > 0 && res.status === 416) {
-        // Our partial is already >= what the server has now (stale, or the
-        // remote file changed) — it can't be resumed, so start clean once.
-        fs.rmSync(partPath, { force: true });
-        return downloadGgufFile(modelId, filename, destPath, onProgress, token);
-    }
-    let resuming = existingBytes > 0 && res.status === 206;
-    if (existingBytes > 0 && !resuming) {
-        // Server ignored the Range request and is sending the whole file
-        // from the start — appending that to the stale partial would
-        // corrupt it, so discard the partial and treat this as fresh.
-        existingBytes = 0;
-    }
-
-    if (!res.ok || !res.body) throw new Error(`Failed to download "${filename}" (HTTP ${res.status}).`);
-
-    const contentLength = Number(res.headers.get("content-length")) || null;
-    const totalBytes = resuming
-        ? (parseContentRangeTotal(res.headers.get("content-range")) ?? (contentLength !== null ? existingBytes + contentLength : null))
-        : contentLength;
-
-    let receivedBytes = existingBytes;
-    const writeStream = fs.createWriteStream(partPath, { flags: resuming ? "a" : "w" });
-    const reader = res.body.getReader();
-    onProgress({ receivedBytes, totalBytes });
-
-    try {
-        while (true) {
-            const { done, value } = await reader.read();
-            if (done) break;
-            receivedBytes += value.byteLength;
-            onProgress({ receivedBytes, totalBytes });
-            await new Promise<void>((resolve, reject) => {
-                writeStream.write(value, (err) => (err ? reject(err) : resolve()));
-            });
-        }
-    } catch (err) {
-        // Deliberately not deleting partPath — a dropped connection here
-        // leaves real, resumable progress on disk for the next attempt.
-        // Waited out rather than fire-and-forget so the partial bytes are
-        // actually flushed to disk before this function returns.
-        await new Promise<void>((resolve) => writeStream.end(() => resolve()));
-        throw err;
-    }
-    await new Promise<void>((resolve, reject) => {
-        writeStream.once("error", reject);
-        writeStream.end(() => resolve());
+    const { downloadGgufFileNative } = await import("./native-downloader");
+    await downloadGgufFileNative(modelId, filename, destPath, token ?? undefined, (err, progress) => {
+        if (err) return; // fatal errors surface via the returned promise's rejection instead
+        onProgress({ receivedBytes: progress.receivedBytes, totalBytes: progress.totalBytes ?? null });
     });
-    if (totalBytes !== null && receivedBytes !== totalBytes) {
-        throw new Error(
-            `Download of "${filename}" was incomplete (got ${receivedBytes} of ${totalBytes} bytes) — try downloading it again to resume.`
-        );
-    }
-    fs.renameSync(partPath, destPath);
 }