feat: add managed vLLM and GPU runtimes

Author: voidstackloop

app/src/local-server-manager.test.ts

@@ -34,8 +34,29 @@ describe("buildServerCommand", () => {
     it("uses distinct fixed ports per backend so both can run at once", () => {
         const mlx = buildServerCommand("mlx", "m", {});
         const rocm = buildServerCommand("rocm", "m", {});
+        const vllm = buildServerCommand("vllm", "m", {});
         const portOf = (args: string[]) => args[args.indexOf("--port") + 1];
         expect(portOf(mlx.args)).not.toBe(portOf(rocm.args));
+        expect(new Set([portOf(mlx.args), portOf(rocm.args), portOf(vllm.args)]).size).toBe(3);
+    });
+
+    it("builds a managed vLLM OpenAI server command", () => {
+        const { command, args } = buildServerCommand("vllm", "meta-llama/Llama-3.1-8B-Instruct", {}, "linux");
+        expect(command).toBe("vllm");
+        expect(args).toEqual(
+            expect.arrayContaining(["serve", "meta-llama/Llama-3.1-8B-Instruct", "--host", "127.0.0.1"])
+        );
+    });
+
+    it("allows a vLLM command override without requiring one", () => {
+        const { command } = buildServerCommand("vllm", "some/model", { vllmCommand: "/opt/vllm/bin/vllm" });
+        expect(command).toBe("/opt/vllm/bin/vllm");
+    });
+
+    it("launches vLLM through WSL automatically on Windows", () => {
+        const { command, args } = buildServerCommand("vllm", "some/model", {}, "win32");
+        expect(command).toBe("wsl.exe");
+        expect(args.slice(0, 4)).toEqual(["--", "vllm", "serve", "some/model"]);
     });
 });
 
@@ -47,4 +68,8 @@ describe("describeSpawnFailure", () => {
     it("points rocm failures at the llama-server binary setting", () => {
         expect(describeSpawnFailure("rocm")).toMatch(/llama-server/);
     });
+
+    it("points vLLM failures at installation rather than endpoint configuration", () => {
+        expect(describeSpawnFailure("vllm")).toMatch(/pip install vllm/);
+    });
 });

app/src/local-server-manager.ts

@@ -12,36 +12,65 @@ import { logger } from "./logger";
 // - "rocm": AMD-GPU inference via a ROCm/HIP build of llama.cpp's
 //   `llama-server` binary (the official llama.cpp releases ship one), run
 //   against the same GGUF files the built-in llama.cpp backend uses.
-export type LocalBackendId = "mlx" | "rocm";
+export type LocalBackendId = "mlx" | "rocm" | "vllm";
 
 export interface LocalBackendConfig {
     // Path to the ROCm llama-server binary. No sensible default beyond PATH
     // lookup — the user downloads a HIP build themselves.
     rocmServerPath?: string;
     // Python interpreter used to launch mlx_lm.server (needs `pip install mlx-lm`).
     mlxPythonPath?: string;
+    // Optional override; managed vLLM uses the `vllm` command from PATH.
+    vllmCommand?: string;
 }
 
 interface RunningServer {
     process: ChildProcess;
     model: string;
     baseUrl: string;
     exited: boolean;
+    activeRequests: number;
+    idleTimer: NodeJS.Timeout | null;
 }
 
 // Fixed per-backend ports so a restarted app reconnects rather than leaking
 // orphan servers across random ports.
-const PORTS: Record<LocalBackendId, number> = { mlx: 8790, rocm: 8791 };
+const PORTS: Record<LocalBackendId, number> = { mlx: 8790, rocm: 8791, vllm: 8792 };
 // First startup can include downloading/loading a multi-GB model.
 const STARTUP_TIMEOUT_MS = 180_000;
 const HEALTH_POLL_MS = 750;
+const configuredIdleMinutes = Number(process.env.OLLAMA_CUSTOM_UI_LOCAL_BACKEND_IDLE_MINUTES ?? 10);
+const IDLE_TIMEOUT_MS = Number.isFinite(configuredIdleMinutes)
+    ? Math.max(0, configuredIdleMinutes) * 60_000
+    : 10 * 60_000;
 
 const servers = new Map<LocalBackendId, RunningServer>();
+const serverStarts = new Map<LocalBackendId, { model: string; promise: Promise<string> }>();
+
+function clearIdleTimer(server: RunningServer): void {
+    if (!server.idleTimer) return;
+    clearTimeout(server.idleTimer);
+    server.idleTimer = null;
+}
+
+function scheduleIdleStop(backend: LocalBackendId, server: RunningServer): void {
+    clearIdleTimer(server);
+    if (IDLE_TIMEOUT_MS === 0 || server.activeRequests > 0 || server.exited) return;
+    server.idleTimer = setTimeout(() => {
+        server.idleTimer = null;
+        if (servers.get(backend) === server && server.activeRequests === 0) {
+            logger.info(`Stopping idle ${backend} runtime to release GPU memory`);
+            stopServer(backend);
+        }
+    }, IDLE_TIMEOUT_MS);
+    server.idleTimer.unref();
+}
 
 export function buildServerCommand(
     backend: LocalBackendId,
     model: string,
-    config: LocalBackendConfig
+    config: LocalBackendConfig,
+    platform: NodeJS.Platform = process.platform
 ): { command: string; args: string[] } {
     const port = PORTS[backend];
     if (backend === "mlx") {
@@ -50,6 +79,16 @@ export function buildServerCommand(
             args: ["-m", "mlx_lm.server", "--model", model, "--port", String(port), "--host", "127.0.0.1"],
         };
     }
+    if (backend === "vllm") {
+        const args = ["serve", model, "--port", String(port), "--host", "127.0.0.1"];
+        if (!config.vllmCommand?.trim() && platform === "win32") {
+            return { command: "wsl.exe", args: ["--", "vllm", ...args] };
+        }
+        return {
+            command: config.vllmCommand?.trim() || "vllm",
+            args,
+        };
+    }
     return {
         command: config.rocmServerPath?.trim() || "llama-server",
         args: [
@@ -63,9 +102,13 @@ export function buildServerCommand(
 }
 
 export function describeSpawnFailure(backend: LocalBackendId): string {
-    return backend === "mlx"
-        ? "Couldn't launch the MLX server — it needs Python with the mlx-lm package (pip install mlx-lm), available on Apple Silicon Macs."
-        : "Couldn't launch llama-server — set the path to a ROCm (HIP) build of llama.cpp's llama-server binary in Settings.";
+    if (backend === "mlx") {
+        return "Couldn't launch the managed MLX runtime — install mlx-lm (pip install mlx-lm) on an Apple Silicon Mac.";
+    }
+    if (backend === "vllm") {
+        return "Couldn't launch the managed vLLM runtime — install vLLM so the vllm command is available (pip install vllm).";
+    }
+    return "Couldn't launch the managed ROCm runtime — install a ROCm/HIP llama-server build and make llama-server available on PATH.";
 }
 
 // Any HTTP response means the server socket is up (a 404 from a route probe
@@ -83,7 +126,7 @@ function sleep(ms: number): Promise<void> {
     return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
-export async function ensureServer(
+async function startOrReuseServer(
     backend: LocalBackendId,
     model: string,
     config: LocalBackendConfig
@@ -94,6 +137,10 @@ export async function ensureServer(
         // Process alive but unresponsive — restart it below.
     }
     if (existing) {
+        if (existing.activeRequests > 0) {
+            throw new Error(`The ${backend} runtime is busy. Wait for the active response before changing models.`);
+        }
+        clearIdleTimer(existing);
         existing.process.kill();
         servers.delete(backend);
     }
@@ -109,7 +156,14 @@ export async function ensureServer(
         throw new Error(describeSpawnFailure(backend));
     }
 
-    const entry: RunningServer = { process: child, model, baseUrl, exited: false };
+    const entry: RunningServer = {
+        process: child,
+        model,
+        baseUrl,
+        exited: false,
+        activeRequests: 0,
+        idleTimer: null,
+    };
     servers.set(backend, entry);
 
     let spawnError: string | null = null;
@@ -129,8 +183,10 @@ export async function ensureServer(
             servers.delete(backend);
             throw new Error(
                 backend === "mlx"
-                    ? "The MLX server exited during startup — check that mlx-lm is installed and the model id is valid."
-                    : "llama-server exited during startup — check that the binary is a working ROCm build and the model file is a valid GGUF."
+                    ? "The MLX runtime exited during startup — check that mlx-lm is installed and the model id is valid."
+                    : backend === "vllm"
+                      ? "The vLLM runtime exited during startup — check that vLLM supports this model and that enough GPU memory is available."
+                      : "The ROCm runtime exited during startup — check that llama-server is a working HIP build and the model is a valid GGUF."
             );
         }
         if (await isReachable(baseUrl)) return baseUrl;
@@ -141,15 +197,64 @@ export async function ensureServer(
     throw new Error(`The ${backend} server didn't become reachable within ${STARTUP_TIMEOUT_MS / 1000}s.`);
 }
 
+export async function ensureServer(
+    backend: LocalBackendId,
+    model: string,
+    config: LocalBackendConfig
+): Promise<string> {
+    const pending = serverStarts.get(backend);
+    if (pending) {
+        if (pending.model === model) return pending.promise;
+        await pending.promise.catch(() => undefined);
+        return ensureServer(backend, model, config);
+    }
+
+    const promise = startOrReuseServer(backend, model, config);
+    serverStarts.set(backend, { model, promise });
+    try {
+        return await promise;
+    } finally {
+        if (serverStarts.get(backend)?.promise === promise) serverStarts.delete(backend);
+    }
+}
+
+export async function acquireServer(
+    backend: LocalBackendId,
+    model: string,
+    config: LocalBackendConfig
+): Promise<{ baseUrl: string; release(): void }> {
+    const current = servers.get(backend);
+    if (current) clearIdleTimer(current);
+    const baseUrl = await ensureServer(backend, model, config);
+    const server = servers.get(backend);
+    if (!server || server.exited || server.model !== model) {
+        throw new Error(`The ${backend} runtime stopped before the request could start.`);
+    }
+    server.activeRequests++;
+    let released = false;
+    return {
+        baseUrl,
+        release(): void {
+            if (released) return;
+            released = true;
+            if (servers.get(backend) !== server) return;
+            server.activeRequests = Math.max(0, server.activeRequests - 1);
+            scheduleIdleStop(backend, server);
+        },
+    };
+}
+
 export function stopServer(backend: LocalBackendId): void {
     const entry = servers.get(backend);
     if (entry) {
+        clearIdleTimer(entry);
         entry.process.kill();
         servers.delete(backend);
     }
 }
 
 export function stopAll(): void {
+    serverStarts.clear();
     for (const backend of [...servers.keys()]) stopServer(backend);
 }
 

app/src/main.ts

@@ -32,7 +32,7 @@ import { setupMenu } from "./menu";
 import { setupAutoUpdater, checkForUpdatesManually } from "./updater";
 import type { ChatMessage, ChatChunk, ChatOptions, ProviderId, ToolDefinition } from "./providers/types";
 
-const PROVIDER_SECRET_KEYS: Record<Exclude<ProviderId, "ollama" | "llamacpp" | "custom" | "mlx" | "rocm">, string> = {
+const PROVIDER_SECRET_KEYS: Record<Exclude<ProviderId, "ollama" | "llamacpp" | "custom" | "mlx" | "rocm" | "vllm">, string> = {
     openai: "openai_api_key",
     anthropic: "anthropic_api_key",
     gemini: "gemini_api_key",
@@ -186,7 +186,7 @@ async function dispatchChat(
     } else if (provider === "llamacpp") {
         const modelPath = path.join(getLlamaCppModelsDir(), model);
         await llamacpp.chat(modelPath, messages, options, onToken, signal, tools);
-    } else if (provider === "mlx" || provider === "rocm") {
+    } else if (provider === "mlx" || provider === "rocm" || provider === "vllm") {
         const settings = settingsStore.getSettings();
         // ROCm serves the same GGUF files as the llama.cpp backend, so the
         // model ref is a filename that must stay inside the models dir; MLX
@@ -200,21 +200,27 @@ async function dispatchChat(
             }
             serverModel = resolved;
         }
-        const baseUrl = await localServers.ensureServer(provider, serverModel, {
+        const lease = await localServers.acquireServer(provider, serverModel, {
             mlxPythonPath: settings.mlxPythonPath,
             rocmServerPath: settings.rocmServerPath,
+            vllmCommand: settings.vllmCommand,
         });
-        // These servers are local and unauthenticated — the "api key" is a
-        // placeholder the OpenAI-compatible client requires but they ignore.
-        await createOpenAiCompatibleChat(`${baseUrl}/v1`, provider === "mlx" ? "MLX" : "ROCm llama-server")(
-            "local",
-            model,
-            messages,
-            options,
-            onToken,
-            signal,
-            tools
-        );
+        try {
+            // Managed runtimes are local and unauthenticated; the key is a
+            // compatibility placeholder for their OpenAI-shaped APIs.
+            const providerLabel = provider === "mlx" ? "MLX" : provider === "vllm" ? "vLLM" : "ROCm llama-server";
+            await createOpenAiCompatibleChat(`${lease.baseUrl}/v1`, providerLabel)(
+                "local",
+                model,
+                messages,
+                options,
+                onToken,
+                signal,
+                tools
+            );
+        } finally {
+            lease.release();
+        }
     } else if (provider === "custom") {
         // model is "<customProviderId>::<actual model id>" — see
         // frontend/src/lib/providers.ts's formatCustomModelRef.

app/src/providers/types.ts

@@ -50,7 +50,7 @@ export interface ChatChunk {
     toolCalls?: ToolCall[];
 }
 
-export type ProviderId = "ollama" | "openai" | "anthropic" | "llamacpp" | "gemini" | "custom" | "mlx" | "rocm";
+export type ProviderId = "ollama" | "openai" | "anthropic" | "llamacpp" | "gemini" | "custom" | "mlx" | "rocm" | "vllm";
 
 export interface ChatOptions {
     temperature?: number;

app/src/settings-store.ts

@@ -95,6 +95,10 @@ export interface AppSettings {
     // Path to a ROCm/HIP build of llama.cpp's llama-server binary — enables
     // the "rocm" provider against the same GGUF dir as the llama.cpp backend.
     rocmServerPath?: string;
+    // Hugging Face model ids or local model paths served by the app-managed
+    // vLLM runtime. The `vllm` executable is discovered from PATH by default.
+    vllmModels?: string[];
+    vllmCommand?: string;
 }
 
 const DEFAULTS: AppSettings = {

frontend/src/lib/providers.ts

@@ -14,10 +14,11 @@ export const PROVIDER_LABELS: Record<ProviderId, string> = {
     custom: "Custom",
     mlx: "MLX (Apple Silicon)",
     rocm: "ROCm (AMD)",
+    vllm: "vLLM (managed)",
 };
 
 // Providers that run models on this machine — no API key, no per-token cost.
-export const LOCAL_PROVIDERS: ProviderId[] = ["ollama", "llamacpp", "mlx", "rocm"];
+export const LOCAL_PROVIDERS: ProviderId[] = ["ollama", "llamacpp", "mlx", "rocm", "vllm"];
 
 // Curated as of this app's last update — model lineups change often, so the
 // model picker also lets you type a custom model ID directly.

frontend/src/pages/Chat.tsx

@@ -1320,7 +1320,7 @@ export default function Chat() {
                                 ))}
                             </SelectGroup>
                         )}
-                        {!!settings?.rocmServerPath && llamaCppModels.length > 0 && (
+                        {llamaCppModels.length > 0 && (
                             <SelectGroup>
                                 <SelectLabel>ROCm (AMD)</SelectLabel>
                                 {llamaCppModels.map((m) => (
@@ -1330,6 +1330,16 @@ export default function Chat() {
                                 ))}
                             </SelectGroup>
                         )}
+                        {(settings?.vllmModels ?? []).length > 0 && (
+                            <SelectGroup>
+                                <SelectLabel>vLLM (managed)</SelectLabel>
+                                {settings!.vllmModels!.map((id) => (
+                                    <SelectItem key={id} value={formatModelRef("vllm", id)}>
+                                        {id}
+                                    </SelectItem>
+                                ))}
+                            </SelectGroup>
+                        )}
                         <SelectGroup>
                             <SelectLabel>ChatGPT</SelectLabel>
                             {OPENAI_MODELS.map((m) => (