|
| 1 | +import * as path from "node:path"; |
| 2 | +import * as ollama from "./ollama-manager"; |
| 3 | +import * as settingsStore from "./settings-store"; |
| 4 | +import * as secretsStore from "./secrets-store"; |
| 5 | +import * as llamacpp from "./llamacpp-manager"; |
| 6 | +import * as localServers from "./local-server-manager"; |
| 7 | +import * as powerMonitor from "./power-monitor"; |
| 8 | +import * as openaiProvider from "./providers/openai"; |
| 9 | +import * as anthropicProvider from "./providers/anthropic"; |
| 10 | +import * as geminiProvider from "./providers/gemini"; |
| 11 | +import { createOpenAiCompatibleChat } from "./providers/openai-compatible"; |
| 12 | +import type { ChatMessage, ChatChunk, ChatOptions, ProviderId, ToolDefinition } from "./providers/types"; |
| 13 | +import { PROVIDER_SECRET_KEYS, customProviderSecretKey, getLlamaCppModelsDir, getEnergyMonitorSettings } from "./app-state"; |
| 14 | + |
| 15 | +// Shared by chat:send (renderer-driven, streams tokens back over IPC) and |
| 16 | +// the scheduled-task runner (background, wants the full text once done) — |
| 17 | +// same provider dispatch and error handling either way. |
| 18 | +export async function dispatchChat( |
| 19 | + provider: ProviderId, |
| 20 | + model: string, |
| 21 | + messages: ChatMessage[], |
| 22 | + options: ChatOptions | undefined, |
| 23 | + onToken: (chunk: ChatChunk) => void, |
| 24 | + signal?: AbortSignal, |
| 25 | + tools?: ToolDefinition[] |
| 26 | +): Promise<void> { |
| 27 | + const currentSettings = settingsStore.getSettings(); |
| 28 | + const customLocal = provider === "custom" |
| 29 | + && currentSettings.customProviders?.find((item) => model.startsWith(`${item.id}::`))?.localGpuBackend; |
| 30 | + const localProvider = ["ollama", "llamacpp", "mlx", "rocm", "vllm"].includes(provider) || !!customLocal; |
| 31 | + const energySettings = getEnergyMonitorSettings(); |
| 32 | + energySettings.enabled = energySettings.enabled && localProvider; |
| 33 | + const backend = provider === "llamacpp" |
| 34 | + ? currentSettings.llamaCppGpuBackend ?? "auto" |
| 35 | + : provider === "rocm" ? "rocm" : provider; |
| 36 | + const initialPromptTokens = Math.max(1, Math.ceil(messages.reduce((sum, message) => sum + message.content.length, 0) / 4)); |
| 37 | + const activity = powerMonitor.beginRequest(provider, model, backend, energySettings, initialPromptTokens); |
| 38 | + const downstreamToken = onToken; |
| 39 | + onToken = (chunk) => { |
| 40 | + activity.onChunk(chunk); |
| 41 | + downstreamToken(chunk); |
| 42 | + }; |
| 43 | + try { |
| 44 | + if (provider === "ollama") { |
| 45 | + await ollama.chat(model, messages, options, onToken, signal, tools); |
| 46 | + } else if (provider === "llamacpp") { |
| 47 | + // Same containment rule as the rocm branch below: the model ref is a |
| 48 | + // renderer-supplied relative path (may include subfolders), and |
| 49 | + // path.join would happily walk ".." segments out of the models dir. |
| 50 | + const root = path.resolve(getLlamaCppModelsDir()); |
| 51 | + const modelPath = path.resolve(root, model); |
| 52 | + if (modelPath === root || !modelPath.startsWith(root + path.sep)) { |
| 53 | + throw new Error(`Model file "${model}" is outside the models directory.`); |
| 54 | + } |
| 55 | + await llamacpp.chat(modelPath, messages, options, onToken, signal, tools); |
| 56 | + } else if (provider === "mlx" || provider === "rocm" || provider === "vllm") { |
| 57 | + const settings = settingsStore.getSettings(); |
| 58 | + // ROCm serves the same GGUF files as the llama.cpp backend, so the |
| 59 | + // model ref is a filename that must stay inside the models dir; MLX |
| 60 | + // models are HF repo ids the server resolves itself. |
| 61 | + let serverModel = model; |
| 62 | + if (provider === "rocm") { |
| 63 | + const root = path.resolve(getLlamaCppModelsDir()); |
| 64 | + const resolved = path.resolve(root, model); |
| 65 | + // Was `resolved !== root && !startsWith(...)` (AND) — that only |
| 66 | + // threw when BOTH conditions held, so a ref resolving to exactly |
| 67 | + // the models dir itself (e.g. "rocm:.") satisfied neither and |
| 68 | + // slipped through, handing the whole directory to llama-server |
| 69 | + // -m as if it were a single model file. |
| 70 | + if (resolved === root || !resolved.startsWith(root + path.sep)) { |
| 71 | + throw new Error(`Model file "${model}" is outside the models directory.`); |
| 72 | + } |
| 73 | + serverModel = resolved; |
| 74 | + } |
| 75 | + const lease = await localServers.acquireServer(provider, serverModel, { |
| 76 | + mlxPythonPath: settings.mlxPythonPath, |
| 77 | + rocmServerPath: settings.rocmServerPath, |
| 78 | + vllmCommand: settings.vllmCommand, |
| 79 | + }); |
| 80 | + try { |
| 81 | + // Managed runtimes are local and unauthenticated; the key is a |
| 82 | + // compatibility placeholder for their OpenAI-shaped APIs. |
| 83 | + const providerLabel = provider === "mlx" ? "MLX" : provider === "vllm" ? "vLLM" : "ROCm llama-server"; |
| 84 | + await createOpenAiCompatibleChat(`${lease.baseUrl}/v1`, providerLabel)( |
| 85 | + "local", |
| 86 | + model, |
| 87 | + messages, |
| 88 | + options, |
| 89 | + onToken, |
| 90 | + signal, |
| 91 | + tools |
| 92 | + ); |
| 93 | + } finally { |
| 94 | + lease.release(); |
| 95 | + } |
| 96 | + } else if (provider === "custom") { |
| 97 | + // model is "<customProviderId>::<actual model id>" — see |
| 98 | + // frontend/src/lib/providers.ts's formatCustomModelRef. |
| 99 | + const sep = model.indexOf("::"); |
| 100 | + if (sep === -1) throw new Error(`Malformed custom model reference: ${model}`); |
| 101 | + const customProviderId = model.slice(0, sep); |
| 102 | + const actualModel = model.slice(sep + 2); |
| 103 | + const config = settingsStore.getSettings().customProviders?.find((p) => p.id === customProviderId); |
| 104 | + if (!config) throw new Error(`Custom provider "${customProviderId}" is no longer configured.`); |
| 105 | + const apiKey = secretsStore.getSecret(customProviderSecretKey(customProviderId)); |
| 106 | + if (!apiKey && !config.localGpuBackend) throw new Error(`No API key set for ${config.name}. Add one in Settings.`); |
| 107 | + await createOpenAiCompatibleChat(config.baseUrl, config.name)( |
| 108 | + apiKey ?? "local-gpu-backend", |
| 109 | + actualModel, |
| 110 | + messages, |
| 111 | + options, |
| 112 | + onToken, |
| 113 | + signal, |
| 114 | + tools |
| 115 | + ); |
| 116 | + } else { |
| 117 | + const secretKey = PROVIDER_SECRET_KEYS[provider]; |
| 118 | + const apiKey = secretsStore.getSecret(secretKey); |
| 119 | + if (!apiKey) throw new Error(`No API key set for ${provider}. Add one in Settings.`); |
| 120 | + const providerFn = |
| 121 | + provider === "openai" ? openaiProvider.chat : provider === "anthropic" ? anthropicProvider.chat : geminiProvider.chat; |
| 122 | + await providerFn(apiKey, model, messages, options, onToken, signal, tools); |
| 123 | + } |
| 124 | + } finally { |
| 125 | + await activity.finish(); |
| 126 | + } |
| 127 | +} |
| 128 | + |
| 129 | +// Runs a single-turn prompt to completion and returns the full text — |
| 130 | +// what the scheduled-task runner needs, as opposed to chat:send's |
| 131 | +// token-by-token streaming back to the renderer. |
| 132 | +export async function completePrompt(provider: ProviderId, model: string, prompt: string): Promise<string> { |
| 133 | + let text = ""; |
| 134 | + await dispatchChat(provider, model, [{ role: "user", content: prompt }], undefined, (chunk) => { |
| 135 | + text += chunk.message?.content ?? ""; |
| 136 | + }); |
| 137 | + return text; |
| 138 | +} |
0 commit comments