feat(core): support multimodal on/off override in settings

Author: qorzj

packages/cli/src/tests/exec-runner.test.ts

@@ -31,6 +31,7 @@ function createSettings(
     reasoningEffort: "high",
     debugLogEnabled: false,
     telemetryEnabled: false,
+    multimodal: "default",
     permissions,
     enabledSkills: {},
     statusline: { enabled: false, refreshMs: 1000, separator: " | ", providers: [] },

packages/core/src/common/model-capabilities.ts

@@ -7,10 +7,26 @@ export const NON_MULTIMODAL_MODELS = new Set([
   "deepseek-reasoner",
 ]);
 
+export type MultimodalMode = "default" | "on" | "off";
+
 export function defaultsToThinkingMode(model: string): boolean {
   return DEEPSEEK_V4_MODELS.has(model);
 }
 
-export function supportsMultimodal(model: string): boolean {
+/**
+ * Whether the given model supports multimodal (image) content.
+ *
+ * `mode` is the resolved `multimodal` configuration:
+ * - `"on"`: always treat the model as multimodal.
+ * - `"off"`: always treat the model as non-multimodal.
+ * - `"default"` (or omitted): infer from the known model list.
+ */
+export function supportsMultimodal(model: string, mode: MultimodalMode = "default"): boolean {
+  if (mode === "on") {
+    return true;
+  }
+  if (mode === "off") {
+    return false;
+  }
   return !NON_MULTIMODAL_MODELS.has(model.trim());
 }

packages/core/src/common/openai-message-converter.ts

@@ -1,5 +1,5 @@
 import type { ChatCompletionMessageParam, ChatCompletionContentPart } from "openai/resources/chat/completions";
-import { supportsMultimodal } from "./model-capabilities";
+import { supportsMultimodal, type MultimodalMode } from "./model-capabilities";
 import type { SessionMessage } from "../session";
 
 export type OpenAIMessageConverterOptions = {
@@ -23,7 +23,12 @@ export class OpenAIMessageConverter {
    * Build the OpenAI messages array from session messages, applying compaction
    * filtering, tool pairing, and format conversion.
    */
-  buildMessages(messages: SessionMessage[], thinkingEnabled: boolean, model: string): ChatCompletionMessageParam[] {
+  buildMessages(
+    messages: SessionMessage[],
+    thinkingEnabled: boolean,
+    model: string,
+    multimodal: MultimodalMode = "default"
+  ): ChatCompletionMessageParam[] {
     const activeMessages = messages.filter((message) => !message.compacted);
     const toolPairings = this.pairToolMessages(activeMessages);
     const openAIMessages: ChatCompletionMessageParam[] = [];
@@ -34,7 +39,7 @@ export class OpenAIMessageConverter {
         continue;
       }
 
-      openAIMessages.push(this.convertMessage(message, thinkingEnabled, model));
+      openAIMessages.push(this.convertMessage(message, thinkingEnabled, model, multimodal));
 
       const toolCalls = this.getAssistantToolCalls(message);
       if (toolCalls.length === 0) {
@@ -49,7 +54,7 @@ export class OpenAIMessageConverter {
 
         const pairedToolIndex = toolPairings.get(this.buildToolPairingKey(index, toolCallIndex));
         if (pairedToolIndex != null) {
-          openAIMessages.push(this.convertMessage(activeMessages[pairedToolIndex], thinkingEnabled, model));
+          openAIMessages.push(this.convertMessage(activeMessages[pairedToolIndex], thinkingEnabled, model, multimodal));
           continue;
         }
 
@@ -87,7 +92,12 @@ export class OpenAIMessageConverter {
   // Private helpers
   // ---------------------------------------------------------------------------
 
-  private convertMessage(message: SessionMessage, thinkingEnabled: boolean, model: string): ChatCompletionMessageParam {
+  private convertMessage(
+    message: SessionMessage,
+    thinkingEnabled: boolean,
+    model: string,
+    multimodal: MultimodalMode = "default"
+  ): ChatCompletionMessageParam {
     const content = this.renderContent(message);
     const base: ChatCompletionMessageParam = {
       role: message.role,
@@ -120,7 +130,7 @@ export class OpenAIMessageConverter {
       const params = Array.isArray(message.contentParams) ? message.contentParams : [message.contentParams];
       for (const param of params) {
         const part = param as ChatCompletionContentPart;
-        if (part && (part.type !== "image_url" || supportsMultimodal(model))) {
+        if (part && (part.type !== "image_url" || supportsMultimodal(model, multimodal))) {
           contentParts.push(part);
         }
       }

packages/core/src/index.ts

@@ -107,6 +107,7 @@ export { killProcessTree } from "./common/process-tree";
 export { launchNotifyScript } from "./common/notify";
 export { reportNewPrompt } from "./common/telemetry";
 export { DEEPSEEK_V4_MODELS, supportsMultimodal, defaultsToThinkingMode } from "./common/model-capabilities";
+export type { MultimodalMode } from "./common/model-capabilities";
 export { findGitBashPath, resolveShellPath, setShellIfWindows } from "./common/shell-utils";
 export { logApiError } from "./common/error-logger";
 export { logOpenAIChatCompletionDebug } from "./common/debug-logger";

packages/core/src/prompt.ts

@@ -7,7 +7,7 @@ import matter from "gray-matter";
 import { fileURLToPath } from "url";
 import type { SessionMessage } from "./session";
 import { findGitBashPath, resolveShellPath } from "./common/shell-utils";
-import { supportsMultimodal } from "./common/model-capabilities";
+import { supportsMultimodal, type MultimodalMode } from "./common/model-capabilities";
 
 const COMPACT_PROMPT_BASE = `Your task is to create a detailed summary of the conversation so far, paying close attention to the user's explicit requests and your previous actions.
 This summary should be thorough in capturing technical details, code patterns, and architectural decisions that would be essential for continuing development work without losing context.
@@ -95,6 +95,7 @@ const SYSTEM_PROMPT_BASE = `You are a helpful software engineer assistant.`;
 
 export type PromptToolOptions = {
   model?: string;
+  multimodal?: MultimodalMode;
   webSearchEnabled?: boolean;
   nonInteractive?: boolean;
 };
@@ -140,7 +141,7 @@ function readToolDocs(extensionRoot: string, options: PromptToolOptions = {}): s
       try {
         const template = fs.readFileSync(fullPath, "utf8");
         const content = entry.endsWith(".ejs")
-          ? ejs.render(template, { supportsMultimodal: supportsMultimodal(options.model ?? "") })
+          ? ejs.render(template, { supportsMultimodal: supportsMultimodal(options.model ?? "", options.multimodal) })
           : template;
         return content.trim();
       } catch {
@@ -695,7 +696,7 @@ export function getTools(_options: PromptToolOptions = {}, externalTools: ToolDe
     },
   });
 
-  if (!supportsMultimodal(_options.model ?? "")) {
+  if (!supportsMultimodal(_options.model ?? "", _options.multimodal)) {
     tools.push({
       type: "function",
       function: {

packages/core/src/session.ts

@@ -57,7 +57,7 @@ import {
 import { clearSessionWorkingDir } from "./tools/bash-handler";
 import { reportNewPrompt } from "./common/telemetry";
 import { OpenAIMessageConverter } from "./common/openai-message-converter";
-import { supportsMultimodal } from "./common/model-capabilities";
+import { supportsMultimodal, type MultimodalMode } from "./common/model-capabilities";
 
 export type { PermissionScope } from "./settings";
 export type {
@@ -338,6 +338,7 @@ export type SessionManagerOptions = {
   createOpenAIClient: CreateOpenAIClient;
   getResolvedSettings: () => {
     model: string;
+    multimodal?: MultimodalMode;
     contextWindow?: number;
     autoCompactWindow?: number;
     webSearchTool?: string;
@@ -368,6 +369,7 @@ export class SessionManager {
   private readonly createOpenAIClient: CreateOpenAIClient;
   private readonly getResolvedSettings: () => {
     model: string;
+    multimodal?: MultimodalMode;
     contextWindow?: number;
     autoCompactWindow?: number;
     webSearchTool?: string;
@@ -415,9 +417,10 @@ export class SessionManager {
   buildOpenAIMessages(
     messages: SessionMessage[],
     thinkingEnabled: boolean,
-    model: string
+    model: string,
+    multimodal?: MultimodalMode
   ): ChatCompletionMessageParam[] {
-    return this.messageConverter.buildMessages(messages, thinkingEnabled, model);
+    return this.messageConverter.buildMessages(messages, thinkingEnabled, model, multimodal);
   }
 
   async initMcpServers(servers?: Record<string, McpServerConfig>): Promise<void> {
@@ -1494,7 +1497,8 @@ ${agentInstructions}
         const messages = this.messageConverter.buildMessages(
           this.prepareSessionMessagesForRequest(this.listSessionMessages(sessionId)),
           thinkingEnabled,
-          model
+          model,
+          this.getResolvedSettings().multimodal
         );
         const thinkingOptions = buildThinkingRequestOptions(thinkingEnabled, baseURL, reasoningEffort);
         const response = await this.createChatCompletionStream(
@@ -1731,9 +1735,15 @@ ${agentInstructions}
     this.saveSessionMessages(sessionId, sessionMessages);
   }
 
-  private getPromptToolOptions(): { model: string; webSearchEnabled: boolean; nonInteractive: boolean } {
+  private getPromptToolOptions(): {
+    model: string;
+    multimodal?: MultimodalMode;
+    webSearchEnabled: boolean;
+    nonInteractive: boolean;
+  } {
     return {
       model: this.getResolvedSettings().model,
+      multimodal: this.getResolvedSettings().multimodal,
       webSearchEnabled: true,
       nonInteractive: this.nonInteractive,
     };
@@ -2327,7 +2337,7 @@ ${agentInstructions}
   }
 
   private preparePromptImages(sessionId: string, prompt: UserPromptContent): UserPromptContent {
-    if (supportsMultimodal(this.getResolvedSettings().model)) {
+    if (supportsMultimodal(this.getResolvedSettings().model, this.getResolvedSettings().multimodal)) {
       return prompt;
     }
 

packages/core/src/settings.ts

@@ -1,4 +1,4 @@
-import { DEEPSEEK_V4_MODELS, defaultsToThinkingMode } from "./common/model-capabilities";
+import { DEEPSEEK_V4_MODELS, defaultsToThinkingMode, type MultimodalMode } from "./common/model-capabilities";
 import * as fs from "fs";
 import * as os from "os";
 import * as path from "path";
@@ -12,6 +12,7 @@ export type DeepcodingEnv = Record<string, string | undefined> & {
   REASONING_EFFORT?: string;
   DEBUG_LOG_ENABLED?: string;
   TELEMETRY_ENABLED?: string;
+  MULTIMODAL?: string;
 };
 
 export type ReasoningEffort = "high" | "max";
@@ -92,6 +93,7 @@ export type DeepcodingSettings = {
   telemetryEnabled?: boolean;
   notify?: string;
   webSearchTool?: string;
+  multimodal?: MultimodalMode;
   mcpServers?: Record<string, McpServerConfig>;
   permissions?: PermissionSettings;
   enabledSkills?: EnabledSkillsSettings;
@@ -112,6 +114,7 @@ export type ResolvedDeepcodingSettings = {
   telemetryEnabled: boolean;
   notify?: string;
   webSearchTool?: string;
+  multimodal: MultimodalMode;
   mcpServers?: Record<string, McpServerConfig>;
   permissions: Required<PermissionSettings>;
   enabledSkills: EnabledSkillsSettings;
@@ -169,6 +172,17 @@ function resolveReasoningEffort(value: unknown): ReasoningEffort | undefined {
   return value === "high" || value === "max" ? value : undefined;
 }
 
+function resolveMultimodalMode(value: unknown): MultimodalMode | undefined {
+  if (typeof value !== "string") {
+    return undefined;
+  }
+  const normalized = value.trim().toLowerCase();
+  if (normalized === "default" || normalized === "on" || normalized === "off") {
+    return normalized;
+  }
+  return undefined;
+}
+
 function parseBoolean(value: unknown): boolean | undefined {
   if (typeof value === "boolean") {
     return value;
@@ -585,6 +599,14 @@ export function resolveSettingsSources(
     trimString(userSettings?.webSearchTool) ||
     "";
 
+  const multimodal =
+    resolveMultimodalMode(systemEnv.MULTIMODAL) ??
+    resolveMultimodalMode(projectSettings?.multimodal) ??
+    resolveMultimodalMode(projectEnv.MULTIMODAL) ??
+    resolveMultimodalMode(userSettings?.multimodal) ??
+    resolveMultimodalMode(userEnv.MULTIMODAL) ??
+    "default";
+
   return {
     env,
     apiKey: trimString(env.API_KEY) || undefined,
@@ -599,6 +621,7 @@ export function resolveSettingsSources(
     telemetryEnabled,
     notify: notify || undefined,
     webSearchTool: webSearchTool || undefined,
+    multimodal,
     mcpServers: mergeMcpServers(userSettings, projectSettings, userEnv, projectEnv, systemEnv),
     permissions: mergePermissions(userSettings, projectSettings),
     enabledSkills: mergeEnabledSkills(userSettings, projectSettings),

packages/core/src/tests/openai-message-converter.test.ts

@@ -110,6 +110,28 @@ test("OpenAIMessageConverter filters image content for non-multimodal models", (
   assert.deepEqual(result[0]?.content, [{ type: "text", text: "Loaded pixel.png" }]);
 });
 
+test("OpenAIMessageConverter multimodal config overrides model-based filtering", () => {
+  const c = converter();
+  const messages: SessionMessage[] = [
+    msg({
+      role: "system",
+      content: "Loaded pixel.png",
+      contentParams: [{ type: "image_url", image_url: { url: "data:image/png;base64,abc" } }],
+    }),
+  ];
+
+  // "off" drops image content even for a multimodal model.
+  const off = c.buildMessages(messages, false, "gpt-4o", "off") as Array<{ content: unknown }>;
+  assert.deepEqual(off[0]?.content, [{ type: "text", text: "Loaded pixel.png" }]);
+
+  // "on" keeps image content even for a non-multimodal model.
+  const on = c.buildMessages(messages, false, "deepseek-chat", "on") as Array<{ content: unknown }>;
+  assert.deepEqual(on[0]?.content, [
+    { type: "text", text: "Loaded pixel.png" },
+    { type: "image_url", image_url: { url: "data:image/png;base64,abc" } },
+  ]);
+});
+
 test("OpenAIMessageConverter injects reasoning_content in thinking mode", () => {
   const c = converter();
   const messages: SessionMessage[] = [msg({ role: "assistant", content: "Final answer", messageParams: null })];

packages/core/src/tests/prompt.test.ts

@@ -43,6 +43,24 @@ test("UnderstandImage is available only to non-multimodal models", () => {
   assert.equal(getSystemPrompt("/tmp/project", { model: "gpt-4o" }).includes("## UnderstandImage"), false);
 });
 
+test("multimodal config overrides model-based multimodal detection", () => {
+  // "off" forces non-multimodal behavior even for a multimodal model.
+  const forcedOffTools = getTools({ model: "gpt-4o", multimodal: "off" }).map((tool) => tool.function.name);
+  assert.equal(forcedOffTools.includes("UnderstandImage"), true);
+  assert.equal(
+    getSystemPrompt("/tmp/project", { model: "gpt-4o", multimodal: "off" }).includes("## UnderstandImage"),
+    true
+  );
+
+  // "on" forces multimodal behavior even for a non-multimodal model.
+  const forcedOnTools = getTools({ model: "deepseek-chat", multimodal: "on" }).map((tool) => tool.function.name);
+  assert.equal(forcedOnTools.includes("UnderstandImage"), false);
+  assert.equal(
+    getSystemPrompt("/tmp/project", { model: "deepseek-chat", multimodal: "on" }).includes("## UnderstandImage"),
+    false
+  );
+});
+
 test("interactive prompt and tools include AskUserQuestion", () => {
   assert.equal(getSystemPrompt("/tmp/project").includes("## AskUserQuestion"), true);
   assert.equal(