From ad7de8e41f6d7be3f8be4adc077d79ab3de8ebf0 Mon Sep 17 00:00:00 2001 From: rainbowgits <164521089+rainbowgits@users.noreply.github.com> Date: Thu, 6 Aug 2026 04:00:24 +0300 Subject: [PATCH] fix(soul): degrade unsupported tool media instead of aborting mid-task When a tool returns image/video the model lacks capability for, omit the media and continue rather than raising after side effects. Also name the config remedy in LLMNotSupported errors (#2588). Co-authored-by: Cursor --- CHANGELOG.md | 2 + docs/en/configuration/providers.md | 8 +- docs/en/faq.md | 5 + docs/en/release-notes/changelog.md | 2 + docs/zh/configuration/providers.md | 8 +- docs/zh/faq.md | 5 + docs/zh/release-notes/changelog.md | 2 + src/kimi_cli/soul/__init__.py | 5 +- src/kimi_cli/soul/kimisoul.py | 22 ++++- src/kimi_cli/soul/message.py | 45 +++++++++ tests/core/test_exceptions.py | 4 +- tests/core/test_grow_context_media.py | 126 ++++++++++++++++++++++++++ tests/core/test_soul_message.py | 55 ++++++++++- tests_e2e/test_wire_errors.py | 6 +- 14 files changed, 282 insertions(+), 13 deletions(-) create mode 100644 tests/core/test_grow_context_media.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e500e09c2..00f4373864 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ Only write entries that are worth mentioning to users. ## Unreleased +- Core: Continue the turn when a tool returns image or video the model is not declared to support — omit the unsupported media with a note instead of aborting mid-task after the tool has already run (#2588) +- Core: Include the config remedy in `LLMNotSupported` errors — tell users to add `capabilities = ["image_in"]` (or similar) to `[models.]` in `config.toml`, or set `KIMI_MODEL_CAPABILITIES` (#2588) - Kosong: Stop sending an empty `anthropic-beta` header when no beta features are declared — adaptive thinking removes the interleaved-thinking beta, which previously left an empty header value that some backends reject ## 1.49.0 (2026-07-16) diff --git a/docs/en/configuration/providers.md b/docs/en/configuration/providers.md index f1367f6fe5..4e77e25a91 100644 --- a/docs/en/configuration/providers.md +++ b/docs/en/configuration/providers.md @@ -133,11 +133,15 @@ Indicates the model always uses thinking mode and cannot be disabled. For exampl ### `image_in` -When image input capability is enabled, you can paste images in conversations (`Ctrl-V`). +When image input capability is enabled, you can paste images in conversations (`Ctrl-V`), and the agent can send image content returned by tools (including MCP tools) to the model. + +If the model is not declared with `image_in` and a tool returns an image, Kimi Code CLI omits that image, continues the turn with a short note, and tells you to add `capabilities = ["image_in"]` under `[models.]` (or set `KIMI_MODEL_CAPABILITIES`). User-pasted images still require `image_in` and fail with that guidance if it is missing. + +Custom model aliases (for example local `openai_legacy` gateways) do not get `image_in` from name-based auto-detection — declare `capabilities` explicitly when the model can accept images. ### `video_in` -When video input capability is enabled, you can send video content in conversations. +When video input capability is enabled, you can send video content in conversations. Tool-returned video is handled the same way as images: without `video_in`, the video is omitted and the turn continues with a note naming the config fix. ## Search and fetch services diff --git a/docs/en/faq.md b/docs/en/faq.md index 4cb33d3883..ad760a462d 100644 --- a/docs/en/faq.md +++ b/docs/en/faq.md @@ -40,8 +40,13 @@ When using `Ctrl-V` to paste an image, if you see "Current model does not suppor Solutions: - **Switch to an image-capable model**: Use a model that supports the `image_in` capability. +- **Declare capabilities on custom aliases**: For local or OpenAI-compatible models under your own alias, add `capabilities = ["image_in"]` to `[models.]` in `config.toml` (or set `KIMI_MODEL_CAPABILITIES`). Name-based auto-detection does not cover arbitrary aliases. - **Check clipboard content**: Make sure the clipboard contains actual image data, not just a file path to an image. +### Tool or MCP screenshot is omitted / capability error + +If a tool (including an MCP tool) returns an image but the selected model has no `image_in` capability, Kimi Code CLI omits the image, continues the turn, and points you at adding `capabilities = ["image_in"]` to the model entry. User-provided images still error with the same config guidance. + ### Working directory deleted or removed If the working directory becomes inaccessible during a session (external drive unplugged, directory deleted, or filesystem unmounted), Kimi Code CLI detects the situation and displays a crash report containing the session ID and work directory path, then exits cleanly. You can recover the session with `kimi -r ` from the correct directory. diff --git a/docs/en/release-notes/changelog.md b/docs/en/release-notes/changelog.md index 055688133c..fba24dff76 100644 --- a/docs/en/release-notes/changelog.md +++ b/docs/en/release-notes/changelog.md @@ -4,6 +4,8 @@ This page documents the changes in each Kimi Code CLI release. ## Unreleased +- Core: Continue the turn when a tool returns image or video the model is not declared to support — omit the unsupported media with a note instead of aborting mid-task after the tool has already run (#2588) +- Core: Include the config remedy in `LLMNotSupported` errors — tell users to add `capabilities = ["image_in"]` (or similar) to `[models.]` in `config.toml`, or set `KIMI_MODEL_CAPABILITIES` (#2588) - Kosong: Stop sending an empty `anthropic-beta` header when no beta features are declared — adaptive thinking removes the interleaved-thinking beta, which previously left an empty header value that some backends reject ## 1.49.0 (2026-07-16) diff --git a/docs/zh/configuration/providers.md b/docs/zh/configuration/providers.md index f07e497333..5411160ff5 100644 --- a/docs/zh/configuration/providers.md +++ b/docs/zh/configuration/providers.md @@ -133,11 +133,15 @@ capabilities = ["thinking", "image_in"] ### `image_in` -启用图片输入能力后，可以在对话中粘贴图片（`Ctrl-V`）。 +启用图片输入能力后，可以在对话中粘贴图片（`Ctrl-V`），代理也可以将工具（包括 MCP 工具）返回的图片内容发给模型。 + +如果模型未声明 `image_in`，而工具返回了图片，Kimi Code CLI 会省略该图片、以简短说明继续当前轮次，并提示你在 `[models.]` 下添加 `capabilities = ["image_in"]`（或设置 `KIMI_MODEL_CAPABILITIES`）。用户粘贴的图片仍然需要 `image_in`；缺失时会报错并给出同样的配置指引。 + +自定义模型别名（例如本地 `openai_legacy` 网关）不会通过名称自动检测获得 `image_in`——当模型可以接受图片时，请显式声明 `capabilities`。 ### `video_in` -启用视频输入能力后，可以在对话中发送视频内容。 +启用视频输入能力后，可以在对话中发送视频内容。工具返回的视频与图片处理方式相同：缺少 `video_in` 时会省略视频，并以说明配置修复方式的提示继续当前轮次。 ## 搜索和抓取服务 diff --git a/docs/zh/faq.md b/docs/zh/faq.md index a82d0b9b0d..36830ebbb7 100644 --- a/docs/zh/faq.md +++ b/docs/zh/faq.md @@ -40,8 +40,13 @@ API 密钥无效可能的原因： 解决方法： - **切换到支持图片的模型**：使用支持 `image_in` 能力的模型。 +- **在自定义别名上声明能力**：对于本地或 OpenAI 兼容模型的自定义别名，请在 `config.toml` 的 `[models.]` 中添加 `capabilities = ["image_in"]`（或设置 `KIMI_MODEL_CAPABILITIES`）。基于名称的自动检测不覆盖任意别名。 - **检查剪贴板内容**：确保剪贴板中确实有图片数据，而非图片文件的路径。 +### 工具或 MCP 截图被省略 / 能力错误 + +如果工具（包括 MCP 工具）返回了图片，但所选模型没有 `image_in` 能力，Kimi Code CLI 会省略该图片、继续当前轮次，并提示你在模型配置中添加 `capabilities = ["image_in"]`。用户提供的图片仍会报错，并给出同样的配置指引。 + ### 工作目录被删除或移除 如果在会话期间工作目录变得不可访问（外置硬盘拔出、目录被删除或文件系统卸载），Kimi Code CLI 会检测到这一情况并显示崩溃报告，包含会话 ID 和工作目录路径，之后干净退出。你可以通过 `kimi -r ` 在正确的目录中恢复会话。 diff --git a/docs/zh/release-notes/changelog.md b/docs/zh/release-notes/changelog.md index 3202ae71b2..8f9213ce71 100644 --- a/docs/zh/release-notes/changelog.md +++ b/docs/zh/release-notes/changelog.md @@ -4,6 +4,8 @@ ## 未发布 +- Core：当工具返回模型未声明支持的图片或视频时继续当前轮次——省略不受支持的媒体并附上说明，而不再在工具已执行后中途中止任务（#2588） +- Core：在 `LLMNotSupported` 错误中给出配置修复提示——告知用户在 `config.toml` 的 `[models.]` 中添加 `capabilities = ["image_in"]`（或类似能力），或设置 `KIMI_MODEL_CAPABILITIES`（#2588） - Kosong：当没有声明任何 beta 特性时不再发送值为空的 `anthropic-beta` 请求头——adaptive thinking 会移除 interleaved-thinking beta，此前会留下一个空值请求头，部分后端会拒绝该请求 ## 1.49.0 (2026-07-16) diff --git a/src/kimi_cli/soul/__init__.py b/src/kimi_cli/soul/__init__.py index 5cde510dc5..61ae9f3b5e 100644 --- a/src/kimi_cli/soul/__init__.py +++ b/src/kimi_cli/soul/__init__.py @@ -34,9 +34,12 @@ def __init__(self, llm: LLM, capabilities: list[ModelCapability]): self.llm = llm self.capabilities = capabilities capabilities_str = "capability" if len(capabilities) == 1 else "capabilities" + caps_list = ", ".join(f'"{cap}"' for cap in capabilities) super().__init__( f"LLM model '{llm.model_name}' does not support required {capabilities_str}: " - f"{', '.join(capabilities)}." + f"{', '.join(capabilities)}. " + f"Add capabilities = [{caps_list}] to [models.] in config.toml " + f"(or set KIMI_MODEL_CAPABILITIES)." ) diff --git a/src/kimi_cli/soul/kimisoul.py b/src/kimi_cli/soul/kimisoul.py index 3f14c2a2f7..bcb80f1bb7 100644 --- a/src/kimi_cli/soul/kimisoul.py +++ b/src/kimi_cli/soul/kimisoul.py @@ -75,7 +75,13 @@ ) from kimi_cli.soul.dynamic_injections.afk_mode import AfkModeInjectionProvider from kimi_cli.soul.dynamic_injections.plan_mode import PlanModeInjectionProvider -from kimi_cli.soul.message import check_message, system, system_reminder, tool_result_to_message +from kimi_cli.soul.message import ( + check_message, + omit_unsupported_media, + system, + system_reminder, + tool_result_to_message, +) from kimi_cli.soul.slash import registry as soul_slash_registry from kimi_cli.soul.toolset import KimiToolset from kimi_cli.tools.dmail import NAME as SendDMail_NAME @@ -1391,13 +1397,19 @@ async def _grow_context(self, result: StepResult, tool_results: list[ToolResult] assert self._runtime.llm is not None tool_messages = [tool_result_to_message(tr) for tr in tool_results] + # Drop unsupported media from tool results instead of aborting mid-task + # after the tool has already run (see #2588). User-provided media still + # raises LLMNotSupported via check_message on the turn/steer paths. + rewritten: list[Message] = [] for tm in tool_messages: - if missing_caps := check_message(tm, self._runtime.llm.capabilities): + omitted_msg, omitted_caps = omit_unsupported_media(tm, self._runtime.llm.capabilities) + if omitted_caps: logger.warning( - "Tool result message requires unsupported capabilities: {caps}", - caps=missing_caps, + "Omitting unsupported media from tool result (missing capabilities: {caps})", + caps=omitted_caps, ) - raise LLMNotSupported(self._runtime.llm, list(missing_caps)) + rewritten.append(omitted_msg) + tool_messages = rewritten await self._context.append_message(result.message) if result.usage is not None: diff --git a/src/kimi_cli/soul/message.py b/src/kimi_cli/soul/message.py index e8e823d452..4e3fc9709e 100644 --- a/src/kimi_cli/soul/message.py +++ b/src/kimi_cli/soul/message.py @@ -90,3 +90,48 @@ def check_message( elif isinstance(part, ThinkPart): capabilities_needed.add("thinking") return capabilities_needed - model_capabilities + + +def omit_unsupported_media( + message: Message, model_capabilities: set[ModelCapability] +) -> tuple[Message, set[ModelCapability]]: + """Replace unsupported image/video parts with a short text note. + + Used for tool results so a missing ``image_in``/``video_in`` capability does + not abort the run after the tool has already executed. Returns the (possibly + rewritten) message and the set of capabilities that were omitted. + """ + omitted = set[ModelCapability]() + new_content: list[ContentPart] = [] + for part in message.content: + if isinstance(part, ImageURLPart) and "image_in" not in model_capabilities: + omitted.add("image_in") + new_content.append( + system( + "Image omitted: model has no image_in capability. " + 'Add capabilities = ["image_in"] to [models.] in config.toml.' + ) + ) + elif isinstance(part, VideoURLPart) and "video_in" not in model_capabilities: + omitted.add("video_in") + new_content.append( + system( + "Video omitted: model has no video_in capability. " + 'Add capabilities = ["video_in"] to [models.] in config.toml.' + ) + ) + else: + new_content.append(part) + + if not omitted: + return message, omitted + + return ( + Message( + role=message.role, + content=new_content, + tool_calls=message.tool_calls, + tool_call_id=message.tool_call_id, + ), + omitted, + ) diff --git a/tests/core/test_exceptions.py b/tests/core/test_exceptions.py index d0f7282628..43566e6c99 100644 --- a/tests/core/test_exceptions.py +++ b/tests/core/test_exceptions.py @@ -15,7 +15,9 @@ def test_soul_exceptions(llm: LLM): raise LLMNotSupported(llm, ["image_in"]) except LLMNotSupported as e: assert str(e) == snapshot( - "LLM model 'mock' does not support required capability: image_in." + "LLM model 'mock' does not support required capability: image_in. " + 'Add capabilities = ["image_in"] to [models.] in config.toml ' + "(or set KIMI_MODEL_CAPABILITIES)." ) try: diff --git a/tests/core/test_grow_context_media.py b/tests/core/test_grow_context_media.py new file mode 100644 index 0000000000..176f613522 --- /dev/null +++ b/tests/core/test_grow_context_media.py @@ -0,0 +1,126 @@ +"""Tests for capability handling of media in tool results during _grow_context (#2588).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from kosong import StepResult +from kosong.message import Message +from kosong.tooling import ToolOk +from kosong.tooling.empty import EmptyToolset + +from kimi_cli.llm import LLM, ModelCapability +from kimi_cli.soul.agent import Agent, Runtime +from kimi_cli.soul.context import Context +from kimi_cli.soul.kimisoul import KimiSoul +from kimi_cli.wire.types import ImageURLPart, TextPart, ToolResult + + +def _make_soul(runtime: Runtime, tmp_path: Path) -> KimiSoul: + agent = Agent( + name="Grow Context Media Test Agent", + system_prompt="Test prompt.", + toolset=EmptyToolset(), + runtime=runtime, + ) + return KimiSoul(agent, context=Context(file_backend=tmp_path / "history.jsonl")) + + +def _runtime_with_llm(runtime: Runtime, llm: LLM) -> Runtime: + return Runtime( + config=runtime.config, + llm=llm, + session=runtime.session, + builtin_args=runtime.builtin_args, + denwa_renji=runtime.denwa_renji, + approval=runtime.approval, + labor_market=runtime.labor_market, + environment=runtime.environment, + notifications=runtime.notifications, + background_tasks=runtime.background_tasks, + skills=runtime.skills, + oauth=runtime.oauth, + additional_dirs=runtime.additional_dirs, + skills_dirs=runtime.skills_dirs, + ) + + +def _llm_with_capabilities(runtime: Runtime, capabilities: set[ModelCapability]) -> LLM: + assert runtime.llm is not None + return LLM( + chat_provider=runtime.llm.chat_provider, + max_context_size=runtime.llm.max_context_size, + capabilities=capabilities, + model_config=runtime.llm.model_config, + provider_config=runtime.llm.provider_config, + ) + + +@pytest.mark.asyncio +async def test_grow_context_omits_image_when_model_lacks_image_in( + runtime: Runtime, tmp_path: Path +) -> None: + soul = _make_soul(_runtime_with_llm(runtime, _llm_with_capabilities(runtime, set())), tmp_path) + + await soul.context.append_message(Message(role="user", content=[TextPart(text="edit doc")])) + await soul.context.append_message( + Message(role="assistant", content=[TextPart(text="edited successfully")]) + ) + + img = ImageURLPart(image_url=ImageURLPart.ImageURL(url="data:image/png;base64,AAA")) + tool_results = [ + ToolResult( + tool_call_id="call_screenshot", + return_value=ToolOk(message="screenshot captured", output=img), + ) + ] + step = StepResult( + id="step1", + message=Message(role="assistant", content=[TextPart(text="taking screenshot")]), + usage=None, + tool_calls=[], + _tool_result_futures={}, + ) + + await soul._grow_context(step, tool_results) + + # Prior side effects remain, and the turn continues with a text stand-in. + assert len(soul.context.history) == 4 + tool_msg = soul.context.history[-1] + assert tool_msg.role == "tool" + assert tool_msg.tool_call_id == "call_screenshot" + assert not any(isinstance(part, ImageURLPart) for part in tool_msg.content) + assert any( + isinstance(part, TextPart) and "Image omitted" in part.text for part in tool_msg.content + ) + + +@pytest.mark.asyncio +async def test_grow_context_keeps_image_when_model_has_image_in( + runtime: Runtime, tmp_path: Path +) -> None: + soul = _make_soul( + _runtime_with_llm(runtime, _llm_with_capabilities(runtime, {"image_in"})), + tmp_path, + ) + + img = ImageURLPart(image_url=ImageURLPart.ImageURL(url="data:image/png;base64,AAA")) + tool_results = [ + ToolResult( + tool_call_id="call_screenshot", + return_value=ToolOk(message="screenshot captured", output=img), + ) + ] + step = StepResult( + id="step1", + message=Message(role="assistant", content=[TextPart(text="taking screenshot")]), + usage=None, + tool_calls=[], + _tool_result_futures={}, + ) + + await soul._grow_context(step, tool_results) + + tool_msg = soul.context.history[-1] + assert any(isinstance(part, ImageURLPart) for part in tool_msg.content) diff --git a/tests/core/test_soul_message.py b/tests/core/test_soul_message.py index c51d5519c9..eb08673953 100644 --- a/tests/core/test_soul_message.py +++ b/tests/core/test_soul_message.py @@ -5,7 +5,12 @@ from kosong.tooling import ToolError, ToolOk from kimi_cli.llm import ModelCapability -from kimi_cli.soul.message import check_message, system, tool_result_to_message +from kimi_cli.soul.message import ( + check_message, + omit_unsupported_media, + system, + tool_result_to_message, +) from kimi_cli.wire.types import ( AudioURLPart, ImageURLPart, @@ -331,3 +336,51 @@ def test_check_message_with_text_only(): missing_capabilities = check_message(message, model_capabilities) assert missing_capabilities == set() + + +def test_omit_unsupported_media_replaces_image_when_missing_capability(): + image_part = ImageURLPart(image_url=ImageURLPart.ImageURL(url="data:image/png;base64,AAA")) + message = Message( + role="tool", + content=[system("screenshot captured"), image_part], + tool_call_id="call_1", + ) + + rewritten, omitted = omit_unsupported_media(message, set()) + + assert omitted == {"image_in"} + assert rewritten.tool_call_id == "call_1" + assert rewritten.content == [ + system("screenshot captured"), + system( + "Image omitted: model has no image_in capability. " + 'Add capabilities = ["image_in"] to [models.] in config.toml.' + ), + ] + assert check_message(rewritten, set()) == set() + + +def test_omit_unsupported_media_keeps_image_when_capability_present(): + image_part = ImageURLPart(image_url=ImageURLPart.ImageURL(url="data:image/png;base64,AAA")) + message = Message(role="tool", content=[image_part], tool_call_id="call_1") + + rewritten, omitted = omit_unsupported_media(message, {"image_in"}) + + assert omitted == set() + assert rewritten is message + + +def test_omit_unsupported_media_replaces_video_when_missing_capability(): + video_part = VideoURLPart(video_url=VideoURLPart.VideoURL(url="https://example.com/v.mp4")) + message = Message(role="tool", content=[video_part], tool_call_id="call_1") + + rewritten, omitted = omit_unsupported_media(message, set()) + + assert omitted == {"video_in"} + assert rewritten.content == [ + system( + "Video omitted: model has no video_in capability. " + 'Add capabilities = ["video_in"] to [models.] in config.toml.' + ) + ] + assert check_message(rewritten, set()) == set() diff --git a/tests_e2e/test_wire_errors.py b/tests_e2e/test_wire_errors.py index 8c08d386fa..ba2424f0ad 100644 --- a/tests_e2e/test_wire_errors.py +++ b/tests_e2e/test_wire_errors.py @@ -164,7 +164,11 @@ def test_llm_not_supported(tmp_path) -> None: { "error": { "code": -32002, - "message": "LLM model 'scripted_echo' does not support required capability: image_in.", + "message": ( + "LLM model 'scripted_echo' does not support required capability: " + 'image_in. Add capabilities = ["image_in"] to [models.] in ' + "config.toml (or set KIMI_MODEL_CAPABILITIES)." + ), "data": None, } }