From f94a5583237f1e70df16c3f178e0993115764526 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 23 Jul 2026 17:46:06 -0700 Subject: [PATCH 01/30] docs: add prompt queueing feature docs to README, website, and plans --- README.md | 442 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 441 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e97674284a4..9a2f7e0fdf0 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,52 @@ uv tool install --native-tls --python python3.12 cecli-dev Use the tool installation so cecli doesn't interfere with your development environment -## Configuration +## Prompt Queueing Feature + +The cecli application now includes a prompt queueing feature that allows users to manage multiple prompts in a first-in-first-out (FIFO) queue with a configurable maximum size. + +### How It Works + +When the prompt queue is enabled, incoming prompts are added to a queue instead of being processed immediately. The queue has a default maximum size of 5 prompts. When the queue is full and a new prompt is added, the oldest prompt is automatically removed to make space for the new one. + +### Configuration + +The maximum queue size can be configured using the `max_queue_size` parameter in your cecli configuration: + +```bash +cecli --max_queue_size 10 +``` + +Or in your `.cecli.conf.yml` file: + +```yaml +max_queue_size: 10 +``` + +### Usage + +1. **Queue Management**: Prompts are automatically queued when the queue is enabled +2. **View Queue**: The TUI interface displays the current queue of prompts +3. **Remove from Queue**: Use the `/queue-remove` command to remove specific prompts from the queue + +### /queue-remove Command + +The `/queue-remove` command allows you to remove specific prompts from the queue: + +```bash +/queue-remove 3 +``` + +This command removes the prompt at index 3 from the queue (0-based indexing). Tab completion is available for prompt IDs. + +### Benefits + +- Prevents overwhelming the system with too many concurrent prompts +- Allows users to review and manage their prompt queue +- Provides better control over prompt processing order +- Automatically handles overflow by removing oldest prompts + +The prompt queueing feature enhances the user experience by providing better control over prompt processing and preventing system overload. The documentation above contains the full set of allowed configuration options but I highly recommend using an `.cecli.conf.yml` file. A good place to get started is: @@ -132,10 +177,405 @@ This command will make sure all commands ran by the coding agent happen in conte * [Advanced Model Configuration](https://github.com/cecli-dev/cecli/blob/main/cecli/website/docs/config/model-aliases.md#advanced-model-settings) * [Additional Documentation](https://cecli.dev/) +* [Agent Mode](https://github.com/cecli-dev/cecli/blob/main/cecli/website/docs/config/agent-mode.md) +* [MCP Configuration](https://github.com/cecli-dev/cecli/blob/main/cecli/website/docs/config/mcp.md) +* [TUI Configuration](https://github.com/cecli-dev/cecli/blob/main/cecli/website/docs/config/tui.md) +* [Skills](https://github.com/cecli-dev/cecli/blob/main/cecli/website/docs/config/skills.md) +* [Subagents](https://github.com/cecli-dev/cecli/blob/main/cecli/website/docs/config/subagents.md) +* [Session Management](https://github.com/cecli-dev/cecli/blob/main/cecli/website/docs/sessions.md) +* [Hooks](https://github.com/cecli-dev/cecli/blob/main/cecli/website/docs/config/hooks.md) +* [Workspaces](https://github.com/cecli-dev/cecli/blob/main/cecli/website/docs/config/workspaces.md) +* [Custom Commands](https://github.com/cecli-dev/cecli/blob/main/cecli/website/docs/config/custom-commands.md) +* [Custom System Prompts](https://github.com/cecli-dev/cecli/blob/main/cecli/website/docs/config/custom-system-prompts.md) +* [Custom Tools](https://github.com/cecli-dev/cecli/blob/main/cecli/website/docs/config/agent-mode.md#creating-custom-tools) +* [Advanced Model Configuration](https://github.com/cecli-dev/cecli/blob/main/cecli/website/docs/config/model-aliases.md#advanced-model-settings) +* [Additional Documentation](https://cecli.dev/) + ## Project Roadmap/Goals The current priorities are to improve core capabilities and user experience of the `cecli` project +1. **Base Asynchronicity (cecli coroutine-experiment branch)** + * [x] Refactor codebase to have the main loop run asynchronously + * [x] Update test harness to work with new asynchronous methods + +2. **Repo Map Accuracy** - [Discussion](https://github.com/cecli-dev/cecli/issues/45) + * [x] [Bias page ranking toward active/editable files in repo map parsing](https://github.com/Aider-AI/aider/issues/2405) + * [x] [Include import information in repo map for richer context](https://github.com/Aider-AI/aider/issues/2688) + * [x] [Handle non-unique symbols that break down in large codebases](https://github.com/Aider-AI/aider/issues/2341) + +3. **Context Discovery** - [Discussion](https://github.com/cecli-dev/cecli/issues/46) + * [ ] Develop AST-based search capabilities + * [x] Enhance file search with ripgrep integration + * [ ] Implement RAG (Retrieval-Augmented Generation) for better code retrieval + * [ ] Build an explicit workflow and local tooling for internal discovery mechanisms + +4. **Context Delivery** - [Discussion](https://github.com/cecli-dev/cecli/issues/47) + * [x] Use workflow for internal discovery to better target file snippets needed for specific tasks (ExploreCode and ReadRange) + * [x] Add support for partial files and code snippets in model completion messages + * [x] Update message request structure for optimal caching + +5. **TUI Experience** - [Discussion](https://github.com/cecli-dev/cecli/issues/48) + * [x] Add a full TUI (probably using textual) to have a visual interface competitive with the other coding agent terminal programs + * [x] Re-integrate pretty output formatting + * [x] Implement a response area, a prompt area with current auto completion capabilities, and a helper area for managing utility commands + +6. **Agent Mode** - [Discussion](https://github.com/cecli-dev/cecli/issues/111) + * [x] Renaming "navigator mode" to "agent mode" for simplicity + * [x] Add an explicit "finished" internal tool + * [x] Add a configuration json setting for agent mode to specify allowed local tools to use, tool call limits, etc. + * [ ] Add a RAG tool for the model to ask questions about the codebase + * [x] Make the system prompts more aggressive about removing unneeded files/content from the context + * [x] Add a plugin-like system for allowing agent mode to use user-defined tools in simple python files + * [x] Add a dynamic tool discovery tool to allow the system to have only the tools it needs in context + +7. **Sub Agents** + * [x] Add `/invoke-agent` command to manually branch a sub agent and return a summary to the main context + * [x] Add an instance-able view of the conversation system so sub agents get their own context and workspaces + * [x] Modify coder classes to have discrete identifiers for themselves/management utilities for them to have their own slices of the world + * [x] Refactor global files like todo lists to live inside instance folders to avoid state conflicts + * [x] Add a `Delegate` tool that launches a sub agent as a background command that the parent model waits for to finish + * [x] Add visibility into active sub agent calls in TUI + +8. **Hooks** + * [x] Add hooks base class for user defined python hooks with an execute method with type and priority settings + * [x] Add hook manager that can accept user defined files and command line commands + * [x] Integrate hook manager with coder classes with hooks for `start`, `end`, `on_message`, `end_message`, `pre_tool`, and `post_tool` + +9. **Efficient File Editing** + * [x] Explore use of hashline file representation for more targeted file editing + * [x] Assuming viability, update SEARCH part of SEARCH/REPLACE with hashline identification (Done with new edit format) + * [x] Update agent mode edit tools to work with hashline identification + * [x] Update internal file diff representation to support hashline propagation + +10. **Dynamic Context Management** + * [x] Update compaction to use observational memory sub agent calls to generate decision records that are used as the compaction basis + * [ ] Persist decision records to disk for sessions with some settings for managing lifetimes of such persistence + * [ ] Integrate RLM to extract information from decision records on disk and other definable notes + * [ ] Add a "describe" tool that launches a sub agent workflow that populates an RLM call's context with: + * Current Conversation History + * Past Decision Records + * Repo Map Found Files + +11. **Quality of Life** + * [ ] Add hot keys support for running repeatable commands like switching between preferred models + * [ ] Unified error message logging inside of `.cecli` directory + +### All Contributors (Both Cecli and Aider main) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
@paul-gauthier@dwash96@tekacs@ErichBSchulz
@ei-grad@joshuavial@chrisnestrud@chr15m
@johbo@fry69@quinlanjager@caseymcc
@shladnik@jamwil@itlackey@tomjuggler
@szmania@vk4s@titusz@bphd
@daniel-vainsencher@1broseidon@akaihola@jalammar
@schpet@iamFIREcrackerJV@KennyDizi
@ivanfioravanti@mdeweerd@itsmeknt@fahmad91
@cheahjs@youknow04@pjcreath@pcamp
@miradnanali@o-nixJonathan Ellis@codeofdusk
@claui@jpshackelford@Taik@Hambaobao
@therealmarv@muravvv@hypn4@gmoz22
@contributor@ctoth@thehunmonkgroup@gcp
@sentienthouseplant@ktakayama@lreeves@nims11
@preynal@tgbender@apaz-cliAlexander Kjeldaas
@zhyuYutaka Matsubara@burnettk@cryptekbits
@deansher@kennyfrc@lentil32@malkoG
@mubashir1osmani@TimPut@zjy1412@savioursho
@jayeshthk@susliko@FeepingCreature@misteral
@aelaguiz@DhirajBhakta@gopar@eltociear
@tao12345666333@jpshack-at-palomar@smh@nhs000
@sannysanoff@ryanfreckleton@mbokinala@yamitzky
@mobyvb@ozapinq@nicolasperez19@varchasgopalaswamy
@ffluk3@tanavamsikrishna@tylersatre@pcgeek86
@tamirzb@taha-yassine@strayer@StevenTCramer
@Skountz@sestrella@rnevius@holoskii
@Netzvamp@peterhadlaw@pauldw@paulmaunders
@omri123@MatthewZMD@mbailey@golergka
@matfat55@mtofano@maledorak@mlang
@marcomayer@you-n-gwangboxue@rti
@prmbiy@omarcinkonis@Oct4Pie@mark-asymbl
@yazgoomichal.sliwa@mdklab@mario7421
liam.liukwmiebach@kAIto47802@jvmncs
@hydai@hstoklosa@gordonlukch@develmusa
@coredevorg@cantalupo555@caetanominuzzo@yzx9
@zackees@wietsevenema@krewenki@vinnymac
@szepeviktor@lattwood@spdustin@henderkes
@daysm@devriesd@daniel-sc@damms005
@curran@cclauss@cjoach@csala
@bexelbie@branchv@bkowalik@h0x91b
@aroffe99@banjo@anjor@andreypopp
@ivnvxd@andreakeesys@ameramayreh@a1ooha
@maliayas@akirak@adrianlzt@codefromthecrypt
@aweis89@aj47@noitcudni@solatis
@webkonstantin@khulnasoft-bot@KebobZ@acro5piano
@josx@joshvera@jklina@jkeys089
@johanvtsJim White@gengjiawen@jevon
@jesstelford@JeongJuhyeon@jackhallam@Mushoz
@zestysoftHenry Fraser@gwpl@garrett-hopper
@filiptrplan@FelixLisczyk@evnoj@erykwieliczko
@elohmeier@emmanuel-ferdman
+ +The current priorities are to improve core capabilities and user experience of the `cecli` project + 1. **Base Asynchronicity (cecli coroutine-experiment branch)** * [x] Refactor codebase to have the main loop run asynchronously * [x] Update test harness to work with new asynchronous methods From 6996949d6d3ff17721f27f14bf85388b638341d3 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 30 Jul 2026 11:46:41 -0700 Subject: [PATCH 02/30] feat: Implement queue data structure in Commands class --- cecli/commands/core.py | 73 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/cecli/commands/core.py b/cecli/commands/core.py index 29823d6cddd..ae534eef838 100644 --- a/cecli/commands/core.py +++ b/cecli/commands/core.py @@ -1,6 +1,7 @@ import json import re import sys +import time import weakref from pathlib import Path @@ -131,6 +132,78 @@ def __init__( self.cmd_running_event.set() self.last_command_show_notification = True + # Prompt queue for CLI-33: in-memory FIFO queue for deferred prompt processing + self.prompt_queue = [] + self._queue_counter = 0 + + # ── Queue Management Methods (CLI-33) ────────────────────────────── + + def _enqueue_prompt(self, text: str) -> dict: + """Add a prompt to the queue and return the queued item. + + Args: + text: The prompt text to enqueue. + + Returns: + dict with keys: id (str), text (str), timestamp (float). + + Raises: + ValueError: If text is empty, None, or exceeds 10000 characters. + RuntimeError: If the queue is at max capacity (100 items). + """ + if not text or not text.strip(): + raise ValueError("Cannot enqueue empty prompt") + if len(text) > 10000: + raise ValueError("Prompt exceeds maximum length of 10000 characters") + if len(self.prompt_queue) >= 100: + raise RuntimeError("Queue is full (max 100 items)") + + self._queue_counter += 1 + item = { + "id": str(self._queue_counter), + "text": text, + "timestamp": time.time(), + } + self.prompt_queue.append(item) + return item + + def _dequeue_prompt(self) -> dict | None: + """Remove and return the first item from the queue (FIFO). + + Returns: + The dequeued item dict, or None if the queue is empty. + """ + if not self.prompt_queue: + return None + return self.prompt_queue.pop(0) + + def _get_queue_length(self) -> int: + """Return the current number of items in the queue.""" + return len(self.prompt_queue) + + def _remove_from_queue(self, index: int) -> dict | None: + """Remove and return the item at the given index. + + Args: + index: 0-based index of the item to remove. + + Returns: + The removed item dict, or None if the index is out of bounds. + """ + if index < 0 or index >= len(self.prompt_queue): + return None + return self.prompt_queue.pop(index) + + def _clear_queue(self) -> list: + """Remove all items from the queue and return them. + + Returns: + List of all items that were in the queue. + """ + items = list(self.prompt_queue) + self.prompt_queue.clear() + return items + def _load_custom_commands(self, custom_commands): """ Load custom commands from plugin paths. From 52cd12bb805e855855ee09adb6e4b75a70e094a1 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 30 Jul 2026 13:36:37 -0700 Subject: [PATCH 03/30] feat: Implement /queue command --- cecli/commands/__init__.py | 3 ++ cecli/commands/queue.py | 86 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 cecli/commands/queue.py diff --git a/cecli/commands/__init__.py b/cecli/commands/__init__.py index 7671804f33b..0d98af26d94 100644 --- a/cecli/commands/__init__.py +++ b/cecli/commands/__init__.py @@ -52,6 +52,7 @@ from .models import ModelsCommand from .multiline_mode import MultilineModeCommand from .paste import PasteCommand +from .queue import QueueCommand from .quit import QuitCommand from .read_only import ReadOnlyCommand from .read_only_stub import ReadOnlyStubCommand @@ -142,6 +143,7 @@ CommandRegistry.register(ModelsCommand) CommandRegistry.register(MultilineModeCommand) CommandRegistry.register(PasteCommand) +CommandRegistry.register(QueueCommand) CommandRegistry.register(QuitCommand) CommandRegistry.register(ReadOnlyCommand) CommandRegistry.register(ReadOnlyStubCommand) @@ -228,6 +230,7 @@ "parse_quoted_filenames", "PasteCommand", "quote_filename", +"QueueCommand", "QuitCommand", "ReadOnlyCommand", "ReadOnlyStubCommand", diff --git a/cecli/commands/queue.py b/cecli/commands/queue.py new file mode 100644 index 00000000000..36c9d3e4047 --- /dev/null +++ b/cecli/commands/queue.py @@ -0,0 +1,86 @@ +"""Queue command for CLI-33: adds a prompt to the processing queue.""" + +from typing import List + +from cecli.commands.utils.base_command import BaseCommand +from cecli.commands.utils.helpers import format_command_result + + +class QueueCommand(BaseCommand): + NORM_NAME = "queue" + DESCRIPTION = "Queue a prompt for processing after current tasks complete" + + @classmethod + async def execute(cls, io, coder, args, **kwargs): + """Execute the queue command with given parameters. + + Args: + io: InputOutput instance + coder: Coder instance (may be None for some commands) + args: Command arguments as string (the prompt text to queue) + **kwargs: Additional context + + Returns: + Formatted result string + """ + # Sad path: coder.commands is None + if not coder.commands: + return format_command_result( + io, cls.NORM_NAME, + error="Command system not available. Cannot queue prompts." + ) + + # Sad path: no args (empty prompt text) + if not args or not args.strip(): + return format_command_result( + io, cls.NORM_NAME, + "Usage: /queue \n" + "Add a prompt to the queue for processing after current tasks complete." + ) + + prompt_text = args.strip() + + # Sad path: prompt exceeds 10000 characters + if len(prompt_text) > 10000: + return format_command_result( + io, cls.NORM_NAME, + error=f"Prompt exceeds maximum length of 10000 characters " + f"(got {len(prompt_text)})." + ) + + # Happy path: enqueue the prompt + try: + item = coder.commands._enqueue_prompt(prompt_text) + position = len(coder.commands.prompt_queue) + io.tool_output( + f"Prompt queued at position {position} (id: {item['id']})" + ) + return f"Successfully executed {cls.NORM_NAME}." + except ValueError as e: + return format_command_result(io, cls.NORM_NAME, error=str(e)) + except RuntimeError as e: + return format_command_result(io, cls.NORM_NAME, error=str(e)) + + @classmethod + def get_completions(cls, io, coder, args) -> List[str]: + """Get completion options for queue command.""" + return [] + + @classmethod + def get_help(cls) -> str: + """Get help text for the queue command.""" + help_text = super().get_help() + help_text += "\nUsage:\n" + help_text += " /queue # Queue a prompt for later processing\n" + help_text += "\nDescription:\n" + help_text += " Adds a prompt to an in-memory FIFO queue. Queued prompts are\n" + help_text += " processed sequentially after the current command completes.\n" + help_text += "\nConstraints:\n" + help_text += " - Maximum prompt length: 10,000 characters\n" + help_text += " - Maximum queue size: 100 items\n" + help_text += " - Queue is in-memory only (lost on session restart)\n" + help_text += "\nExamples:\n" + help_text += " /queue Review the changes in src/main.py\n" + help_text += " /queue Write unit tests for the new feature\n" + help_text += "\nSee also: /list-queue, /remove-queue\n" + return help_text \ No newline at end of file From 9f3dd885f3a1738750e49795bf36cc34e7b7e218 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 1 Aug 2026 19:20:31 -0700 Subject: [PATCH 04/30] feat: Implement prompt queue feature and update plan --- cecli/commands/list_queue.py | 69 ++++++++++++++++++++ cecli/commands/remove_queue.py | 113 +++++++++++++++++++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 cecli/commands/list_queue.py create mode 100644 cecli/commands/remove_queue.py diff --git a/cecli/commands/list_queue.py b/cecli/commands/list_queue.py new file mode 100644 index 00000000000..72305b09d8c --- /dev/null +++ b/cecli/commands/list_queue.py @@ -0,0 +1,69 @@ +"""List-queue command for CLI-33: displays all prompts in the processing queue.""" + +import datetime +from typing import List + +from cecli.commands.utils.base_command import BaseCommand +from cecli.commands.utils.helpers import format_command_result + + +class ListQueueCommand(BaseCommand): + NORM_NAME = "list-queue" + DESCRIPTION = "List all prompts currently in the queue" + + @classmethod + async def execute(cls, io, coder, args, **kwargs): + """Execute the list-queue command with given parameters. + + Args: + io: InputOutput instance + coder: Coder instance (may be None for some commands) + args: Command arguments (unused for list-queue) + **kwargs: Additional context + + Returns: + Formatted result string + """ + # Sad path: coder.commands is None + if not coder.commands: + return format_command_result( + io, cls.NORM_NAME, error="Command system not available. Cannot list queue." + ) + + queue = coder.commands.prompt_queue + + # Sad path: empty queue + if not queue: + io.tool_output("Queue is empty.") + return f"Successfully executed {cls.NORM_NAME}." + + # Happy path: display numbered list + lines = [] + for i, item in enumerate(queue, start=1): + text = item["text"] + display_text = text[:80] + "..." if len(text) > 80 else text + ts = datetime.datetime.fromtimestamp(item["timestamp"]).strftime("%H:%M:%S") + lines.append(f"[{i}] {display_text} ({ts})") + + io.tool_output("\n".join(lines)) + return f"Successfully executed {cls.NORM_NAME}." + + @classmethod + def get_completions(cls, io, coder, args) -> List[str]: + """Get completion options for list-queue command.""" + return [] + + @classmethod + def get_help(cls) -> str: + """Get help text for the list-queue command.""" + help_text = super().get_help() + help_text += "\nUsage:\n" + help_text += " /list-queue # Display all queued prompts\n" + help_text += "\nDescription:\n" + help_text += " Displays a numbered list of all prompts currently in the queue,\n" + help_text += " showing each prompt's position, text (truncated to 80 chars),\n" + help_text += " and the time it was queued.\n" + help_text += "\nExamples:\n" + help_text += " /list-queue # Shows all queued prompts\n" + help_text += "\nSee also: /queue, /remove-queue\n" + return help_text diff --git a/cecli/commands/remove_queue.py b/cecli/commands/remove_queue.py new file mode 100644 index 00000000000..fc555869fe7 --- /dev/null +++ b/cecli/commands/remove_queue.py @@ -0,0 +1,113 @@ +"""Remove-queue command for CLI-33: removes prompts from the processing queue.""" + +from typing import List + +from cecli.commands.utils.base_command import BaseCommand +from cecli.commands.utils.helpers import format_command_result + + +class RemoveQueueCommand(BaseCommand): + NORM_NAME = "remove-queue" + DESCRIPTION = "Remove a prompt from the queue by index, or '*' to clear all" + + @classmethod + async def execute(cls, io, coder, args, **kwargs): + """Execute the remove-queue command with given parameters. + + Args: + io: InputOutput instance + coder: Coder instance (may be None for some commands) + args: Command arguments (index number, '*', or empty for interactive) + **kwargs: Additional context + + Returns: + Formatted result string + """ + # Sad path: coder.commands is None + if not coder.commands: + return format_command_result( + io, cls.NORM_NAME, error="Command system not available. Cannot remove from queue." + ) + + # Sad path: empty queue + if coder.commands._get_queue_length() == 0: + return format_command_result( + io, cls.NORM_NAME, error="Queue is empty. Nothing to remove." + ) + + # Handle wildcard: clear entire queue + if args and args.strip() == "*": + items = coder.commands._clear_queue() + count = len(items) + io.tool_output(f"Removed all {count} queued prompt(s).") + return f"Successfully executed {cls.NORM_NAME}." + + # Handle specific index + if args and args.strip(): + try: + index = int(args.strip()) - 1 # Convert to 0-based + except ValueError: + return format_command_result( + io, + cls.NORM_NAME, + error=f"Invalid index: '{args.strip()}'. Please provide a number or '*'.", + ) + + item = coder.commands._remove_from_queue(index) + if item is None: + queue_len = coder.commands._get_queue_length() + return format_command_result( + io, + cls.NORM_NAME, + error=f"Index {args.strip()} is out of range. Queue has {queue_len} item(s).", + ) + + io.tool_output(f"Removed: {item['text'][:80]}") + return f"Successfully executed {cls.NORM_NAME}." + + # Interactive mode: no args provided + queue = coder.commands.prompt_queue + io.tool_output("Queued prompts:") + for i, item in enumerate(queue, 1): + text = item["text"][:80] + if len(item["text"]) > 80: + text += "..." + io.tool_output(f" [{i}] {text}") + + io.tool_output("\nEnter index to remove, '*' to clear all, or press Enter to cancel:") + # In non-interactive mode, just show usage + return format_command_result( + io, cls.NORM_NAME, "Usage: /remove-queue | /remove-queue *" + ) + + @classmethod + def get_completions(cls, io, coder, args) -> List[str]: + """Get completion options for remove-queue command.""" + if not coder.commands: + return [] + + queue_len = coder.commands._get_queue_length() + completions = [str(i) for i in range(1, queue_len + 1)] + completions.append("*") + return completions + + @classmethod + def get_help(cls) -> str: + """Get help text for the remove-queue command.""" + help_text = super().get_help() + help_text += "\nUsage:\n" + help_text += " /remove-queue # Remove prompt at given index\n" + help_text += " /remove-queue * # Clear the entire queue\n" + help_text += " /remove-queue # Interactive mode (shows list, prompts for index)\n" + help_text += "\nDescription:\n" + help_text += " Removes a prompt from the in-memory queue by its index number.\n" + help_text += " Use '*' to clear all queued prompts at once.\n" + help_text += ( + " Without arguments, displays the queue and prompts for interactive selection.\n" + ) + help_text += "\nExamples:\n" + help_text += " /remove-queue 1 # Remove the first queued prompt\n" + help_text += " /remove-queue * # Clear the entire queue\n" + help_text += " /remove-queue # Interactive mode\n" + help_text += "\nSee also: /queue, /list-queue\n" + return help_text From 4e0b519db0dc3ed882c0c727ac254290c14740ca Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 1 Aug 2026 19:25:33 -0700 Subject: [PATCH 05/30] feat: Implement prompt queue functionality --- cecli/commands/__init__.py | 7 +++++- cecli/commands/core.py | 47 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/cecli/commands/__init__.py b/cecli/commands/__init__.py index 0d98af26d94..c9d7ccbddb8 100644 --- a/cecli/commands/__init__.py +++ b/cecli/commands/__init__.py @@ -37,6 +37,7 @@ from .include_skill import IncludeSkillCommand from .lint import LintCommand from .list_mcp import ListMcpCommand +from .list_queue import ListQueueCommand from .list_sessions import ListSessionsCommand from .list_skills import ListSkillsCommand from .load import LoadCommand @@ -60,6 +61,7 @@ from .reasoning_effort import ReasoningEffortCommand from .remove_hook import RemoveHookCommand from .remove_mcp import RemoveMcpCommand +from .remove_queue import RemoveQueueCommand from .remove_skill import RemoveSkillCommand from .report import ReportCommand from .reset import ResetCommand @@ -128,6 +130,7 @@ CommandRegistry.register(IncludeSkillCommand) CommandRegistry.register(LintCommand) CommandRegistry.register(ListMcpCommand) +CommandRegistry.register(ListQueueCommand) CommandRegistry.register(ListSessionsCommand) CommandRegistry.register(ListSkillsCommand) CommandRegistry.register(LoadCommand) @@ -150,6 +153,7 @@ CommandRegistry.register(ReasoningEffortCommand) CommandRegistry.register(RemoveHookCommand) CommandRegistry.register(RemoveMcpCommand) +CommandRegistry.register(RemoveQueueCommand) CommandRegistry.register(RemoveSkillCommand) CommandRegistry.register(ReportCommand) CommandRegistry.register(ResetCommand) @@ -230,7 +234,7 @@ "parse_quoted_filenames", "PasteCommand", "quote_filename", -"QueueCommand", + "QueueCommand", "QuitCommand", "ReadOnlyCommand", "ReadOnlyStubCommand", @@ -238,6 +242,7 @@ "ReloadProgramSignal", "RemoveHookCommand", "RemoveMcpCommand", + "RemoveQueueCommand", "RemoveSkillCommand", "ReportCommand", "ResetCommand", diff --git a/cecli/commands/core.py b/cecli/commands/core.py index ae534eef838..ccdbfdc18d6 100644 --- a/cecli/commands/core.py +++ b/cecli/commands/core.py @@ -1,3 +1,4 @@ +import asyncio import json import re import sys @@ -135,6 +136,11 @@ def __init__( # Prompt queue for CLI-33: in-memory FIFO queue for deferred prompt processing self.prompt_queue = [] self._queue_counter = 0 + self._queue_lock = asyncio.Lock() + self._processing_queue = False + + # Commands that should NOT trigger auto-processing of the queue + self._MANAGEMENT_COMMANDS = {"queue", "list-queue", "remove-queue"} # ── Queue Management Methods (CLI-33) ────────────────────────────── @@ -204,6 +210,40 @@ def _clear_queue(self) -> list: self.prompt_queue.clear() return items + async def _process_queued_prompts(self): + """Process all prompts currently in the queue sequentially. + + This method is called from the finally block of execute() after + cmd_running_event is set, ensuring the system is idle before + processing queued prompts. Management commands (queue, list-queue, + remove-queue) are excluded from triggering this method. + + Uses _processing_queue flag to prevent re-entrant processing + (e.g., if a queued prompt itself queues another prompt). + """ + self._processing_queue = True + try: + while self.prompt_queue: + item = self._dequeue_prompt() + if not item: + break + if self.io: + self.io.tool_output(f"Processing queued prompt (id: {item['id']})...") + try: + await self.run(item["text"]) + except SwitchCoderSignal: + raise + except ReloadProgramSignal: + raise + except Exception as e: + if self.io: + self.io.tool_error( + f"Error processing queued prompt (id: {item['id']}): {e}" + ) + continue + finally: + self._processing_queue = False + def _load_custom_commands(self, custom_commands): """ Load custom commands from plugin paths. @@ -327,6 +367,13 @@ async def execute(self, cmd_name, args, coder=None, **kwargs): self.cmd_running_event.set() if self.coder.tui and self.coder.tui(): self.coder.tui().refresh() + # Queue processing integration: auto-process queued prompts when system is idle + if ( + self.prompt_queue + and cmd_name not in self._MANAGEMENT_COMMANDS + and not self._processing_queue + ): + await self._process_queued_prompts() def matching_commands(self, inp): words = inp.strip().split() From 92a09c5176f060ce9101ff7d01e5baee7702392b Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 2 Aug 2026 00:17:58 -0700 Subject: [PATCH 06/30] style: fix formatting in queue.py (CLI-33) --- cecli/commands/queue.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/cecli/commands/queue.py b/cecli/commands/queue.py index 36c9d3e4047..7438ff42326 100644 --- a/cecli/commands/queue.py +++ b/cecli/commands/queue.py @@ -26,16 +26,16 @@ async def execute(cls, io, coder, args, **kwargs): # Sad path: coder.commands is None if not coder.commands: return format_command_result( - io, cls.NORM_NAME, - error="Command system not available. Cannot queue prompts." + io, cls.NORM_NAME, error="Command system not available. Cannot queue prompts." ) # Sad path: no args (empty prompt text) if not args or not args.strip(): return format_command_result( - io, cls.NORM_NAME, + io, + cls.NORM_NAME, "Usage: /queue \n" - "Add a prompt to the queue for processing after current tasks complete." + "Add a prompt to the queue for processing after current tasks complete.", ) prompt_text = args.strip() @@ -43,18 +43,17 @@ async def execute(cls, io, coder, args, **kwargs): # Sad path: prompt exceeds 10000 characters if len(prompt_text) > 10000: return format_command_result( - io, cls.NORM_NAME, + io, + cls.NORM_NAME, error=f"Prompt exceeds maximum length of 10000 characters " - f"(got {len(prompt_text)})." + f"(got {len(prompt_text)}).", ) # Happy path: enqueue the prompt try: item = coder.commands._enqueue_prompt(prompt_text) position = len(coder.commands.prompt_queue) - io.tool_output( - f"Prompt queued at position {position} (id: {item['id']})" - ) + io.tool_output(f"Prompt queued at position {position} (id: {item['id']})") return f"Successfully executed {cls.NORM_NAME}." except ValueError as e: return format_command_result(io, cls.NORM_NAME, error=str(e)) @@ -83,4 +82,4 @@ def get_help(cls) -> str: help_text += " /queue Review the changes in src/main.py\n" help_text += " /queue Write unit tests for the new feature\n" help_text += "\nSee also: /list-queue, /remove-queue\n" - return help_text \ No newline at end of file + return help_text From 939186bb05b6275ac1e11b7061cab8a679f820a3 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 2 Aug 2026 00:27:08 -0700 Subject: [PATCH 07/30] fix: Improve management command handling and queue processing --- cecli/commands/core.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cecli/commands/core.py b/cecli/commands/core.py index ccdbfdc18d6..f28226e8452 100644 --- a/cecli/commands/core.py +++ b/cecli/commands/core.py @@ -336,7 +336,8 @@ async def execute(self, cmd_name, args, coder=None, **kwargs): return self.last_command_show_notification = command_class.show_completion_notification - self.cmd_running_event.clear() + if cmd_name not in self._MANAGEMENT_COMMANDS: + self.cmd_running_event.clear() try: kwargs.update( @@ -386,6 +387,12 @@ def matching_commands(self, inp): return matching_commands, first_word, rest_inp async def run(self, inp, coder=None, **kwargs): + if inp.startswith("/"): + words = inp.strip().split() + cmd_name = words[0][1:] + rest_inp = inp[len(words[0]) :].strip() + return await self.execute(cmd_name, rest_inp, coder=coder, **kwargs) + if inp.startswith("!!!"): return await self.execute( "run", inp[3:], coder=coder, background=True, suppress_add=True From ca3b48924b1c352d65addc10c8d25b11eda98b69 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 2 Aug 2026 00:34:21 -0700 Subject: [PATCH 08/30] feat: Implement prompt queue feature (CLI-33) --- README.md | 73 ++++++++++++++++++------------ cecli/tests/test_queue_commands.py | 0 2 files changed, 43 insertions(+), 30 deletions(-) create mode 100644 cecli/tests/test_queue_commands.py diff --git a/README.md b/README.md index 9a2f7e0fdf0..f8fdfc7b73d 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ ### Links -[Documentation](https://cecli.dev/docs/) 🞄 -[Discord Chat](https://discord.gg/AX9ZEA7nJn) 🞄 +[Documentation](https://cecli.dev/docs/) +[Discord Chat](https://discord.gg/AX9ZEA7nJn) [Issue Queue](https://github.com/cecli-dev/cecli/issues) @@ -52,52 +52,65 @@ uv tool install --native-tls --python python3.12 cecli-dev Use the tool installation so cecli doesn't interfere with your development environment -## Prompt Queueing Feature +## Prompt Queue Management -The cecli application now includes a prompt queueing feature that allows users to manage multiple prompts in a first-in-first-out (FIFO) queue with a configurable maximum size. +The `cecli` CLI includes a prompt queueing feature (`CLI-33`) that allows users to manage multiple prompts in a first-in-first-out (FIFO) in-memory queue. The queue is tied to the user's CLI session and does not persist across restarts. -### How It Works +### Queue Lifecycle -When the prompt queue is enabled, incoming prompts are added to a queue instead of being processed immediately. The queue has a default maximum size of 5 prompts. When the queue is full and a new prompt is added, the oldest prompt is automatically removed to make space for the new one. +1. **Enqueue**: `/queue ` adds a prompt to the queue. +2. **List**: `/list-queue` displays all queued prompts with index numbers and timestamps. +3. **Remove**: `/remove-queue [index|*]` removes a specific item by index, clears the entire queue with `*`, or provides interactive selection when called with no arguments. +4. **Auto-Process**: After the current command completes and the system is idle (`cmd_running_event` is set), queued prompts are processed sequentially in FIFO order. -### Configuration +### Commands -The maximum queue size can be configured using the `max_queue_size` parameter in your cecli configuration: +- `/queue ` — Adds a prompt to the queue. Confirms with the queue position. +- `/list-queue` — Displays a numbered list of queued prompts (`[index] text (timestamp)`). Shows "Queue is empty" when appropriate. +- `/remove-queue ` — Removes the prompt at the given 0-based index. +- `/remove-queue *` — Clears the entire queue. +- `/remove-queue` (no args) — Enters interactive selection mode. -```bash -cecli --max_queue_size 10 -``` +### Queue Limits -Or in your `.cecli.conf.yml` file: +- **Max Queue Size**: 100 items (hard limit). New prompts are rejected with a warning when the queue is full. +- **Max Prompt Length**: 10,000 characters per queued item. Prompts exceeding this limit are rejected. +- **In-Memory Only**: The queue is stored on the `Commands` instance (`cecli/commands/core.py`) and is lost when the CLI session restarts. -```yaml -max_queue_size: 10 -``` +### Queue Processing Integration -### Usage +Queue processing is triggered in the `finally` block of `Commands.execute()` after `cmd_running_event.set()`. Management commands (`queue`, `list-queue`, `remove-queue`) are excluded from triggering auto-processing to prevent unexpected behavior. A `_processing_queue` boolean flag prevents infinite recursion if a queued prompt itself queues another item. -1. **Queue Management**: Prompts are automatically queued when the queue is enabled -2. **View Queue**: The TUI interface displays the current queue of prompts -3. **Remove from Queue**: Use the `/queue-remove` command to remove specific prompts from the queue +### Example Workflow -### /queue-remove Command +```bash +# Queue multiple prompts +/queue "refactor database layer" +/queue "add unit tests for user service" -The `/queue-remove` command allows you to remove specific prompts from the queue: +# View queued prompts +/list-queue +# Output: [1] refactor database layer (2026-08-01 10:30:00) +# [2] add unit tests for user service (2026-08-01 10:30:05) -```bash -/queue-remove 3 +# Remove a specific queued prompt +/remove-queue 1 + +# Clear the entire queue +/remove-queue * + +# Queued prompts process automatically after the current command completes ``` -This command removes the prompt at index 3 from the queue (0-based indexing). Tab completion is available for prompt IDs. +### Management Command Guard + +The management commands (`/queue`, `/list-queue`, `/remove-queue`) must not trigger auto-processing of queued items. Their execution is isolated so that the current prompt continues uninterrupted. When any of these commands is entered while another prompt is being processed, they execute immediately without clearing `cmd_running_event` or steering the ongoing command. -### Benefits +### Thread Safety -- Prevents overwhelming the system with too many concurrent prompts -- Allows users to review and manage their prompt queue -- Provides better control over prompt processing order -- Automatically handles overflow by removing oldest prompts +The queue uses a single-threaded async event loop. List operations on `prompt_queue` are naturally safe within the async loop. An `asyncio.Lock` (`_queue_lock`) protects all read and write operations to ensure atomic updates. -The prompt queueing feature enhances the user experience by providing better control over prompt processing and preventing system overload. +The prompt queueing feature enhances the user experience by providing robust prompt management capabilities, increasing efficiency, and preventing interruptions during ongoing tasks. The documentation above contains the full set of allowed configuration options but I highly recommend using an `.cecli.conf.yml` file. A good place to get started is: diff --git a/cecli/tests/test_queue_commands.py b/cecli/tests/test_queue_commands.py new file mode 100644 index 00000000000..e69de29bb2d From 876b1a1c32d4853c7d43322b5b95b07791848708 Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 10 Aug 2026 21:41:47 +0200 Subject: [PATCH 09/30] fix: deep-merge CLI agent-config with config file values --- cecli/main.py | 30 ++++++++ tests/basic/test_agent_config_merge.py | 94 ++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 tests/basic/test_agent_config_merge.py diff --git a/cecli/main.py b/cecli/main.py index 8e76c2ea411..e2861aab959 100644 --- a/cecli/main.py +++ b/cecli/main.py @@ -105,6 +105,31 @@ def convert_yaml_to_json_string(value): return value +def merge_agent_config(cli_agent_config: str, file_agent_config) -> str: + """ + Deep-merge the config-file agent-config into the CLI agent-config so CLI + values override individual keys while keys provided only by the config + files (e.g. skills_paths, skills_init) are preserved instead of being + discarded wholesale when --agent-config is passed on the CLI. + + configargparse discards config-file values for options that are also given + on the command line, so without this merge a CLI --agent-config silently + drops every agent-config key that lives only in .cecli.conf.yml. + """ + try: + from cecli.helpers.config_utils import deep_merge + + file_ac = file_agent_config + if isinstance(file_ac, str): + file_ac = json.loads(file_ac) + cli_ac = json.loads(cli_agent_config) + if isinstance(file_ac, dict) and file_ac and isinstance(cli_ac, dict): + return json.dumps(deep_merge(file_ac, cli_ac, deep_merge_arrays=False)) + except Exception: + pass + return cli_agent_config + + def check_config_files_for_yes(config_files): from cecli.decoding import safe_open @@ -731,6 +756,11 @@ async def main_async( if hasattr(args, "agent_config") and args.agent_config is not None: args.agent_config = convert_yaml_to_json_string(args.agent_config) + # CLI --agent-config should deep-merge with (not replace) the + # agent-config from the merged config files so file-only keys + # (e.g. skills_paths, skills_init) are preserved. + file_agent_config = merged_config.get("agent-config") or merged_config.get("agent_config") + args.agent_config = merge_agent_config(args.agent_config, file_agent_config) if hasattr(args, "tui_config") and args.tui_config is not None: args.tui_config = convert_yaml_to_json_string(args.tui_config) if hasattr(args, "mcp_servers") and args.mcp_servers is not None: diff --git a/tests/basic/test_agent_config_merge.py b/tests/basic/test_agent_config_merge.py new file mode 100644 index 00000000000..8b9e3d9b58d --- /dev/null +++ b/tests/basic/test_agent_config_merge.py @@ -0,0 +1,94 @@ +import json +import os +import tempfile + +import yaml + +from cecli.args import get_parser +from cecli.helpers import config_utils +from cecli.main import convert_yaml_to_json_string, merge_agent_config + + +def test_cli_agent_config_merges_with_config_file(): + """CLI --agent-config must deep-merge with the config-file agent-config + instead of replacing it wholesale (regression: file-only keys dropped).""" + file_ac = { + "command_timeout": 0, + "skip_cli_confirmations": True, + "tools_paths": ["/tmp/mytools"], + "skills_paths": ["/tmp/skills"], + } + cli = json.dumps({"command_timeout": 120}) + merged = json.loads(merge_agent_config(cli, file_ac)) + assert merged["command_timeout"] == 120 # CLI wins per-key + assert merged["skip_cli_confirmations"] is True # file-only key preserved + assert merged["tools_paths"] == ["/tmp/mytools"] # file-only key preserved + assert merged["skills_paths"] == ["/tmp/skills"] # file-only key preserved + + +def test_file_agent_config_given_as_json_string(): + """Merged config from read_and_merge_all_configs holds agent-config as a + JSON string (YAML block scalar) - the helper must handle that form.""" + file_ac = '{"command_timeout": 0, "skills_paths": ["/tmp/skills"]}' + merged = json.loads(merge_agent_config('{"skip_cli_confirmations": true}', file_ac)) + assert merged["skip_cli_confirmations"] is True + assert merged["command_timeout"] == 0 + assert merged["skills_paths"] == ["/tmp/skills"] + + +def test_no_file_agent_config_returns_cli_unchanged(): + """Without a config-file agent-config the merge must be a no-op.""" + cli = '{"command_timeout": 120}' + assert merge_agent_config(cli, None) == cli + assert merge_agent_config(cli, {}) == cli + assert merge_agent_config(cli, "not json") == cli + + +def test_nested_dict_keys_deep_merged(): + """Nested dicts under agent-config are merged recursively, CLI wins.""" + file_ac = {"nested": {"a": 1, "b": 2}} + merged = json.loads(merge_agent_config('{"nested": {"b": 9, "c": 3}}', file_ac)) + assert merged["nested"] == {"a": 1, "b": 9, "c": 3} + + +def test_cli_array_replaces_file_array(): + """Arrays are not deep-merged: CLI list values win wholesale.""" + file_ac = {"skills_paths": ["/tmp/file-skills"]} + merged = json.loads(merge_agent_config('{"skills_paths": ["/tmp/cli-skills"]}', file_ac)) + assert merged["skills_paths"] == ["/tmp/cli-skills"] + + +def test_main_async_pipeline_preserves_file_keys(tmp_path): + """Replicates main_async: merge config files -> temp yaml -> parser -> + CLI parse -> temp file deleted -> convert + merge against merged_config. + + Guards against regressing to the broken baseline (re-parsing argv after + the temp config file was already unlinked yields no config-file values). + """ + conf = tmp_path / ".cecli.conf.yml" + conf.write_text( + "agent-config: |\n" + ' {"skills_paths": ["./.cecli/skills"], "skills_init": ["android-cli"]}\n' + ) + paths = [str(conf)] + merged_config = config_utils.read_and_merge_all_configs(paths, [], paths) + + fd, tmp = tempfile.mkstemp(suffix=".yml", prefix="cecli_merged_") + os.close(fd) + with open(tmp, "w") as f: + yaml.dump(merged_config, f) + try: + parser = get_parser([tmp], None) + argv = ['--agent-config={"command_timeout": 0}'] + args, _ = parser.parse_known_args(argv) + finally: + os.unlink(tmp) # main_async deletes the temp file before the merge point + + args.agent_config = convert_yaml_to_json_string(args.agent_config) + file_ac = merged_config.get("agent-config") + args.agent_config = merge_agent_config(args.agent_config, file_ac) + + merged = json.loads(args.agent_config) + assert merged["command_timeout"] == 0 + assert merged["skills_paths"] == ["./.cecli/skills"] + assert merged["skills_init"] == ["android-cli"] From 8c9ac2f3b44e6c18f8c710eb9a91ca4574d95ea8 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 10 Aug 2026 21:02:42 -0400 Subject: [PATCH 10/30] Draft 1 of LLM Reduction/LiteLLM Reduction, Support for: /completions (and the openai spec) /responses (statelessly) /messages (for anthropic) and whatever gemini's is actually called --- cecli/coders/base_coder.py | 49 +- cecli/coders/copypaste_coder.py | 10 +- cecli/coders/editblock_func_coder.py | 6 +- cecli/coders/wholefile_func_coder.py | 6 +- cecli/exceptions.py | 4 +- cecli/helpers/leak_detect.py | 74 +- cecli/helpers/llms/__init__.py | 35 + cecli/helpers/llms/config.py | 128 ++ cecli/helpers/llms/domains/__init__.py | 25 + cecli/helpers/llms/domains/chat.py | 293 +++++ cecli/helpers/llms/domains/gemini.py | 634 ++++++++++ cecli/helpers/llms/domains/messages.py | 535 ++++++++ cecli/helpers/llms/domains/responses.py | 467 +++++++ cecli/helpers/llms/formatters/__init__.py | 38 + cecli/helpers/llms/formatters/reasoning.py | 91 ++ cecli/helpers/llms/formatters/thinking.py | 76 ++ cecli/helpers/llms/identifiers.py | 93 ++ cecli/helpers/llms/litellm_compat.py | 1104 +++++++++++++++++ cecli/helpers/llms/pipeline.py | 129 ++ cecli/helpers/llms/providers/__init__.py | 48 + cecli/helpers/llms/providers/anthropic.py | 20 + cecli/helpers/llms/providers/base.py | 66 + cecli/helpers/llms/providers/deepseek.py | 20 + cecli/helpers/llms/providers/gemini.py | 37 + .../helpers/llms/providers/github_copilot.py | 228 ++++ cecli/helpers/llms/providers/meta.py | 22 + cecli/helpers/llms/providers/openai.py | 20 + cecli/helpers/llms/providers/openrouter.py | 21 + cecli/helpers/llms/runtime.py | 43 + cecli/helpers/llms/types.py | 209 ++++ cecli/helpers/llms/utils.py | 76 ++ cecli/helpers/model_providers.py | 237 +--- cecli/helpers/responses.py | 19 +- cecli/llm.py | 150 +-- cecli/main.py | 19 +- cecli/models.py | 5 +- requirements.txt | 147 +-- requirements/common-constraints.txt | 5 +- requirements/requirements.in | 18 +- tests/basic/test_exceptions.py | 11 +- tests/basic/test_main.py | 6 +- tests/basic/test_reasoning.py | 6 +- tests/basic/test_sendchat.py | 12 +- tests/coders/test_copypaste_coder.py | 5 + tests/coders/test_tool_call_consolidation.py | 21 +- 45 files changed, 4703 insertions(+), 565 deletions(-) create mode 100644 cecli/helpers/llms/__init__.py create mode 100644 cecli/helpers/llms/config.py create mode 100644 cecli/helpers/llms/domains/__init__.py create mode 100644 cecli/helpers/llms/domains/chat.py create mode 100644 cecli/helpers/llms/domains/gemini.py create mode 100644 cecli/helpers/llms/domains/messages.py create mode 100644 cecli/helpers/llms/domains/responses.py create mode 100644 cecli/helpers/llms/formatters/__init__.py create mode 100644 cecli/helpers/llms/formatters/reasoning.py create mode 100644 cecli/helpers/llms/formatters/thinking.py create mode 100644 cecli/helpers/llms/identifiers.py create mode 100644 cecli/helpers/llms/litellm_compat.py create mode 100644 cecli/helpers/llms/pipeline.py create mode 100644 cecli/helpers/llms/providers/__init__.py create mode 100644 cecli/helpers/llms/providers/anthropic.py create mode 100644 cecli/helpers/llms/providers/base.py create mode 100644 cecli/helpers/llms/providers/deepseek.py create mode 100644 cecli/helpers/llms/providers/gemini.py create mode 100644 cecli/helpers/llms/providers/github_copilot.py create mode 100644 cecli/helpers/llms/providers/meta.py create mode 100644 cecli/helpers/llms/providers/openai.py create mode 100644 cecli/helpers/llms/providers/openrouter.py create mode 100644 cecli/helpers/llms/runtime.py create mode 100644 cecli/helpers/llms/types.py create mode 100644 cecli/helpers/llms/utils.py diff --git a/cecli/coders/base_coder.py b/cecli/coders/base_coder.py index 3a854757446..25b1bd4c792 100755 --- a/cecli/coders/base_coder.py +++ b/cecli/coders/base_coder.py @@ -3491,9 +3491,9 @@ async def add_assistant_reply_to_cur_messages(self): msg = response_dict["choices"][0]["message"] if self.partial_response_tool_calls: - msg["tool_calls"] = self.partial_response_tool_calls + msg["tool_calls"] = [_tool_call_to_dict(tc) for tc in self.partial_response_tool_calls] elif self.partial_response_function_call: - msg["function_call"] = self.partial_response_function_call + msg["function_call"] = _function_call_to_dict(self.partial_response_function_call) if "reasoning_content" not in msg: msg["reasoning_content"] = self.partial_response_reasoning_content @@ -3611,7 +3611,7 @@ async def check_for_file_mentions(self, content): return prompts.added_files.format(fnames=", ".join(added_fnames)) async def send(self, messages, model=None, functions=None, tools=None): - from litellm.types.utils import ModelResponse + ModelResponse = litellm.types.utils.ModelResponse self.interrupt_event.clear() self.got_reasoning_content = False @@ -3745,7 +3745,7 @@ async def send(self, messages, model=None, functions=None, tools=None): self.io.ai_output(json.dumps(args, indent=4)) async def show_send_output(self, completion): - from litellm.types.utils import ModelResponse + ModelResponse = litellm.types.utils.ModelResponse if self.verbose: print(completion) @@ -4008,6 +4008,15 @@ def consolidate_chunks(self): for key, value in psf.items(): if isinstance(value, list): message_provider_specific_fields.setdefault(key, []).extend(value) + elif ( + key in message_provider_specific_fields + and isinstance(message_provider_specific_fields[key], dict) + and isinstance(value, dict) + ): + # Merge dict-valued metadata (e.g. gemini per-call + # function_call_signatures) so parallel tool calls + # each keep their own signature across chunks. + message_provider_specific_fields[key].update(value) elif value is not None: message_provider_specific_fields[key] = value except (AttributeError, IndexError): @@ -4101,7 +4110,8 @@ def _build_tool_calls_from_chunks(self): parallel call is preserved, correctly ordered, and keeps its provider-specific fields (e.g. thought signatures) attached. """ - from litellm.types.utils import ChatCompletionMessageToolCall, Function + ChatCompletionMessageToolCall = litellm.types.utils.ChatCompletionMessageToolCall + Function = litellm.types.utils.Function tool_calls_dict = {} @@ -4632,7 +4642,12 @@ async def apply_updates(self): def parse_partial_args(self): # dump(self.partial_response_function_call) - data = self.partial_response_function_call.get("arguments") + function_call = self.partial_response_function_call + if isinstance(function_call, dict): + data = function_call.get("arguments") + else: + data = getattr(function_call, "arguments", None) + if not data: return @@ -4910,3 +4925,25 @@ def format_command_with_prefix(self, command): else: # Append the command to the prefix with a space return f"{command_prefix} {command}" + + +def _tool_call_to_dict(tc): + """Normalize a tool call (dict or litellm-shaped object) to a wire-format dict.""" + if isinstance(tc, dict): + return tc + + if hasattr(tc, "to_dict"): + return tc.to_dict() + + return tc + + +def _function_call_to_dict(function_call): + """Normalize a function call (dict or litellm-shaped Function) to a dict.""" + if isinstance(function_call, dict): + return function_call + + if hasattr(function_call, "to_dict"): + return function_call.to_dict() + + return function_call diff --git a/cecli/coders/copypaste_coder.py b/cecli/coders/copypaste_coder.py index 5df361291d5..fff77e6637a 100644 --- a/cecli/coders/copypaste_coder.py +++ b/cecli/coders/copypaste_coder.py @@ -259,11 +259,11 @@ def _safe_token_count(text): ], created=int(time.time()), model=model.name, - usage={ - "prompt_tokens": prompt_tokens, - "completion_tokens": completion_tokens, - "total_tokens": total_tokens, - }, + usage=litellm.Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + ), ) kwargs = dict(model=model.name, messages=messages, stream=False) diff --git a/cecli/coders/editblock_func_coder.py b/cecli/coders/editblock_func_coder.py index c0f463d5bb3..57b56ac5883 100644 --- a/cecli/coders/editblock_func_coder.py +++ b/cecli/coders/editblock_func_coder.py @@ -92,7 +92,11 @@ def render_incremental_response(self, final=False): return res async def _update_files(self): - name = self.partial_response_function_call.get("name") + function_call = self.partial_response_function_call + if isinstance(function_call, dict): + name = function_call.get("name") + else: + name = getattr(function_call, "name", None) if name and name != "replace_lines": raise ValueError(f'Unknown function_call name="{name}", use name="replace_lines"') diff --git a/cecli/coders/wholefile_func_coder.py b/cecli/coders/wholefile_func_coder.py index 1c53188ac33..f543984ab77 100644 --- a/cecli/coders/wholefile_func_coder.py +++ b/cecli/coders/wholefile_func_coder.py @@ -113,7 +113,11 @@ def live_diffs(self, fname, content, final): return "\n".join(show_diff) async def _update_files(self): - name = self.partial_response_function_call.get("name") + function_call = self.partial_response_function_call + if isinstance(function_call, dict): + name = function_call.get("name") + else: + name = getattr(function_call, "name", None) if name and name != "write_file": raise ValueError(f'Unknown function_call name="{name}", use name="write_file"') diff --git a/cecli/exceptions.py b/cecli/exceptions.py index df067f684e7..f3c6d5b27f1 100644 --- a/cecli/exceptions.py +++ b/cecli/exceptions.py @@ -62,7 +62,7 @@ def __init__(self): self._load() def _load(self, strict=False): - import litellm + from cecli.llm import litellm for var in dir(litellm): if var.endswith("Error"): @@ -83,7 +83,7 @@ def exceptions_tuple(self): def get_ex_info(self, ex): """Return the ExInfo for a given exception instance""" - import litellm + from cecli.llm import litellm if ex.__class__ is litellm.APIConnectionError: if "google.auth" in str(ex): diff --git a/cecli/helpers/leak_detect.py b/cecli/helpers/leak_detect.py index 320bf217eb8..ed020fa7d5d 100644 --- a/cecli/helpers/leak_detect.py +++ b/cecli/helpers/leak_detect.py @@ -275,61 +275,81 @@ def guppy_summary(self) -> Optional[str]: # ── Report ── - def print_report(self) -> None: - """Print a human-readable memory report to stdout.""" - print("=" * 70) - print("MemorySnapshot Report") - print(f" Objects scanned : {len(self._all_objects):,}") + def print_report(self, out: Optional[Any] = None) -> None: + """Print a human-readable memory report to *out* (defaults to sys.stdout). + + Passing a file-like object (e.g. ``io.StringIO``) allows callers to + capture the report as text instead of writing to the terminal. + """ + if out is None: + out = sys.stdout + + print("=" * 70, file=out) + print("MemorySnapshot Report", file=out) + print(f" Objects scanned : {len(self._all_objects):,}", file=out) print( - f" Deep sizes : {'yes (pympler)' if self._has_pympler else 'no (sys.getsizeof)'}" + f" Deep sizes : {'yes (pympler)' if self._has_pympler else 'no (sys.getsizeof)'}", + file=out, ) - print(f" Guppy available : {'yes' if self._has_guppy else 'no'}") - print("=" * 70) + print(f" Guppy available : {'yes' if self._has_guppy else 'no'}", file=out) + print("=" * 70, file=out) # 1. Largest objects - print("\n>> TOP 15 LARGEST OBJECTS (any type)") - print(" " + "-" * 60) + print("\n>> TOP 15 LARGEST OBJECTS (any type)", file=out) + print(" " + "-" * 60, file=out) for item in self.largest_objects(15): - print(f" {item.size_kb:>10.1f} KB {item.type_name:<15s} {item.repr_str}") + print( + f" {item.size_kb:>10.1f} KB {item.type_name:<15s} {item.repr_str}", + file=out, + ) # 2. Largest dicts - print("\n>> LARGEST DICTS") - print(" " + "-" * 60) + print("\n>> LARGEST DICTS", file=out) + print(" " + "-" * 60, file=out) for item in self.largest_dicts(5): d = item.obj keys_preview = list(d.keys())[:5] if isinstance(d, dict) else [] - print(f" {item.size_kb:>10.1f} KB dict[{len(d)} keys] keys={keys_preview!r}") + print( + f" {item.size_kb:>10.1f} KB dict[{len(d)} keys] keys={keys_preview!r}", + file=out, + ) # 3. Largest lists - print("\n>> LARGEST LISTS") - print(" " + "-" * 60) + print("\n>> LARGEST LISTS", file=out) + print(" " + "-" * 60, file=out) for item in self.largest_lists(5): lst = item.obj first = lst[0] if lst else "empty" - print(f" {item.size_kb:>10.1f} KB list[{len(lst)} items] first={first!r}") + print( + f" {item.size_kb:>10.1f} KB list[{len(lst)} items] first={first!r}", + file=out, + ) # 4. Largest class instances - print("\n>> LARGEST CLASS INSTANCES (custom)") - print(" " + "-" * 60) + print("\n>> LARGEST CLASS INSTANCES (custom)", file=out) + print(" " + "-" * 60, file=out) for item in self.largest_class_instances(n=10): - print(f" {item.size_kb:>10.1f} KB {item.type_name}") + print(f" {item.size_kb:>10.1f} KB {item.type_name}", file=out) # 5. Type summary - print("\n>> TYPE SUMMARY (total size per type)") - print(" " + "-" * 60) + print("\n>> TYPE SUMMARY (total size per type)", file=out) + print(" " + "-" * 60, file=out) for ts in self.type_summary(12): - print(f" {ts.total_size_kb:>10.1f} KB ({ts.count:>7,} objs) {ts.type_name}") + print( + f" {ts.total_size_kb:>10.1f} KB ({ts.count:>7,} objs) {ts.type_name}", + file=out, + ) # 6. Guppy summary if available if self._has_guppy: - print("\n>> GUPPY HEAP SUMMARY") - print(" " + "-" * 60) + print("\n>> GUPPY HEAP SUMMARY", file=out) + print(" " + "-" * 60, file=out) summary = self.guppy_summary() if summary: for line in summary.split("\n"): - print(f" {line}") + print(f" {line}", file=out) - print() + print(file=out) # ── Internal helpers ── diff --git a/cecli/helpers/llms/__init__.py b/cecli/helpers/llms/__init__.py new file mode 100644 index 00000000000..802a92df6ee --- /dev/null +++ b/cecli/helpers/llms/__init__.py @@ -0,0 +1,35 @@ +"""LiteLLM-free LLM communication stack for cecli. + +The ``cecli.helpers.llms`` package replaces the litellm-backed call path with a +small, lazy-loading dispatcher that mirrors the structure of +``cecli/helpers/model_config/``: + +- :mod:`cecli.helpers.llms.config` - provider defaults + model config resolution +- :mod:`cecli.helpers.llms.domains` - one module per API family (chat, + responses, messages, gemini) +- :mod:`cecli.helpers.llms.providers` - per-provider custom logic (auth, + headers, response repair) with an extensible :class:`ProviderAdapter` base +- :mod:`cecli.helpers.llms.formatters` - sectional per-domain formatters + (reasoning, thinking) +- :mod:`cecli.helpers.llms.pipeline` - the ``acompletion()`` dispatcher + +Public API: :func:`acompletion`, :func:`resolve_model_config`, +:func:`get_api_key`. +""" + +from __future__ import annotations + +from .config import get_api_key, resolve_model_config +from .pipeline import acompletion +from .runtime import set_verify_ssl +from .types import CompletionChunk, CompletionResponse, ToolCall + +__all__ = [ + "acompletion", + "resolve_model_config", + "get_api_key", + "CompletionResponse", + "CompletionChunk", + "ToolCall", + "set_verify_ssl", +] diff --git a/cecli/helpers/llms/config.py b/cecli/helpers/llms/config.py new file mode 100644 index 00000000000..98201733ba1 --- /dev/null +++ b/cecli/helpers/llms/config.py @@ -0,0 +1,128 @@ +"""Model config resolution for the llms package. + +Reuses cecli's ``model_config`` pipeline (``get_default_config``) and +``model_providers`` (``ModelProviderManager`` + ``PROVIDER_CONFIGS``) as the +source of truth for base URL / api-key env / extra headers, with hardcoded +fallbacks for the built-in providers. +""" + +from __future__ import annotations + +import os +from typing import Any, Dict, Optional + +from cecli.helpers.model_config.pipeline import get_default_config +from cecli.helpers.model_providers import ModelProviderManager + +#: Built-in provider defaults (base URLs / key env). Custom providers (chutes, +#: ...) are resolved from cecli PROVIDER_CONFIGS. +PROVIDER_DEFAULTS: Dict[str, Dict[str, Any]] = { + "openai": {"api_base": "https://api.openai.com/v1", "api_key_env": "OPENAI_API_KEY"}, + "anthropic": {"api_base": "https://api.anthropic.com", "api_key_env": "ANTHROPIC_API_KEY"}, + "deepseek": {"api_base": "https://api.deepseek.com/v1", "api_key_env": "DEEPSEEK_API_KEY"}, + "openrouter": {"api_base": "https://openrouter.ai/api/v1", "api_key_env": "OPENROUTER_API_KEY"}, + "gemini": { + "api_base": "https://generativelanguage.googleapis.com", + "api_key_env": "GEMINI_API_KEY", + }, + "github_copilot": {"api_base": "https://api.githubcopilot.com", "api_key_env": None}, + "meta": {"api_base": "https://api.meta.ai/v1", "api_key_env": "META_API_KEY"}, + "chutes": {"api_base": "https://llm.chutes.ai/v1/", "api_key_env": "CHUTES_API_KEY"}, +} + + +def resolve_model_config(model: str) -> Dict[str, Any]: + """Resolve provider/api-base/api-key/API-family for a model. + + Priority for api_base: explicit env ``{PROVIDER}_API_BASE`` > cecli + ``PROVIDER_CONFIGS[provider].api_base`` > built-in default. + """ + cfg = get_default_config(model) + llm_block = cfg.get("llm") or {} + api_block = cfg.get("api") or {} + provider = llm_block.get("litellm_provider") or ( + model.split("/", 1)[0] if "/" in model else None + ) + route = model.split("/", 1)[1] if "/" in model else model + + mpm = ModelProviderManager() + pcfg = mpm.get_provider_config(provider) or {} + + env_base = os.environ.get(f"{provider.upper()}_API_BASE") if provider else None + api_base = ( + env_base + or pcfg.get("api_base") + or PROVIDER_DEFAULTS.get(provider or "", {}).get("api_base") + ) + api_base = api_base.rstrip("/") if api_base else None + + # github_copilot resolves its endpoint from the authenticated session + # (api-key.json endpoints.api), never from a caller-supplied base. + if provider == "github_copilot": + from .providers.github_copilot import copilot_api_base + + api_base = (env_base or copilot_api_base()).rstrip("/") + + key_envs = pcfg.get("api_key_env") or [ + PROVIDER_DEFAULTS.get(provider or "", {}).get("api_key_env") + ] + key_env = next((e for e in key_envs if e), None) + + mode = llm_block.get("mode") or "chat" + endpoints = llm_block.get("supported_endpoints") or [] + + # API family: responses > anthropic messages > gemini > chat completions + if provider == "github_copilot": + # Copilot supports all three; claude -> anthropic-native /v1/messages, + # gpt-5-mini -> chat, everything else -> responses. + if "claude" in route.lower(): + family = "messages" + elif "gpt-5-mini" in route.lower(): + family = "chat" + else: + family = "responses" + elif mode == "responses" or "/v1/responses" in endpoints: + family = "responses" + elif provider == "anthropic": + family = "messages" + elif provider == "gemini": + family = "gemini" + else: + family = "chat" + + extra_headers = dict(pcfg.get("default_headers") or {}) + extra_body = dict(api_block.get("extra_body") or {}) + + return { + "model": model, + "provider": provider, + "route": route, + "family": family, + "api_base": api_base, + "api_key_env": key_env, + "extra_headers": extra_headers, + "extra_body": extra_body, + "api_block": api_block, + "llm_block": llm_block, + } + + +def get_api_key(resolved: Dict[str, Any], api_key: Optional[str]) -> Optional[str]: + """Return the API key: explicit arg, else env, else copilot auth, else None.""" + if api_key: + return api_key + + if resolved.get("provider") == "github_copilot": + from .providers.github_copilot import copilot_api_key + + return copilot_api_key() + + env_name = resolved.get("api_key_env") + + if env_name: + return os.environ.get(env_name) + + return None + + +__all__ = ["PROVIDER_DEFAULTS", "resolve_model_config", "get_api_key"] diff --git a/cecli/helpers/llms/domains/__init__.py b/cecli/helpers/llms/domains/__init__.py new file mode 100644 index 00000000000..398103b81cf --- /dev/null +++ b/cecli/helpers/llms/domains/__init__.py @@ -0,0 +1,25 @@ +"""Per-API-family adapters (sectional). + +One module per API family: chat (OpenAI /v1/chat/completions), responses +(OpenAI /v1/responses), messages (Anthropic /v1/messages), gemini +(generateContent). Each exports ``*_complete`` / ``*_stream`` entry points +plus payload builders and response normalizers. +""" + +from __future__ import annotations + +from .chat import chat_complete, chat_stream +from .gemini import gemini_complete, gemini_stream +from .messages import anthropic_complete, anthropic_stream +from .responses import responses_complete, responses_stream + +__all__ = [ + "chat_complete", + "chat_stream", + "responses_complete", + "responses_stream", + "anthropic_complete", + "anthropic_stream", + "gemini_complete", + "gemini_stream", +] diff --git a/cecli/helpers/llms/domains/chat.py b/cecli/helpers/llms/domains/chat.py new file mode 100644 index 00000000000..3622f0475b1 --- /dev/null +++ b/cecli/helpers/llms/domains/chat.py @@ -0,0 +1,293 @@ +"""OpenAI-compatible /v1/chat/completions adapter. + +Covers deepseek, openrouter, chutes, and github_copilot chat models. Reasoning +is extracted via :func:`cecli.helpers.llms.utils.extract_reasoning` which +handles the three wild shapes (reasoning_content / reasoning / +reasoning_details). Reasoning tokens reported without streamed text are marked +redacted (``Message.reasoning_redacted``). +""" + +from __future__ import annotations + +import json +from typing import Any, AsyncIterator, Dict, List, Optional + +from ..runtime import VERIFY_SSL, make_client +from ..types import ( + Choice, + CompletionChunk, + CompletionResponse, + Part, + PartsMessage, + ReasoningPart, + TextPart, + ToolCall, + ToolCallPart, + Usage, + parts_message_to_message, +) +from ..utils import extract_reasoning, sse_json_lines + +DEFAULT_TIMEOUT = 120.0 + + +def chat_payload( + resolved: Dict[str, Any], + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]], + stream: bool, + kwargs: Dict[str, Any], +) -> Dict[str, Any]: + payload: Dict[str, Any] = { + "model": resolved["route"], + "messages": messages, + "stream": stream, + } + + if tools: + payload["tools"] = tools + payload["tool_choice"] = kwargs.get("tool_choice", "auto") + + api_block = resolved.get("api_block") or {} + if api_block.get("reasoning_effort"): + payload["reasoning_effort"] = api_block["reasoning_effort"] + + if api_block.get("thinking"): + payload["thinking"] = api_block["thinking"] + + if api_block.get("parallel_tool_calls") is not None: + payload["parallel_tool_calls"] = api_block["parallel_tool_calls"] + + max_tokens = kwargs.get("max_tokens") + if max_tokens: + payload["max_tokens"] = max_tokens + + temperature = kwargs.get("temperature") + if temperature is not None: + payload["temperature"] = temperature + + # Some OpenAI-compatible providers expose a prompt-cache key as a setting + # (e.g. meta's prompt_cache_key). Pass it through when the caller provides it. + prompt_cache_key = kwargs.get("prompt_cache_key") + if prompt_cache_key: + payload["prompt_cache_key"] = prompt_cache_key + + if stream: + stream_options = dict(kwargs.get("stream_options") or {}) + stream_options.setdefault("include_usage", True) + else: + stream_options = kwargs.get("stream_options") + + if stream_options: + payload["stream_options"] = stream_options + + payload.update(resolved.get("extra_body") or {}) + payload.update(kwargs.get("extra_body") or {}) + return payload + + +async def chat_complete( + resolved: Dict[str, Any], + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]], + key: Optional[str], + headers: Dict[str, str], + kwargs: Dict[str, Any], +) -> CompletionResponse: + url = f"{resolved['api_base']}/chat/completions" + payload = chat_payload(resolved, messages, tools, False, kwargs) + hdrs = {"Authorization": f"Bearer {key}", "Content-Type": "application/json", **headers} + + async with make_client(timeout=DEFAULT_TIMEOUT, verify=VERIFY_SSL) as client: + resp = await client.post(url, json=payload, headers=hdrs) + resp.raise_for_status() + data = resp.json() + + return normalize_chat_response(data, resolved["model"]) + + +async def chat_stream( + resolved: Dict[str, Any], + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]], + key: Optional[str], + headers: Dict[str, str], + kwargs: Dict[str, Any], +) -> AsyncIterator[CompletionChunk]: + url = f"{resolved['api_base']}/chat/completions" + payload = chat_payload(resolved, messages, tools, True, kwargs) + hdrs = {"Authorization": f"Bearer {key}", "Content-Type": "application/json", **headers} + + async with make_client(timeout=DEFAULT_TIMEOUT, verify=VERIFY_SSL) as client: + async with client.stream("POST", url, json=payload, headers=hdrs) as resp: + resp.raise_for_status() + last_finish_reason = None + + async for json_obj in sse_json_lines(resp): + chunk = parse_chat_chunk(json_obj) + + if not chunk: + continue + + if chunk.finish_reason: + last_finish_reason = chunk.finish_reason + elif chunk.usage is not None and last_finish_reason: + # The trailing ``include_usage`` chunk carries cumulative + # usage but often no finish_reason of its own -- carry the + # last one forward so the final emitted chunk exposes it. + chunk.finish_reason = last_finish_reason + + yield chunk + + +def normalize_chat_response(data: Dict[str, Any], model: str) -> CompletionResponse: + usage_raw = data.get("usage") or {} + reasoning_tokens = (usage_raw.get("completion_tokens_details") or {}).get( + "reasoning_tokens" + ) or 0 + choices: List[Choice] = [] + + for raw in data.get("choices", []): + msg = raw.get("message") or {} + parts: List[Part] = [] + content = msg.get("content") + + if isinstance(content, str) and content: + parts.append(TextPart(text=content)) + elif isinstance(content, list): + for block in content: + if isinstance(block, dict) and block.get("type") == "text" and block.get("text"): + parts.append(TextPart(text=block["text"])) + + reasoning = extract_reasoning(msg) + + if reasoning: + parts.append(ReasoningPart(text=reasoning)) + elif reasoning_tokens > 0: + # Provider reports reasoning tokens but withholds the text. + parts.append(ReasoningPart(redacted=True)) + + for tc in msg.get("tool_calls") or []: + fn = tc.get("function") or {} + args_raw = fn.get("arguments") or "{}" + + try: + args = json.loads(args_raw) if isinstance(args_raw, str) else args_raw + except json.JSONDecodeError: + args = {"_raw": args_raw} + + parts.append( + ToolCallPart( + name=fn.get("name", ""), + arguments=args, + tool_call_id=tc.get("id"), + ) + ) + + pm = PartsMessage( + role=msg.get("role", "assistant"), + parts=parts, + provider_metadata=msg.get("provider_specific_fields") or {}, + ) + choices.append( + Choice( + index=raw.get("index", 0), + message=parts_message_to_message(pm), + finish_reason=raw.get("finish_reason"), + ) + ) + + usage = _usage_from_raw(usage_raw) or Usage() + return CompletionResponse( + id=data.get("id"), + model=model, + choices=choices, + usage=usage, + provider_specific_fields=data.get("provider_specific_fields") or {}, + ) + + +def parse_chat_chunk(data: Dict[str, Any]) -> Optional[CompletionChunk]: + choices = data.get("choices") or [] + usage_raw = data.get("usage") + + # ``include_usage`` streams put the cumulative usage on the final chunk, + # which still carries a (finish) ``choices`` entry -- parse it regardless. + usage = _usage_from_raw(usage_raw) + + if not choices: + if usage is not None: + return CompletionChunk(usage=usage) + + return None + + delta = choices[0].get("delta") or {} + text = delta.get("content") or "" + reasoning = extract_reasoning(delta) or "" + tool_calls = [] + + # Tool-call deltas arrive as fragments keyed by provider ``index``: the + # first fragment carries id+name, later fragments only argument deltas. + # Preserve that contract -- consumers (base_coder / stream_chunk_builder) + # merge fragments by index and concatenate the ``_fragment`` JSON. + for tc in delta.get("tool_calls") or []: + fn = tc.get("function") or {} + args_raw = fn.get("arguments") or "" + tool_calls.append( + ToolCall( + id=tc.get("id", ""), name=fn.get("name", ""), arguments={"_fragment": args_raw} + ) + ) + + # Some providers report reasoning tokens on the delta (or per-chunk usage) + # instead of only on the final usage chunk. Surface the token total on the + # chunk usage so the redacted-reasoning marker (reasoning_tokens > 0 with + # no streamed text) is not lost for consumers reading the details. + delta_details = delta.get("completion_tokens_details") + + if isinstance(delta_details, dict) and (delta_details.get("reasoning_tokens") or 0) > 0: + if usage is None: + usage = Usage(completion_tokens_details=dict(delta_details)) + elif not usage.completion_tokens_details: + usage.completion_tokens_details = dict(delta_details) + + finish_reason = choices[0].get("finish_reason") + + # Drop pure-noise deltas (role-only / blank chunks): nothing to emit and + # nothing for consumers to do with them. + if not text and not reasoning and not tool_calls and finish_reason is None and usage is None: + return None + + return CompletionChunk( + text=text, + reasoning=reasoning, + tool_calls=tool_calls, + finish_reason=finish_reason, + usage=usage, + ) + + +def _usage_from_raw(usage_raw: Optional[Dict[str, Any]]) -> Optional[Usage]: + """Build a Usage from an OpenAI usage payload (or None when absent).""" + if not isinstance(usage_raw, dict): + return None + + return Usage( + prompt_tokens=usage_raw.get("prompt_tokens"), + completion_tokens=usage_raw.get("completion_tokens"), + total_tokens=usage_raw.get("total_tokens"), + prompt_cache_hit_tokens=usage_raw.get("prompt_cache_hit_tokens"), + cache_read_input_tokens=usage_raw.get("cache_read_input_tokens"), + cache_creation_input_tokens=usage_raw.get("cache_creation_input_tokens"), + prompt_tokens_details=usage_raw.get("prompt_tokens_details"), + completion_tokens_details=usage_raw.get("completion_tokens_details"), + ) + + +__all__ = [ + "chat_payload", + "chat_complete", + "chat_stream", + "normalize_chat_response", + "parse_chat_chunk", +] diff --git a/cecli/helpers/llms/domains/gemini.py b/cecli/helpers/llms/domains/gemini.py new file mode 100644 index 00000000000..d19337e51fa --- /dev/null +++ b/cecli/helpers/llms/domains/gemini.py @@ -0,0 +1,634 @@ +"""Gemini generateContent / streamGenerateContent adapter. + +Reasoning effort maps to ``generationConfig.thinkingConfig`` via +:func:`gemini_thinking_config` (mirrors litellm Vertex +``_map_reasoning_effort_to_thinking_level``): Gemini 3+ models use +``thinkingLevel`` + ``includeThoughts``; older models use ``thinkingBudget``. +""" + +from __future__ import annotations + +import json +from typing import Any, AsyncIterator, Dict, List, Optional + +from ..runtime import VERIFY_SSL, make_client +from ..types import ( + Choice, + CompletionChunk, + CompletionResponse, + Part, + PartsMessage, + ReasoningPart, + TextPart, + ToolCall, + ToolCallPart, + Usage, + parts_message_to_message, +) +from ..utils import sse_json_lines, system_prompt + +DEFAULT_TIMEOUT = 120.0 + +#: Per-stream tool-call index map (``call_id -> index``), reset by + +#: Per-stream tool-call index map (``call_id -> index``), reset by +#: :func:`gemini_stream`. Gemini streams each parallel functionCall part as a +#: separate SSE chunk; assigning a distinct monotonic index per call_id keeps +#: them from collapsing onto index 0 in the litellm shim's accumulation. +_stream_state: Dict[str, Dict[str, int]] = {"tool_indices": {}} + + +def gemini_payload( + resolved: Dict[str, Any], + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]], + kwargs: Dict[str, Any], +) -> Dict[str, Any]: + system = system_prompt(messages) + # Map assistant tool-call ids to function metadata (name + native id / + # signature) so tool results can be encoded as ``functionResponse`` parts. + # Gemini requires a function response for each pending functionCall; a bare + # text part makes the model re-invoke the tool instead of consuming the + # result. + call_meta = _build_call_meta(messages) + + payload: Dict[str, Any] = { + "contents": _encode_contents(messages, call_meta), + } + + if system: + payload["systemInstruction"] = {"parts": [{"text": system}]} + + if tools: + payload["tools"] = [{"functionDeclarations": [gemini_tool(t) for t in tools]}] + + api_block = resolved.get("api_block") or {} + gen_config = payload.setdefault("generationConfig", {}) + + if api_block.get("reasoning_effort"): + gen_config["thinkingConfig"] = gemini_thinking_config( + resolved, api_block["reasoning_effort"] + ) + elif api_block.get("thinking"): + gen_config["thinkingConfig"] = { + "thinkingBudget": api_block["thinking"].get("budget_tokens", 8192) + } + + max_tokens = kwargs.get("max_tokens") + if max_tokens: + payload.setdefault("generationConfig", {})["maxOutputTokens"] = max_tokens + + temperature = kwargs.get("temperature") + if temperature is not None: + payload.setdefault("generationConfig", {})["temperature"] = temperature + payload.update(resolved.get("extra_body") or {}) + payload.update(kwargs.get("extra_body") or {}) + return payload + + +def gemini_thinking_config(resolved: Dict[str, Any], effort: str) -> Dict[str, Any]: + """Map reasoning_effort to Gemini thinkingConfig. + + Mirrors litellm ``VertexGeminiConfig._map_reasoning_effort_to_thinking_level``: + Gemini 3+ models use ``thinkingLevel`` + ``includeThoughts`` instead of the + older ``thinkingBudget``. gemini-3-flash supports the "minimal" level. + """ + route = (resolved.get("route") or "").lower() + is_gemini3flash = "gemini-3" in route and "flash" in route + include = effort != "disable" and effort != "none" + + level_map = { + "minimal": "minimal" if is_gemini3flash else "low", + "low": "low", + "medium": "medium" if is_gemini3flash else "high", + "high": "high", + "disable": "minimal" if is_gemini3flash else "low", + "none": "minimal" if is_gemini3flash else "low", + } + level = level_map.get(effort, "high") + return {"thinkingLevel": level, "includeThoughts": include} + + +def gemini_content( + msg: Dict[str, Any], name_by_call_id: Optional[Dict[str, Any]] = None +) -> Dict[str, Any]: + """Encode one chat message as a Gemini ``Content`` dict. + + ``name_by_call_id`` maps a tool-call id to either a plain function name or + a ``{"name": ..., "signature": ...}`` dict (see :func:`gemini_payload`); + tool results become ``functionResponse`` parts that echo the original call + id/signature when available. Assistant turns replay prior thought parts + (with their ``thoughtSignature``) for multi-turn reasoning. + """ + if msg.get("role") == "tool": + return _tool_content(msg, name_by_call_id) + + if msg.get("role") == "assistant": + return _model_content(msg) + + # user (and any non-assistant role, e.g. system, which folds into user). + content = msg.get("content") + + if isinstance(content, str): + return {"role": "user", "parts": [{"text": content}]} + + parts: List[Dict[str, Any]] = [] + + if content: + parts.append({"text": json.dumps(content)}) + + return {"role": "user", "parts": parts} + + +def gemini_tool(tool: Dict[str, Any]) -> Dict[str, Any]: + fn = tool.get("function") or {} + return { + "name": fn.get("name", ""), + "description": fn.get("description", ""), + "parameters": _gemini_schema(fn.get("parameters", {"type": "object", "properties": {}})), + } + + +async def gemini_complete( + resolved: Dict[str, Any], + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]], + key: Optional[str], + headers: Dict[str, str], + kwargs: Dict[str, Any], +) -> CompletionResponse: + url = f"{resolved['api_base']}/v1beta/models/{resolved['route']}:generateContent" + payload = gemini_payload(resolved, messages, tools, kwargs) + hdrs = {"Content-Type": "application/json", **headers} + params = {"key": key} if key else {} + + async with make_client(timeout=DEFAULT_TIMEOUT, verify=VERIFY_SSL) as client: + resp = await client.post(url, json=payload, headers=hdrs, params=params) + resp.raise_for_status() + data = resp.json() + + return normalize_gemini_response(data, resolved["model"]) + + +async def gemini_stream( + resolved: Dict[str, Any], + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]], + key: Optional[str], + headers: Dict[str, str], + kwargs: Dict[str, Any], +) -> AsyncIterator[CompletionChunk]: + url = f"{resolved['api_base']}/v1beta/models/{resolved['route']}:streamGenerateContent" + payload = gemini_payload(resolved, messages, tools, kwargs) + hdrs = {"Content-Type": "application/json", **headers} + params = {"key": key, "alt": "sse"} if key else {"alt": "sse"} + has_seen_tool_calls = False + _stream_state["tool_indices"] = {} + + async with make_client(timeout=DEFAULT_TIMEOUT, verify=VERIFY_SSL) as client: + async with client.stream("POST", url, json=payload, headers=hdrs, params=params) as resp: + resp.raise_for_status() + async for json_obj in sse_json_lines(resp): + chunk = parse_gemini_chunk(json_obj) + + if chunk: + if chunk.tool_calls: + has_seen_tool_calls = True + + # Once a functionCall has streamed, the turn's finish must + # read "tool_calls" even if a later chunk reports STOP. + if has_seen_tool_calls and chunk.finish_reason is not None: + chunk.finish_reason = "tool_calls" + + yield chunk + + +def normalize_gemini_response(data: Dict[str, Any], model: str) -> CompletionResponse: + parts: List[Part] = [] + provider_fields: Dict[str, Any] = {} + finish = None + saw_function_call = False + thought_signature = None + + for candidate in data.get("candidates") or []: + content = candidate.get("content") or {} + + for part in content.get("parts") or []: + if "text" in part: + if part.get("thought"): + text = part.get("text") or "" + parts.append(ReasoningPart(text=text, redacted=not text)) + else: + parts.append(TextPart(text=part.get("text") or "")) + + if "functionCall" in part: + fc = part["functionCall"] + saw_function_call = True + call_part = ToolCallPart( + name=fc.get("name", ""), + arguments=fc.get("args") or {}, + tool_call_id=fc.get("id"), + ) + + # Gemini attaches thoughtSignature as a SIBLING of the + # functionCall part, not as a field inside it. + sig = part.get("thoughtSignature") or fc.get("signature") + if sig: + call_part.provider_metadata["signature"] = sig + + parts.append(call_part) + + if candidate.get("thoughtSignature"): + thought_signature = candidate["thoughtSignature"] + + finish = _map_finish_reason(candidate.get("finishReason")) + + if saw_function_call: + finish = "tool_calls" + + if thought_signature: + # Echo the thoughtSignature back on the next request (the encoder + # replays prior thought parts with it). Keep the raw camelCase field + # plus the snake_case alias the cecli request pipeline reads. + provider_fields["thoughtSignature"] = thought_signature + provider_fields["thought_signature"] = thought_signature + + # Preserve raw thought parts for verbatim replay, and per-call functionCall + # signatures so tool results echo the original id/signature. + thought_parts = [] + for p in parts: + if isinstance(p, ReasoningPart): + thought_entry = {"text": p.text, "thought": True} + + if p.provider_metadata.get("signature"): + thought_entry["signature"] = p.provider_metadata["signature"] + + thought_parts.append(thought_entry) + + if thought_parts: + provider_fields["thought_parts"] = thought_parts + + call_signatures = { + p.tool_call_id: p.provider_metadata["signature"] + for p in parts + if isinstance(p, ToolCallPart) and p.tool_call_id and p.provider_metadata.get("signature") + } + + if call_signatures: + provider_fields["function_call_signatures"] = call_signatures + + pm = PartsMessage(role="assistant", parts=parts, provider_metadata=provider_fields) + message = parts_message_to_message(pm) + usage = _gemini_usage(data.get("usageMetadata") or {}) + return CompletionResponse( + id=data.get("responseId"), + model=model, + choices=[Choice(index=0, message=message, finish_reason=finish)], + usage=usage, + provider_specific_fields=provider_fields, + ) + + +def parse_gemini_chunk(data: Dict[str, Any]) -> Optional[CompletionChunk]: + chunk = CompletionChunk() + parts: List[str] = [] + reasoning: List[str] = [] + tool_calls: List[ToolCall] = [] + saw_function_call = False + + for candidate in data.get("candidates") or []: + content = candidate.get("content") or {} + + for part in content.get("parts") or []: + if "text" in part: + if part.get("thought"): + reasoning.append(part["text"]) + else: + parts.append(part["text"]) + + if "functionCall" in part: + fc = part["functionCall"] + saw_function_call = True + call_id = fc.get("id") or f"call_{len(tool_calls)}" + + # Assign a stable per-stream index keyed by call id so parallel + # functionCall parts streamed in separate chunks do not collapse + # onto index 0 in the litellm shim. + indices = _stream_state["tool_indices"] + + if call_id not in indices: + indices[call_id] = len(indices) + + tool_calls.append( + ToolCall( + id=call_id, + name=fc.get("name", ""), + arguments=fc.get("args") or {}, + index=indices[call_id], + ) + ) + + # Capture the native thoughtSignature (a sibling of functionCall) + # so streamed tool calls can echo it back on the next turn. + sig = part.get("thoughtSignature") + + if sig: + chunk.provider_specific_fields.setdefault("function_call_signatures", {})[ + call_id + ] = sig + + if candidate.get("finishReason"): + chunk.finish_reason = _map_finish_reason(candidate.get("finishReason")) + + if saw_function_call: + chunk.finish_reason = "tool_calls" + + chunk.text = "".join(parts) + chunk.reasoning = "".join(reasoning) + chunk.tool_calls = tool_calls + chunk.usage = _gemini_usage(data.get("usageMetadata") or {}) + + if ( + not chunk.text + and not chunk.reasoning + and not chunk.tool_calls + and not chunk.finish_reason + and not chunk.usage + ): + return None + + return chunk + + +# --------------------------------------------------------------------------- +# Private helpers (kept at the bottom so the core logic reads first) +# --------------------------------------------------------------------------- + + +_FINISH_MAP = { + "STOP": "stop", + "MAX_TOKENS": "length", + "SAFETY": "content_filter", + "RECITATION": "content_filter", + "LANGUAGE": "content_filter", + "BLOCKLIST": "content_filter", + "PROHIBITED_CONTENT": "content_filter", + "SPII": "content_filter", + "IMAGE_SAFETY": "content_filter", + "MALFORMED_FUNCTION_CALL": "tool_calls", +} + + +def _map_finish_reason(raw: Optional[str]) -> Optional[str]: + """Map a Gemini ``finishReason`` to the normalized finish_reason.""" + if raw is None: + return None + + return _FINISH_MAP.get(raw, raw) + + +def _gemini_usage(usage_raw: Dict[str, Any]) -> Optional[Usage]: + """Map Gemini ``usageMetadata`` onto :class:`Usage`. + + ``thoughtsTokenCount`` is added to the output tokens when + ``is_candidate_token_count_inclusive`` is False, and is surfaced on + ``completion_tokens_details["reasoning_tokens"]``; + ``cachedContentTokenCount`` maps to ``prompt_cache_hit_tokens``. + """ + if not usage_raw: + return None + + candidates = usage_raw.get("candidatesTokenCount") + thoughts = usage_raw.get("thoughtsTokenCount") + completion = candidates + + if thoughts and usage_raw.get("is_candidate_token_count_inclusive") is False: + completion = (candidates or 0) + thoughts + + details = {"reasoning_tokens": thoughts} if thoughts is not None else None + + return Usage( + prompt_tokens=usage_raw.get("promptTokenCount"), + completion_tokens=completion, + total_tokens=usage_raw.get("totalTokenCount"), + prompt_cache_hit_tokens=usage_raw.get("cachedContentTokenCount"), + completion_tokens_details=details, + ) + + +def _build_call_meta(messages: List[Dict[str, Any]]) -> Dict[str, Any]: + """Map assistant tool-call ids to ``{"name", "signature"}`` metadata. + + Signatures come from ``provider_specific_fields["function_call_signatures"]`` + (captured by :func:`normalize_gemini_response` from native ``functionCall`` + parts) so ``functionResponse`` parts echo the original id/signature. + """ + meta: Dict[str, Any] = {} + + for m in messages: + if m.get("role") != "assistant": + continue + + psf = m.get("provider_specific_fields") or {} + signatures = psf.get("function_call_signatures") or {} + + for tc in m.get("tool_calls") or []: + call_id = tc.get("id") + + if not call_id: + continue + + fn = tc.get("function") or {} + meta[call_id] = { + "name": fn.get("name", ""), + "signature": signatures.get(call_id), + } + + return meta + + +def _call_meta(name_by_call_id: Optional[Dict[str, Any]], call_id: Optional[str]) -> Dict[str, Any]: + """Resolve a tool-call id to metadata (tolerates legacy ``{id: name}`` maps).""" + if not name_by_call_id or not call_id: + return {} + + entry = name_by_call_id.get(call_id) + + if isinstance(entry, dict): + return entry + + return {"name": entry or ""} + + +def _tool_content(msg: Dict[str, Any], name_by_call_id: Optional[Dict[str, Any]]) -> Dict[str, Any]: + """Encode a tool-result message as a ``functionResponse`` user Content.""" + meta = _call_meta(name_by_call_id, msg.get("tool_call_id")) + name = meta.get("name") or "" + + if name: + fr: Dict[str, Any] = { + "name": name, + "response": {"output": msg.get("content") or ""}, + } + call_id = msg.get("tool_call_id") + + if call_id: + fr["id"] = call_id + + part_dict: Dict[str, Any] = {"functionResponse": fr} + + if meta.get("signature"): + part_dict["thoughtSignature"] = meta["signature"] + + return {"role": "user", "parts": [part_dict]} + + # No matching functionCall recorded -- fall back to plain text. + return {"role": "user", "parts": [{"text": msg.get("content") or ""}]} + + +def _model_content(msg: Dict[str, Any]) -> Dict[str, Any]: + """Encode an assistant message, replaying prior thought parts. + + Gemini requires the full thinking turn (thought parts + ``thoughtSignature``) + echoed back in the history for multi-turn reasoning, even when thoughts are + hidden. The parts/signature captured by :func:`normalize_gemini_response` + live on ``provider_specific_fields``; when an upstream consumer dropped + them, fall back to reconstructing a single thought part from + ``reasoning_content``. + """ + psf = msg.get("provider_specific_fields") or {} + thought_parts = psf.get("thought_parts") or [] + parts: List[Dict[str, Any]] = [] + + for tp in thought_parts: + tp_out: Dict[str, Any] = {"text": tp.get("text", ""), "thought": True} + + # The normalizer may stash the thought-part signature under either key + # ("thoughtSignature" preferred, "signature" tolerated). + tp_sig = tp.get("thoughtSignature") or tp.get("signature") + + if tp_sig: + tp_out["thoughtSignature"] = tp_sig + + parts.append(tp_out) + + # Fallback: reconstruct a single thought part from reasoning_content when + # the richer provider metadata was dropped by an upstream consumer. Thought + # parts must precede any visible text/functionCall parts in the Content. + if not thought_parts: + reasoning = msg.get("reasoning_content") + + if isinstance(reasoning, str) and reasoning.strip(): + parts.append({"text": reasoning, "thought": True}) + + content = msg.get("content") + + if isinstance(content, str): + if content or not parts: + parts.append({"text": content}) + elif content: + parts.append({"text": json.dumps(content)}) + + call_signatures = psf.get("function_call_signatures") or {} + + for tc in msg.get("tool_calls") or []: + fn = tc.get("function") or {} + args_raw = fn.get("arguments") or "{}" + + try: + args = json.loads(args_raw) if isinstance(args_raw, str) else args_raw + except json.JSONDecodeError: + args = {} + + fc: Dict[str, Any] = {"name": fn.get("name", ""), "args": args} + + if tc.get("id"): + fc["id"] = tc["id"] + + part_dict: Dict[str, Any] = {"functionCall": fc} + sig = call_signatures.get(tc["id"]) if tc.get("id") else None + + if sig: + part_dict["thoughtSignature"] = sig + + parts.append(part_dict) + + content_dict: Dict[str, Any] = {"role": "model", "parts": parts} + signature = psf.get("thoughtSignature") or psf.get("thought_signature") + + if signature and signature != "skip_thought_signature_validator": + content_dict["thoughtSignature"] = signature + + return content_dict + + +def _encode_contents( + messages: List[Dict[str, Any]], name_by_call_id: Optional[Dict[str, Any]] +) -> List[Dict[str, Any]]: + """Encode the message list, merging consecutive same-role Contents. + + Gemini rejects histories with repeated ``user`` or ``model`` roles. System + messages are hoisted to ``systemInstruction`` by the caller, so any + remaining ``system`` role folds into ``user`` (``gemini_content`` maps it), + and tool results are already ``user``-role ``functionResponse`` parts. + Merging keeps e.g. multiple tool results (plus a following user text turn) + in ONE Content right after the assistant tool-call turn. + """ + encoded = [gemini_content(m, name_by_call_id) for m in messages if m.get("role") != "system"] + merged: List[Dict[str, Any]] = [] + + for content in encoded: + if merged and merged[-1].get("role") == content.get("role"): + merged[-1]["parts"].extend(content.get("parts") or []) + else: + merged.append(content) + + return merged + + +#: JSON-Schema keywords Gemini's ``FunctionDeclaration.parameters`` rejects +#: (OpenAPI 3.0 strict subset). ``additionalProperties`` is emitted by +#: OpenAI-style strict tool schemas and caused a 400 for AgentCoder tools. +_GEMINI_UNSUPPORTED_SCHEMA_KEYS = frozenset({"additionalProperties", "$schema", "$defs"}) + + +def _gemini_schema(schema: Dict[str, Any]) -> Dict[str, Any]: + """Recursively strip JSON-Schema keywords Gemini function declarations reject. + + Gemini validates ``FunctionDeclaration.parameters`` against a strict + subset of OpenAPI 3.0 and returns 400 for unknown keys (observed: + ``Unknown name "additionalProperties"`` on AgentCoder tool schemas). + Everything else -- including ``default``, which Gemini does support -- is + preserved verbatim. + """ + if not isinstance(schema, dict): + return schema + + cleaned: Dict[str, Any] = {} + + for key, value in schema.items(): + if key in _GEMINI_UNSUPPORTED_SCHEMA_KEYS: + continue + + if isinstance(value, dict): + cleaned[key] = _gemini_schema(value) + elif isinstance(value, list): + cleaned[key] = [ + _gemini_schema(item) if isinstance(item, dict) else item for item in value + ] + else: + cleaned[key] = value + + return cleaned + + +__all__ = [ + "gemini_payload", + "gemini_thinking_config", + "gemini_content", + "gemini_tool", + "gemini_complete", + "gemini_stream", + "normalize_gemini_response", + "parse_gemini_chunk", +] diff --git a/cecli/helpers/llms/domains/messages.py b/cecli/helpers/llms/domains/messages.py new file mode 100644 index 00000000000..94d2ffcc2e5 --- /dev/null +++ b/cecli/helpers/llms/domains/messages.py @@ -0,0 +1,535 @@ +"""Anthropic /v1/messages adapter (claude + github_copilot anthropic-native). + +Claude 5+ uses adaptive thinking via ``output_config``; older Claude uses the +``thinking`` block. Thinking signatures and redacted-thinking payloads are +stashed in ``provider_specific_fields["anthropic"]`` as an ordered content-block +list so later turns replay the exact block sequence Anthropic verifies by +position. For github_copilot the adapter uses Bearer auth + messages-proxy +headers (merged by the provider adapter) instead of ``x-api-key``. +""" + +from __future__ import annotations + +import json +from typing import Any, AsyncIterator, Dict, List, Optional + +from ..formatters import format_thinking +from ..runtime import VERIFY_SSL, make_client +from ..types import ( + Choice, + CompletionChunk, + CompletionResponse, + Message, + Part, + PartsMessage, + ReasoningPart, + TextPart, + ToolCall, + ToolCallPart, + Usage, + parts_message_to_message, +) +from ..utils import sse_json_lines, system_prompt + +DEFAULT_TIMEOUT = 120.0 + + +def anthropic_payload( + resolved: Dict[str, Any], + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]], + stream: bool, + kwargs: Dict[str, Any], +) -> Dict[str, Any]: + system = system_prompt(messages) + payload: Dict[str, Any] = { + "model": resolved["route"], + "messages": [anthropic_message(m) for m in messages if m.get("role") != "system"], + "max_tokens": ( + kwargs.get("max_tokens") or resolved.get("llm_block", {}).get("max_tokens") or 4096 + ), + "stream": stream, + } + + if system: + payload["system"] = system + + if tools: + payload["tools"] = [anthropic_tool(t) for t in tools] + + api_block = resolved.get("api_block") or {} + + # Claude 5+ uses adaptive thinking via ``output_config.effort``; pre-5 + # Claude uses the ``thinking`` block. Gate on the model generation so + # e.g. claude-haiku-4-5 (no effort support) never receives output_config. + format_thinking(resolved.get("provider"), resolved.get("route"), resolved.get("llm_block"))( + payload, api_block + ) + + temperature = kwargs.get("temperature") + if temperature is not None: + payload["temperature"] = temperature + + payload.update(resolved.get("extra_body") or {}) + payload.update(kwargs.get("extra_body") or {}) + return payload + + +def anthropic_message(msg: Dict[str, Any]) -> Dict[str, Any]: + role = msg.get("role") + + if role == "tool": + return { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": msg.get("tool_call_id", ""), + "content": msg.get("content") or "", + } + ], + } + + if role == "assistant": + # Prefer the stashed Anthropic content blocks (thinking signatures, + # redacted-thinking payloads, interleaved tool_use order) so later + # turns replay the exact block sequence Anthropic verifies by position. + blocks = _anthropic_message_content(msg) + + if blocks is not None: + return {"role": "assistant", "content": blocks} + + content = msg.get("content") or "" + blocks = [] + + if content: + blocks.append({"type": "text", "text": content}) + + for tc in msg.get("tool_calls") or []: + fn = tc.get("function") or {} + args_raw = fn.get("arguments") or "{}" + + try: + args = json.loads(args_raw) if isinstance(args_raw, str) else args_raw + except json.JSONDecodeError: + args = {} + + blocks.append( + { + "type": "tool_use", + "id": tc.get("id", ""), + "name": fn.get("name", ""), + "input": args, + } + ) + + return {"role": "assistant", "content": blocks} + + content = msg.get("content") + return {"role": role, "content": content if isinstance(content, str) else json.dumps(content)} + + +def anthropic_tool(tool: Dict[str, Any]) -> Dict[str, Any]: + fn = tool.get("function") or {} + return { + "name": fn.get("name", ""), + "description": fn.get("description", ""), + "input_schema": fn.get("parameters", {"type": "object", "properties": {}}), + } + + +async def anthropic_complete( + resolved: Dict[str, Any], + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]], + key: Optional[str], + headers: Dict[str, str], + kwargs: Dict[str, Any], +) -> CompletionResponse: + url = f"{resolved['api_base']}/v1/messages" + payload = anthropic_payload(resolved, messages, tools, False, kwargs) + + if resolved.get("provider") == "github_copilot": + # Copilot /v1/messages proxy: Bearer auth + messages-proxy headers + # (already merged into `headers` by the provider adapter), no x-api-key. + hdrs = { + "Content-Type": "application/json", + **headers, + } + else: + hdrs = { + "x-api-key": key or "", + "anthropic-version": "2023-06-01", + "Content-Type": "application/json", + **headers, + } + + async with make_client(timeout=DEFAULT_TIMEOUT, verify=VERIFY_SSL) as client: + resp = await client.post(url, json=payload, headers=hdrs) + resp.raise_for_status() + data = resp.json() + + return normalize_anthropic_response(data, resolved["model"]) + + +async def anthropic_stream( + resolved: Dict[str, Any], + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]], + key: Optional[str], + headers: Dict[str, str], + kwargs: Dict[str, Any], +) -> AsyncIterator[CompletionChunk]: + url = f"{resolved['api_base']}/v1/messages" + payload = anthropic_payload(resolved, messages, tools, True, kwargs) + + if resolved.get("provider") == "github_copilot": + hdrs = { + "Content-Type": "application/json", + **headers, + } + else: + hdrs = { + "x-api-key": key or "", + "anthropic-version": "2023-06-01", + "Content-Type": "application/json", + **headers, + } + + # Content-block state for stateless thinking-signature replay: each + # content_block_start opens a block (text/thinking/redacted_thinking/ + # tool_use) that we accumulate in SSE order, so the final message_delta + # chunk can carry the ordered blocks (signatures included) back for storage. + blocks: Dict[int, Dict[str, Any]] = {} + current: Optional[int] = None + + async with make_client(timeout=DEFAULT_TIMEOUT, verify=VERIFY_SSL) as client: + async with client.stream("POST", url, json=payload, headers=hdrs) as resp: + resp.raise_for_status() + + async for json_obj in sse_json_lines(resp): + evt = json_obj.get("type") or "" + index = json_obj.get("index") + + if evt == "content_block_start": + block = json_obj.get("content_block") or {} + btype = block.get("type") + current = index + + if btype == "text": + blocks[index] = {"type": "text", "text": block.get("text") or ""} + + elif btype == "thinking": + blocks[index] = { + "type": "thinking", + "thinking": block.get("thinking") or "", + "signature": block.get("signature"), + } + + elif btype == "redacted_thinking": + blocks[index] = { + "type": "redacted_thinking", + "data": block.get("data") or "", + "signature": block.get("signature"), + } + + elif btype == "tool_use": + blocks[index] = { + "type": "tool_use", + "id": block.get("id", ""), + "name": block.get("name", ""), + "input": block.get("input") or {}, + } + + elif evt == "content_block_delta": + delta = json_obj.get("delta") or {} + dtype = delta.get("type") + entry = blocks.get(current) if current is not None else None + + if entry is not None: + if dtype == "text_delta": + entry["text"] += delta.get("text") or "" + + elif dtype == "thinking_delta": + entry["thinking"] += delta.get("thinking") or "" + + elif dtype == "signature_delta": + entry["signature"] = delta.get("signature") + + elif evt == "content_block_stop": + current = None + + chunk = parse_anthropic_chunk(json_obj) + + if chunk: + if evt == "message_delta" and blocks: + ordered = [blocks[key] for key in sorted(blocks)] + + if ordered: + chunk.provider_specific_fields = {"anthropic": ordered} + + yield chunk + + +def normalize_anthropic_response(data: Dict[str, Any], model: str) -> CompletionResponse: + if data.get("type") == "error" or data.get("is_error"): + return CompletionResponse( + id=data.get("id"), + model=model, + choices=[Choice(index=0, message=Message(role="assistant"), finish_reason="error")], + provider_specific_fields={"error": data.get("error") or data}, + ) + + parts: List[Part] = [] + blocks: List[Dict[str, Any]] = [] + + for block in data.get("content") or []: + btype = block.get("type") + + if btype == "text": + text = block.get("text") or "" + parts.append(TextPart(text=text)) + blocks.append({"type": "text", "text": text}) + + elif btype == "thinking": + thinking_text = block.get("thinking") or "" + signature = block.get("signature") + + if thinking_text.strip() or signature: + parts.append(ReasoningPart(text=thinking_text)) + blocks.append( + {"type": "thinking", "thinking": thinking_text, "signature": signature} + ) + + elif btype == "redacted_thinking": + parts.append(ReasoningPart(redacted=True)) + blocks.append( + { + "type": "redacted_thinking", + "data": block.get("data") or "", + "signature": block.get("signature"), + } + ) + + elif btype == "tool_use": + parts.append( + ToolCallPart( + name=block.get("name", ""), + arguments=block.get("input") or {}, + tool_call_id=block.get("id", ""), + ) + ) + blocks.append( + { + "type": "tool_use", + "id": block.get("id", ""), + "name": block.get("name", ""), + "input": block.get("input") or {}, + } + ) + + provider_fields = {"anthropic": blocks} if blocks else {} + + pm = PartsMessage(role="assistant", parts=parts, provider_metadata=provider_fields) + message = parts_message_to_message(pm) + + usage = _anthropic_usage(data.get("usage") or {}) + + if data.get("service_tier"): + details = dict(usage.completion_tokens_details or {}) + details["service_tier"] = data["service_tier"] + usage.completion_tokens_details = details + + return CompletionResponse( + id=data.get("id"), + model=model, + choices=[ + Choice(index=0, message=message, finish_reason=_finish_reason(data.get("stop_reason"))) + ], + usage=usage, + provider_specific_fields=provider_fields, + ) + + +def parse_anthropic_chunk(data: Dict[str, Any]) -> Optional[CompletionChunk]: + evt = data.get("type") or "" + chunk = CompletionChunk() + + if evt == "content_block_delta": + delta = data.get("delta") or {} + dtype = delta.get("type") + index = data.get("index") + + if dtype == "text_delta": + chunk.text = delta.get("text") or "" + + elif dtype == "thinking_delta": + chunk.reasoning = delta.get("thinking") or "" + + elif dtype == "signature_delta": + # Nothing visible to stream; anthropic_stream attaches the + # signature to the open thinking block for next-turn replay. + return None + + elif dtype == "input_json_delta": + chunk.tool_calls = [ + ToolCall( + id="", + name="", + arguments={"_index": index, "_fragment": delta.get("partial_json") or ""}, + ) + ] + + else: + return None + + return chunk + + if evt == "content_block_start": + block = data.get("content_block") or {} + btype = block.get("type") + index = data.get("index") + + if btype == "tool_use": + chunk.tool_calls = [ + ToolCall( + id=block.get("id", ""), + name=block.get("name", ""), + arguments={"_index": index, "_fragment": ""}, + ) + ] + + return chunk + + if btype == "thinking": + chunk.reasoning = block.get("thinking") or "" + + return chunk + + if btype == "redacted_thinking": + chunk.reasoning = "[encrypted thinking block present]" + + return chunk + + return None + + if evt == "message_delta": + delta = data.get("delta") or {} + chunk.finish_reason = _finish_reason(delta.get("stop_reason")) + chunk.usage = _anthropic_usage(data.get("usage") or {}) + + return chunk + + if evt == "error" or data.get("is_error"): + chunk.finish_reason = "error" + + return chunk + + return None + + +def _finish_reason(stop_reason: Optional[str]) -> Optional[str]: + """Map an Anthropic ``stop_reason`` to the normalized finish_reason.""" + if not stop_reason: + return None + + return { + "end_turn": "stop", + "max_tokens": "length", + "stop_sequence": "stop", + "tool_use": "tool_calls", + "refusal": "content_filter", + "pause_turn": "stop", + }.get(stop_reason, stop_reason) + + +def _anthropic_usage(usage_raw: Dict[str, Any]) -> Usage: + """Build a normalized Usage from an Anthropic usage block.""" + input_tokens = usage_raw.get("input_tokens") or 0 + output_tokens = usage_raw.get("output_tokens") or 0 + cache_read = usage_raw.get("cache_read_input_tokens") or 0 + cache_creation = usage_raw.get("cache_creation_input_tokens") or 0 + + details: Dict[str, Any] = {} + output_details = usage_raw.get("output_tokens_details") or {} + thinking_tokens = output_details.get("thinking_tokens") + + if thinking_tokens is not None: + details["reasoning_tokens"] = thinking_tokens + + service_tier = usage_raw.get("service_tier") + + if service_tier is not None: + details["service_tier"] = service_tier + + return Usage( + prompt_tokens=input_tokens or None, + completion_tokens=output_tokens or None, + total_tokens=(input_tokens + cache_read + cache_creation) or None, + cache_read_input_tokens=usage_raw.get("cache_read_input_tokens"), + cache_creation_input_tokens=usage_raw.get("cache_creation_input_tokens"), + completion_tokens_details=details or None, + ) + + +def _anthropic_message_content(msg: Dict[str, Any]) -> Optional[List[Dict[str, Any]]]: + """Rebuild wire-format assistant content blocks from stashed metadata. + + Returns None when the message carries no Anthropic stash so the caller can + fall back to the plain text + tool_calls encoding. + """ + psf = msg.get("provider_specific_fields") or {} + blocks = psf.get("anthropic") + + if not isinstance(blocks, list): + return None + + content: List[Dict[str, Any]] = [] + + for block in blocks: + if not isinstance(block, dict): + continue + + btype = block.get("type") + + if btype == "text": + content.append({"type": "text", "text": block.get("text") or ""}) + + elif btype == "thinking": + entry: Dict[str, Any] = {"type": "thinking", "thinking": block.get("thinking") or ""} + + if block.get("signature"): + entry["signature"] = block["signature"] + + content.append(entry) + + elif btype == "redacted_thinking": + entry = {"type": "redacted_thinking", "data": block.get("data") or ""} + + if block.get("signature"): + entry["signature"] = block["signature"] + + content.append(entry) + + elif btype == "tool_use": + content.append( + { + "type": "tool_use", + "id": block.get("id", ""), + "name": block.get("name", ""), + "input": block.get("input") or {}, + } + ) + + return content or None + + +__all__ = [ + "anthropic_payload", + "anthropic_message", + "anthropic_tool", + "anthropic_complete", + "anthropic_stream", + "normalize_anthropic_response", + "parse_anthropic_chunk", +] diff --git a/cecli/helpers/llms/domains/responses.py b/cecli/helpers/llms/domains/responses.py new file mode 100644 index 00000000000..6830afac4a9 --- /dev/null +++ b/cecli/helpers/llms/domains/responses.py @@ -0,0 +1,467 @@ +"""OpenAI /v1/responses adapter. + +Covers gpt-5.x and meta (muse-spark). Responses-mode models return reasoning as +``reasoning`` items with ``content``/``summary`` blocks, or as an opaque +``encrypted_content`` blob (meta muse-spark) which is stashed on +``provider_specific_fields["reasoning_items"]`` so ``to_responses_input`` can +replay it verbatim on the next turn (stateless round-trip); the assistant +message is marked ``reasoning_redacted`` instead of fabricating placeholder +text. +""" + +from __future__ import annotations + +import json +from typing import Any, AsyncIterator, Dict, List, Optional + +from ..runtime import VERIFY_SSL, make_client +from ..types import ( + Choice, + CompletionChunk, + CompletionResponse, + Part, + PartsMessage, + ReasoningPart, + TextPart, + ToolCall, + ToolCallPart, + Usage, + parts_message_to_message, +) +from ..utils import sse_json_lines, system_prompt + +DEFAULT_TIMEOUT = 120.0 + +#: Per-stream correlation state for the SSE loop. ``responses_stream`` resets +#: this before each request; ``parse_responses_chunk`` reads/updates it so +#: event-to-event correlation (function_call ``item_id`` -> call_id/name and +#: reasoning item capture) survives while keeping the public single-argument +#: signature of ``parse_responses_chunk`` stable. +_stream_state: Dict[str, Any] = {"tool_items": {}, "reasoning_items": {}} + + +def responses_payload( + resolved: Dict[str, Any], + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]], + stream: bool, + kwargs: Dict[str, Any], +) -> Dict[str, Any]: + payload: Dict[str, Any] = { + "model": resolved["route"], + "input": to_responses_input(messages), + "stream": stream, + "store": False, + } + + if tools: + payload["tools"] = [responses_tool(t) for t in tools] + + api_block = resolved.get("api_block") or {} + if api_block.get("reasoning_effort"): + payload["reasoning"] = {"effort": api_block["reasoning_effort"], "summary": "auto"} + # Encrypted reasoning blobs (meta muse-spark) are only returned when + # explicitly requested; without them prior reasoning items cannot be + # replayed on the next turn. + payload["include"] = ["reasoning.encrypted_content"] + + if api_block.get("parallel_tool_calls") is not None: + payload["parallel_tool_calls"] = api_block["parallel_tool_calls"] + + max_tokens = kwargs.get("max_tokens") + if max_tokens: + payload["max_output_tokens"] = max_tokens + + temperature = kwargs.get("temperature") + if temperature is not None: + payload["temperature"] = temperature + + payload.update(resolved.get("extra_body") or {}) + payload.update(kwargs.get("extra_body") or {}) + return payload + + +def to_responses_input(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Convert OpenAI chat messages to responses-API input items.""" + items: List[Dict[str, Any]] = [] + + for msg in messages: + role = msg.get("role") + content = msg.get("content") + + if role == "system": + continue # handled via instructions at call site + + if role == "assistant": + # Replay stashed reasoning items BEFORE the assistant message item + # so the provider can continue the encrypted reasoning state + # (stateless round-trip: the whole conversation is re-sent). + for r_item in _stashed_reasoning_items(msg): + items.append(_reasoning_input_item(r_item)) + + # Assistant turns must use ``output_text`` content blocks; Copilot / + # OpenAI reject ``input_text`` on assistant messages with HTTP 400 + # ("Supported values are: 'output_text' and 'refusal'"). + if content: + text = content if isinstance(content, str) else json.dumps(content) + items.append( + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": text}], + } + ) + + # Prior assistant tool calls are ``function_call`` input items + # (``function_call_output`` is reserved for tool results below). + for tc in msg.get("tool_calls") or []: + fn = tc.get("function") or {} + items.append( + { + "type": "function_call", + "call_id": tc.get("id", ""), + "name": fn.get("name", ""), + "arguments": fn.get("arguments", ""), + } + ) + continue + + if role == "tool": + items.append( + { + "type": "function_call_output", + "call_id": msg.get("tool_call_id", ""), + "output": content or "", + } + ) + continue + + text = content if isinstance(content, str) else json.dumps(content) + items.append( + {"type": "message", "role": role, "content": [{"type": "input_text", "text": text}]} + ) + + return items + + +def responses_tool(tool: Dict[str, Any]) -> Dict[str, Any]: + fn = tool.get("function") or {} + return { + "type": "function", + "name": fn.get("name", ""), + "description": fn.get("description", ""), + "parameters": fn.get("parameters", {"type": "object", "properties": {}}), + "strict": tool.get("strict", False), + } + + +async def responses_complete( + resolved: Dict[str, Any], + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]], + key: Optional[str], + headers: Dict[str, str], + kwargs: Dict[str, Any], +) -> CompletionResponse: + url = f"{resolved['api_base']}/responses" + payload = responses_payload(resolved, messages, tools, False, kwargs) + system = system_prompt(messages) + + if system: + payload["instructions"] = system + + hdrs = {"Authorization": f"Bearer {key}", "Content-Type": "application/json", **headers} + + async with make_client(timeout=DEFAULT_TIMEOUT, verify=VERIFY_SSL) as client: + resp = await client.post(url, json=payload, headers=hdrs) + resp.raise_for_status() + data = resp.json() + + return normalize_responses_response(data, resolved["model"]) + + +async def responses_stream( + resolved: Dict[str, Any], + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]], + key: Optional[str], + headers: Dict[str, str], + kwargs: Dict[str, Any], +) -> AsyncIterator[CompletionChunk]: + url = f"{resolved['api_base']}/responses" + payload = responses_payload(resolved, messages, tools, True, kwargs) + system = system_prompt(messages) + + if system: + payload["instructions"] = system + + hdrs = {"Authorization": f"Bearer {key}", "Content-Type": "application/json", **headers} + + _reset_stream_state() + + async with make_client(timeout=DEFAULT_TIMEOUT, verify=VERIFY_SSL) as client: + async with client.stream("POST", url, json=payload, headers=hdrs) as resp: + resp.raise_for_status() + async for json_obj in sse_json_lines(resp): + chunk = parse_responses_chunk(json_obj) + + if chunk: + yield chunk + + +def normalize_responses_response(data: Dict[str, Any], model: str) -> CompletionResponse: + parts: List[Part] = [] + provider_fields: Dict[str, Any] = {} + + for item in data.get("output") or []: + item_type = item.get("type") + + if item_type == "reasoning": + for block in item.get("content") or []: + if block.get("type") == "reasoning_text" and block.get("text"): + parts.append(ReasoningPart(text=block["text"])) + + for block in item.get("summary") or []: + if block.get("type") == "summary_text" and block.get("text"): + parts.append(ReasoningPart(text=block["text"])) + + # Encrypted reasoning (e.g. meta muse-spark): an opaque blob that + # must be echoed back verbatim on the next turn. Stash the whole + # item (id + summary + encrypted_content) under + # provider_specific_fields["reasoning_items"] and mark the message + # redacted instead of fabricating placeholder text. + if item.get("encrypted_content"): + provider_fields.setdefault("reasoning_items", []).append(item) + parts.append(ReasoningPart(redacted=True)) + + elif item_type == "function_call": + args_raw = item.get("arguments") or "{}" + + try: + args = json.loads(args_raw) if isinstance(args_raw, str) else args_raw + except json.JSONDecodeError: + args = {"_raw": args_raw} + + parts.append( + ToolCallPart( + name=item.get("name", ""), + arguments=args, + tool_call_id=item.get("call_id") or item.get("id"), + ) + ) + + elif item_type == "message": + for block in item.get("content") or []: + if block.get("type") == "output_text" and block.get("text"): + parts.append(TextPart(text=block["text"])) + + message = parts_message_to_message( + PartsMessage(role="assistant", parts=parts, provider_metadata=provider_fields) + ) + finish = data.get("status") + + if finish == "completed": + finish = "stop" + + usage = _build_usage(data.get("usage") or {}) + return CompletionResponse( + id=data.get("id"), + model=model, + choices=[Choice(index=0, message=message, finish_reason=finish)], + usage=usage, + provider_specific_fields=provider_fields, + ) + + +def parse_responses_chunk(data: Dict[str, Any]) -> Optional[CompletionChunk]: + """Parse one responses-API SSE event into a normalized chunk.""" + evt = data.get("type") or "" + chunk = CompletionChunk() + + if evt == "response.output_text.delta": + chunk.text = data.get("delta") or "" + return chunk + + if evt == "response.reasoning_summary_text.delta": + chunk.reasoning = data.get("delta") or "" + return chunk + + if evt == "response.reasoning_text.delta": + chunk.reasoning = data.get("delta") or "" + return chunk + + if evt == "response.output_item.added": + _on_output_item_added(data.get("item") or {}, data.get("output_index")) + return None + + if evt == "response.output_item.done": + _on_output_item_done(data.get("item") or {}, data.get("output_index")) + return None + + if evt == "response.function_call_arguments.delta": + # OpenAI correlates deltas via item_id (== the function_call item id). + # Copilot sends a *rotating* opaque item_id on every delta event, so + # fall back to output_index (stable per item in the output array). + meta = _stream_state["tool_items"].get(data.get("item_id") or "") or {} + + if not meta: + meta = _stream_state["tool_items"].get(data.get("output_index")) or {} + + chunk.tool_calls = [ + ToolCall( + id=meta.get("call_id") or "", + name=meta.get("name") or "", + arguments={"_fragment": data.get("delta") or ""}, + ) + ] + return chunk + + if evt == "response.completed": + resp = data.get("response") or {} + chunk.finish_reason = "stop" if resp.get("status") == "completed" else resp.get("status") + + _capture_final_reasoning(resp) + + # Reasoning metadata rides on the authoritative completed event; the + # per-event ciphertext differs, so only the final items are emitted. + if _stream_state["reasoning_items"]: + chunk.provider_specific_fields = { + "reasoning_items": list(_stream_state["reasoning_items"].values()) + } + + chunk.usage = _build_usage(resp.get("usage") or {}) + return chunk + + return None + + +def _reset_stream_state() -> None: + """Clear per-stream correlation state before a new SSE loop.""" + _stream_state["tool_items"] = {} + _stream_state["reasoning_items"] = {} + + +def _on_output_item_added(item: Dict[str, Any], output_index: Optional[int] = None) -> None: + """Register function_call / reasoning items when they first appear. + + ``response.function_call_arguments.delta`` events carry the item_id on + OpenAI but a rotating opaque item_id on Copilot, so the call_id + name are + registered under BOTH the item id and its ``output_index`` to let the delta + handler correlate either way. + """ + item_type = item.get("type") + + if item_type == "function_call": + meta = { + "call_id": item.get("call_id") or item.get("id"), + "name": item.get("name") or "", + } + + if item.get("id"): + _stream_state["tool_items"][item["id"]] = meta + + if output_index is not None: + _stream_state["tool_items"][output_index] = meta + + elif item_type == "reasoning": + _stream_state["reasoning_items"][item.get("id")] = item + + +def _on_output_item_done(item: Dict[str, Any], output_index: Optional[int] = None) -> None: + """Refresh item metadata from the ``output_item.done`` event. + + The done event carries a fuller function_call (call_id) and reasoning item + (encrypted_content may only be populated here) than the added event. + """ + item_type = item.get("type") + + if item_type == "function_call": + meta = { + "call_id": item.get("call_id") or item.get("id"), + "name": item.get("name") or "", + } + + if item.get("id"): + _stream_state["tool_items"][item["id"]] = meta + + if output_index is not None: + _stream_state["tool_items"][output_index] = meta + + elif item_type == "reasoning": + _stream_state["reasoning_items"][item.get("id")] = item + + +def _capture_final_reasoning(resp: Dict[str, Any]) -> None: + """Overwrite per-event reasoning ciphertext with the authoritative items. + + The ``encrypted_content`` observed on ``output_item.added``/``done`` can + differ from the final value; the ``output`` embedded in the + ``response.completed`` event is authoritative. + """ + items = [ + item + for item in resp.get("output") or [] + if item.get("type") == "reasoning" and item.get("encrypted_content") + ] + _stream_state["reasoning_items"] = {item.get("id"): item for item in items} + + +def _stashed_reasoning_items(msg: Dict[str, Any]) -> List[Dict[str, Any]]: + """Collect stashed reasoning items from a stored assistant message. + + The normalizer stores whole ``reasoning`` items under + ``provider_specific_fields["reasoning_items"]``; ``helpers.requests`` may + hoist them to the message top level (``msg["reasoning_items"]``) before + this point, so both locations are honoured. + """ + raw = (msg.get("provider_specific_fields") or {}).get("reasoning_items") + if raw is None: + raw = msg.get("reasoning_items") + + if isinstance(raw, dict): + return [raw] + + if isinstance(raw, list): + return [item for item in raw if isinstance(item, dict)] + + return [] + + +def _reasoning_input_item(item: Dict[str, Any]) -> Dict[str, Any]: + """Build the responses-API input item that replays a prior reasoning item.""" + return { + "type": "reasoning", + "id": item.get("id"), + "encrypted_content": item.get("encrypted_content"), + "summary": item.get("summary") or [], + } + + +def _build_usage(usage_raw: Dict[str, Any]) -> Usage: + """Map responses-API usage fields onto the normalized ``Usage`` shape. + + ``input_tokens_details``/``output_tokens_details`` are preserved wholesale + so ``output_tokens_details.reasoning_tokens`` lands on + ``completion_tokens_details`` for reasoning-token accounting. + """ + input_details = usage_raw.get("input_tokens_details") or {} + output_details = usage_raw.get("output_tokens_details") or {} + return Usage( + prompt_tokens=usage_raw.get("input_tokens"), + completion_tokens=usage_raw.get("output_tokens"), + total_tokens=usage_raw.get("total_tokens"), + prompt_cache_hit_tokens=input_details.get("cached_tokens"), + prompt_tokens_details=input_details, + completion_tokens_details=output_details, + ) + + +__all__ = [ + "responses_payload", + "to_responses_input", + "responses_tool", + "responses_complete", + "responses_stream", + "normalize_responses_response", + "parse_responses_chunk", +] diff --git a/cecli/helpers/llms/formatters/__init__.py b/cecli/helpers/llms/formatters/__init__.py new file mode 100644 index 00000000000..19ea095b00d --- /dev/null +++ b/cecli/helpers/llms/formatters/__init__.py @@ -0,0 +1,38 @@ +"""Sectional per-domain formatters for the llms package. + +Mirrors ``cecli/helpers/model_config/formatters/``: each module is a +cross-cutting concern (reasoning, thinking) exporting provider-specific +functions plus a ``format_*`` dispatcher selected by provider/route/record. +""" + +from __future__ import annotations + +from .reasoning import ( + anthropic_reasoning, + format_reasoning, + gemini_reasoning, + generic_reasoning, + meta_reasoning, + openrouter_reasoning, +) +from .thinking import ( + anthropic_5_thinking, + anthropic_thinking, + format_thinking, + gemini_thinking, + noop, +) + +__all__ = [ + "format_reasoning", + "generic_reasoning", + "openrouter_reasoning", + "anthropic_reasoning", + "gemini_reasoning", + "meta_reasoning", + "format_thinking", + "noop", + "gemini_thinking", + "anthropic_5_thinking", + "anthropic_thinking", +] diff --git a/cecli/helpers/llms/formatters/reasoning.py b/cecli/helpers/llms/formatters/reasoning.py new file mode 100644 index 00000000000..20ad50b91d8 --- /dev/null +++ b/cecli/helpers/llms/formatters/reasoning.py @@ -0,0 +1,91 @@ +"""Sectional reasoning formatters for the llms package. + +Mirrors ``cecli/helpers/model_config/formatters/reasoning.py``: a +:func:`format_reasoning` dispatcher selects the per-provider reasoning +extractor for a model. The extractors post-process a normalized message / +response dict to pull reasoning out of the provider-specific shapes. +""" + +from __future__ import annotations + +from typing import Any, Callable, Dict, Optional + +from ..identifiers import is_anthropic, is_gemini, is_meta, is_openrouter +from ..utils import extract_reasoning + + +def format_reasoning(provider: Optional[str], route: str, record: Optional[Dict]) -> Callable: + """Return the reasoning extractor for a model (default: generic).""" + if is_gemini(provider, route, record): + return gemini_reasoning + + if is_anthropic(provider, route, record): + return anthropic_reasoning + + if is_meta(provider, route, record): + return meta_reasoning + + if is_openrouter(provider, route, record): + return openrouter_reasoning + + return generic_reasoning + + +def generic_reasoning(msg: Dict[str, Any]) -> str: + """Generic extraction: reasoning_content / reasoning / reasoning_details.""" + return extract_reasoning(msg) + + +def openrouter_reasoning(msg: Dict[str, Any]) -> str: + """OpenRouter puts reasoning in ``reasoning`` + ``reasoning_details``. + + The generic extractor already handles those shapes; kept as a named + provider hook so future OpenRouter-specific shapes can be added here. + """ + return extract_reasoning(msg) + + +def anthropic_reasoning(block: Dict[str, Any]) -> str: + """Anthropic thinking blocks: text, or a signature-bearing encrypted block.""" + if block.get("type") != "thinking": + return "" + + thinking_text = block.get("thinking") or "" + + if thinking_text.strip(): + return thinking_text + + if block.get("signature"): + return "[encrypted thinking block present]" + + return "" + + +def gemini_reasoning(part: Dict[str, Any]) -> str: + """Gemini ``thought`` parts carry the reasoning text.""" + if part.get("thought") and "text" in part: + return part["text"] + + return "" + + +def meta_reasoning(item: Dict[str, Any]) -> str: + """Meta responses-mode reasoning: encrypted_content is opaque. + + Returns the placeholder marker when only an encrypted blob is present so + callers can tell reasoning happened (usage shows reasoning_tokens). + """ + if item.get("encrypted_content"): + return "[encrypted reasoning present]" + + return "" + + +__all__ = [ + "format_reasoning", + "generic_reasoning", + "openrouter_reasoning", + "anthropic_reasoning", + "gemini_reasoning", + "meta_reasoning", +] diff --git a/cecli/helpers/llms/formatters/thinking.py b/cecli/helpers/llms/formatters/thinking.py new file mode 100644 index 00000000000..2e513a9a27d --- /dev/null +++ b/cecli/helpers/llms/formatters/thinking.py @@ -0,0 +1,76 @@ +"""Sectional thinking-config formatters for the llms package. + +Mirrors ``cecli/helpers/model_config/formatters/thinking.py``: a +:func:`format_thinking` dispatcher selects the per-provider thinking +configuration builder, mapping the generic reasoning-effort/thinking shape +onto the provider's request field (Gemini thinkingConfig, Anthropic +output_config / thinking block, chat reasoning_effort). +""" + +from __future__ import annotations + +from typing import Any, Callable, Dict, Optional + +from ..identifiers import is_anthropic, is_claude_5_plus, is_gemini + + +def format_thinking(provider: Optional[str], route: str, record: Optional[Dict]) -> Callable: + """Return the thinking-config builder for a model (default: noop).""" + if is_gemini(provider, route, record): + return gemini_thinking + + if is_anthropic(provider, route, record) and is_claude_5_plus(provider, route, record): + return anthropic_5_thinking + + if is_anthropic(provider, route, record): + return anthropic_thinking + + return noop + + +def noop(payload: Dict[str, Any], api_block: Dict[str, Any]) -> Dict[str, Any]: + """Default: leave the payload untouched.""" + return payload + + +def gemini_thinking(payload: Dict[str, Any], api_block: Dict[str, Any]) -> Dict[str, Any]: + """Gemini: map reasoning_effort/thinking onto ``generationConfig.thinkingConfig``.""" + from ..domains.gemini import gemini_thinking_config + + gen_config = payload.setdefault("generationConfig", {}) + + if api_block.get("reasoning_effort"): + gen_config["thinkingConfig"] = gemini_thinking_config( + {"route": payload.get("model", "")}, api_block["reasoning_effort"] + ) + elif api_block.get("thinking"): + gen_config["thinkingConfig"] = { + "thinkingBudget": api_block["thinking"].get("budget_tokens", 8192) + } + + return payload + + +def anthropic_5_thinking(payload: Dict[str, Any], api_block: Dict[str, Any]) -> Dict[str, Any]: + """Claude 5+: adaptive thinking via ``output_config.effort``.""" + if api_block.get("reasoning_effort"): + payload["output_config"] = {"effort": api_block["reasoning_effort"]} + + return payload + + +def anthropic_thinking(payload: Dict[str, Any], api_block: Dict[str, Any]) -> Dict[str, Any]: + """Pre-Claude-5: the ``thinking`` block (type enabled + budget).""" + if api_block.get("thinking"): + payload["thinking"] = api_block["thinking"] + + return payload + + +__all__ = [ + "format_thinking", + "noop", + "gemini_thinking", + "anthropic_5_thinking", + "anthropic_thinking", +] diff --git a/cecli/helpers/llms/identifiers.py b/cecli/helpers/llms/identifiers.py new file mode 100644 index 00000000000..cd6f655ee03 --- /dev/null +++ b/cecli/helpers/llms/identifiers.py @@ -0,0 +1,93 @@ +"""Provider / API-family identifier helpers for the llms package. + +These predicates classify a model by provider and route from its name and the +resolved config, mirroring ``cecli/helpers/model_config/identifiers.py``. +Centralizing them keeps the domain adapters and provider adapters free of +repeated provider matching. +""" + +from __future__ import annotations + +import re +from typing import Any, Dict, Optional + + +def _haystack(provider: Optional[str], route: str, record: Optional[Dict[str, Any]]) -> str: + """Lowercased, space-joined provider/route/record-provider for matching.""" + provider = (provider or "").lower() + route = (route or "").lower() + record_provider = ((record or {}).get("litellm_provider") or "").lower() + return " ".join([provider, route, record_provider]) + + +def is_anthropic(provider: Optional[str], route: str, record: Optional[Dict[str, Any]]) -> bool: + """True when the model is an Anthropic-family model (Claude).""" + haystack = _haystack(provider, route, record) + return "anthropic" in haystack or "claude" in (route or "").lower() + + +def is_gemini(provider: Optional[str], route: str, record: Optional[Dict[str, Any]]) -> bool: + """True when the model is a Gemini-series model.""" + return "gemini" in _haystack(provider, route, record) + + +def is_github_copilot( + provider: Optional[str], route: str, record: Optional[Dict[str, Any]] +) -> bool: + """True when the model is served through GitHub Copilot.""" + provider = (provider or "").lower() + record_provider = ((record or {}).get("litellm_provider") or "").lower() + return provider == "github_copilot" or record_provider == "github_copilot" + + +def is_meta(provider: Optional[str], route: str, record: Optional[Dict[str, Any]]) -> bool: + """True when the model is a Meta-provider model.""" + provider = (provider or "").lower() + record_provider = ((record or {}).get("litellm_provider") or "").lower() + return provider == "meta" or record_provider == "meta" + + +def is_openrouter(provider: Optional[str], route: str, record: Optional[Dict[str, Any]]) -> bool: + """True when the model is served through OpenRouter.""" + provider = (provider or "").lower() + record_provider = ((record or {}).get("litellm_provider") or "").lower() + return provider == "openrouter" or record_provider == "openrouter" + + +def is_claude_5_plus(provider: Optional[str], route: str, record: Optional[Dict[str, Any]]) -> bool: + """True for Claude 5+ models, which use adaptive thinking + output_config. + + Claude 5+ does not accept ``thinking.type.enabled``; thinking is controlled + via ``thinking.type.adaptive`` and ``output_config.effort``. + """ + route = (route or "").lower() + match = re.search(r"claude[^\d]*(\d+)", route) + + if not match: + return False + + return int(match.group(1)) >= 5 + + +def gpt_version(route: str) -> float: + """Return the leading ``gpt-`` model version, or 0 when not a gpt model. + + e.g. ``gpt-5.6-luna`` -> 5.6, ``gpt-5`` -> 5, ``claude-3`` -> 0. + """ + match = re.match(r"^gpt-(\d+(?:\.\d+)?)", (route or "").lower()) + + if not match: + return 0 + + return float(match.group(1)) + + +__all__ = [ + "is_anthropic", + "is_gemini", + "is_github_copilot", + "is_meta", + "is_openrouter", + "is_claude_5_plus", + "gpt_version", +] diff --git a/cecli/helpers/llms/litellm_compat.py b/cecli/helpers/llms/litellm_compat.py new file mode 100644 index 00000000000..aa9976e7513 --- /dev/null +++ b/cecli/helpers/llms/litellm_compat.py @@ -0,0 +1,1104 @@ +"""LiteLLM-shaped compat facade backed by the litellm-free ``cecli.helpers.llms`` dispatcher. + +cecli historically routed every model request through ``litellm.acompletion`` and +consumed litellm's response objects (``ModelResponse``, streaming chunks, +``types.utils`` tool-call types, the ``litellm.*Error`` exception taxonomy, +``model_cost``, ``encode``/``token_counter``, ``validate_environment`` and +``transcription``). + +This module replaces that dependency with a thin, lazily-loaded facade that +keeps the same public attribute surface while delegating the actual HTTP work +to :mod:`cecli.helpers.llms` (a ~35 MB import footprint vs litellm's ~205 MB). + +The shims below are intentionally **mutable dataclasses**: cecli's callers +reassign fields in place (``response.usage = ...``, ``message.tool_calls = ...``, +``chunk._hidden_params["created_at"] = ...``). +""" + +from __future__ import annotations + +import asyncio +import json +import os +import warnings +from dataclasses import dataclass, field +from types import SimpleNamespace +from typing import Any, Dict, List, Optional + +import httpx + +from cecli.dump import dump # noqa: F401 + +warnings.filterwarnings("ignore", category=UserWarning, module="pydantic") + +SITE_URL = "https://cecli.dev" +APP_NAME = "cecli" + +os.environ["OR_SITE_URL"] = SITE_URL +os.environ["OR_APP_NAME"] = APP_NAME + + +# --------------------------------------------------------------------------- +# Litellm-shaped response shims (mutable dataclasses) +# --------------------------------------------------------------------------- + + +@dataclass +class Function: + """Tool-call function payload (litellm ``types.utils.Function`` shape).""" + + name: Optional[str] = None + arguments: str = "" + + def to_dict(self) -> Dict[str, Any]: + """Serialize to the wire-format function dict.""" + return {"name": self.name, "arguments": self.arguments} + + +@dataclass +class ChatCompletionMessageToolCall: + """A tool call attached to a message or a stream delta.""" + + id: Optional[str] = None + type: Optional[str] = "function" + function: Optional[Function] = None + index: Optional[int] = None + provider_specific_fields: Optional[Dict[str, Any]] = None + + def __post_init__(self) -> None: + if isinstance(self.function, dict): + self.function = _coerce_function(self.function) + + def to_dict(self) -> Dict[str, Any]: + """Serialize to the wire-format tool-call dict (id/type/index/function).""" + return _tool_call_to_dict(self) + + +@dataclass +class Message: + role: str = "assistant" + content: Optional[str] = None + tool_calls: List[ChatCompletionMessageToolCall] = field(default_factory=list) + function_call: Optional[Dict[str, Any]] = None + reasoning_content: Optional[str] = None + reasoning: Optional[str] = None + reasoning_redacted: bool = False + provider_specific_fields: Dict[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + self.tool_calls = [_coerce_tool_call(tc) for tc in self.tool_calls or []] + + def to_dict(self) -> Dict[str, Any]: + """Serialize to the wire-format message dict.""" + return _message_to_dict(self) + + +@dataclass +class Choices: + index: int = 0 + message: Optional[Message] = None + finish_reason: Optional[str] = None + + def __post_init__(self) -> None: + if isinstance(self.message, dict): + self.message = _coerce_message(self.message) + + +@dataclass +class Usage: + prompt_tokens: Optional[int] = None + completion_tokens: Optional[int] = None + total_tokens: Optional[int] = None + # Provider cache/usage details preserved for token & cost logging. + prompt_cache_hit_tokens: Optional[int] = None + cache_read_input_tokens: Optional[int] = None + cache_creation_input_tokens: Optional[int] = None + prompt_tokens_details: Optional[Dict[str, Any]] = None + completion_tokens_details: Optional[Dict[str, Any]] = None + + +@dataclass +class Delta: + role: Optional[str] = None + content: Optional[str] = None + tool_calls: List[ChatCompletionMessageToolCall] = field(default_factory=list) + function_call: Optional[Dict[str, Any]] = None + reasoning_content: Optional[str] = None + reasoning: Optional[str] = None + provider_specific_fields: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class StreamChoice: + index: int = 0 + delta: Optional[Delta] = None + finish_reason: Optional[str] = None + + +@dataclass +class StreamChunk: + id: Optional[str] = None + model: Optional[str] = None + choices: List[StreamChoice] = field(default_factory=list) + usage: Optional[Usage] = None + created: Optional[int] = None + _hidden_params: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class ModelResponse: + id: Optional[str] = None + model: Optional[str] = None + object: Optional[str] = None + system_fingerprint: Optional[Any] = None + choices: List[Choices] = field(default_factory=list) + usage: Optional[Usage] = None + created: Optional[int] = None + _hidden_params: Dict[str, Any] = field(default_factory=dict) + provider_specific_fields: Dict[str, Any] = field(default_factory=dict) + + def __init__(self, *args: Any, **kwargs: Any) -> None: + """Dataclass init tolerant of unknown keys + raw dict coercion. + + litellm responses are pydantic models that accept a wide range of keys + (``object``, ``system_fingerprint``, provider-specific extras). Tests + and cecli callers construct ``ModelResponse(**raw_dict)``, so unknown + keys are stashed in ``_extra`` instead of raising and nested raw dicts + are converted into their shim classes. + """ + field_names = [ + "id", + "model", + "object", + "system_fingerprint", + "choices", + "usage", + "created", + "_hidden_params", + "provider_specific_fields", + ] + for name, value in zip(field_names, args): + kwargs.setdefault(name, value) + + self.id = kwargs.pop("id", None) + self.model = kwargs.pop("model", None) + self.object = kwargs.pop("object", None) + self.system_fingerprint = kwargs.pop("system_fingerprint", None) + self.choices = kwargs.pop("choices", []) + self.usage = kwargs.pop("usage", None) + self.created = kwargs.pop("created", None) + self._hidden_params = kwargs.pop("_hidden_params", {}) + self.provider_specific_fields = kwargs.pop("provider_specific_fields", {}) + self._extra = kwargs + + self.choices = [_coerce_choice(c) for c in self.choices or []] + if isinstance(self.usage, dict): + self.usage = _coerce_usage(self.usage) + + def model_dump(self) -> Dict[str, Any]: + """Pydantic-style dump consumed by ``base_coder`` consolidation.""" + return { + "id": self.id, + "model": self.model, + "created": self.created, + "choices": [_choice_to_dict(c) for c in self.choices], + "usage": _usage_to_dict(self.usage), + } + + +# --------------------------------------------------------------------------- +# Serialization helpers (model_dump shape) +# --------------------------------------------------------------------------- + + +def _choice_to_dict(choice: Choices) -> Dict[str, Any]: + return { + "index": choice.index, + "finish_reason": choice.finish_reason, + "message": _message_to_dict(choice.message) if choice.message else None, + } + + +def _message_to_dict(message: Message) -> Dict[str, Any]: + return { + "role": message.role, + "content": message.content, + "tool_calls": [_tool_call_to_dict(tc) for tc in message.tool_calls] or None, + "function_call": _function_to_dict(message.function_call), + "reasoning_content": message.reasoning_content, + "reasoning_redacted": message.reasoning_redacted, + "provider_specific_fields": message.provider_specific_fields, + } + + +def _tool_call_to_dict(tc: ChatCompletionMessageToolCall) -> Dict[str, Any]: + function = tc.function + return { + "id": tc.id, + "type": tc.type, + "index": tc.index, + "function": { + "name": function.name if function else None, + "arguments": function.arguments if function else "", + }, + "provider_specific_fields": tc.provider_specific_fields, + } + + +def _function_to_dict(fn: Optional[Function]) -> Optional[Dict[str, Any]]: + """Serialize a Function shim to its wire-format dict.""" + if not fn: + return None + + return {"name": fn.name, "arguments": fn.arguments} + + +def _usage_to_dict(usage: Optional[Usage]) -> Optional[Dict[str, Any]]: + if not usage: + return None + return { + "prompt_tokens": usage.prompt_tokens, + "completion_tokens": usage.completion_tokens, + "total_tokens": usage.total_tokens, + "prompt_cache_hit_tokens": getattr(usage, "prompt_cache_hit_tokens", None), + "cache_read_input_tokens": getattr(usage, "cache_read_input_tokens", None), + "cache_creation_input_tokens": getattr(usage, "cache_creation_input_tokens", None), + "prompt_tokens_details": getattr(usage, "prompt_tokens_details", None), + "completion_tokens_details": getattr(usage, "completion_tokens_details", None), + } + + +def _coerce_choice(value: Any) -> Any: + """Convert a raw choice dict (or pass through a Choices object).""" + if isinstance(value, Choices): + return value + if isinstance(value, dict): + return Choices( + index=value.get("index", 0), + message=value.get("message"), + finish_reason=value.get("finish_reason"), + ) + return value + + +def _coerce_message(value: Any) -> Any: + """Convert a raw message dict (or pass through a Message object).""" + if isinstance(value, Message): + return value + if isinstance(value, dict): + reasoning = value.get("reasoning_content") or value.get("reasoning") + return Message( + role=value.get("role", "assistant"), + content=value.get("content"), + tool_calls=value.get("tool_calls") or [], + function_call=value.get("function_call"), + reasoning_content=reasoning, + reasoning=reasoning, + reasoning_redacted=value.get("reasoning_redacted", False), + provider_specific_fields=value.get("provider_specific_fields") or {}, + ) + return value + + +def _coerce_tool_call(value: Any) -> Any: + """Convert a raw tool-call dict (or pass through a tool-call object).""" + if isinstance(value, ChatCompletionMessageToolCall): + return value + if isinstance(value, dict): + return ChatCompletionMessageToolCall( + id=value.get("id"), + type=value.get("type", "function"), + function=value.get("function"), + index=value.get("index"), + provider_specific_fields=value.get("provider_specific_fields"), + ) + return value + + +def _coerce_function(value: Any) -> Any: + """Convert a raw function dict (or pass through a Function object).""" + if isinstance(value, Function): + return value + if isinstance(value, dict): + return Function( + name=value.get("name"), + arguments=value.get("arguments") or "", + ) + return value + + +def _coerce_usage(value: Any) -> Optional[Usage]: + """Convert a raw usage dict (or pass through a Usage object).""" + if isinstance(value, Usage): + return value + if isinstance(value, dict): + return Usage( + prompt_tokens=value.get("prompt_tokens"), + completion_tokens=value.get("completion_tokens"), + total_tokens=value.get("total_tokens"), + prompt_cache_hit_tokens=value.get("prompt_cache_hit_tokens"), + cache_read_input_tokens=value.get("cache_read_input_tokens"), + cache_creation_input_tokens=value.get("cache_creation_input_tokens"), + prompt_tokens_details=value.get("prompt_tokens_details"), + completion_tokens_details=value.get("completion_tokens_details"), + ) + return value + + +# --------------------------------------------------------------------------- +# Litellm-shaped exceptions +# --------------------------------------------------------------------------- + +#: Names exposed on the facade as ``litellm.``. Must match the EXCEPTIONS +#: list in ``cecli/exceptions.py`` exactly (the strict check iterates +#: ``dir(litellm)`` for every name ending in "Error"). +_EXCEPTION_NAMES = [ + "APIConnectionError", + "APIError", + "APIResponseValidationError", + "AuthenticationError", + "AzureOpenAIError", + "BadGatewayError", + "BadRequestError", + "BudgetExceededError", + "ContentPolicyViolationError", + "ContextWindowExceededError", + "ErrorEventError", + "ImageFetchError", + "InternalServerError", + "InvalidRequestError", + "JSONSchemaValidationError", + "NotFoundError", + "OpenAIError", + "PermissionDeniedError", + "RateLimitError", + "RouterRateLimitError", + "ServiceUnavailableError", + "UnprocessableEntityError", + "UnsupportedParamsError", + "Timeout", +] + + +class _FacadeException(Exception): + """Base class for litellm-shaped exceptions raised by the facade.""" + + def __init__( + self, + message: Optional[str] = None, + status_code: Optional[int] = None, + **kwargs: Any, + ) -> None: + super().__init__(message or "") + self.status_code = status_code + + +class APIConnectionError(_FacadeException): + pass + + +class APIError(_FacadeException): + pass + + +class APIResponseValidationError(_FacadeException): + pass + + +class AuthenticationError(_FacadeException): + pass + + +class AzureOpenAIError(_FacadeException): + pass + + +class BadGatewayError(_FacadeException): + pass + + +class BadRequestError(_FacadeException): + pass + + +class BudgetExceededError(_FacadeException): + pass + + +class ContentPolicyViolationError(_FacadeException): + pass + + +class ContextWindowExceededError(_FacadeException): + pass + + +class ErrorEventError(_FacadeException): + pass + + +class ImageFetchError(_FacadeException): + pass + + +class InternalServerError(_FacadeException): + pass + + +class InvalidRequestError(_FacadeException): + pass + + +class JSONSchemaValidationError(_FacadeException): + pass + + +class NotFoundError(_FacadeException): + pass + + +class OpenAIError(_FacadeException): + pass + + +class PermissionDeniedError(_FacadeException): + pass + + +class RateLimitError(_FacadeException): + pass + + +class RouterRateLimitError(_FacadeException): + pass + + +class ServiceUnavailableError(_FacadeException): + pass + + +class UnprocessableEntityError(_FacadeException): + pass + + +class UnsupportedParamsError(_FacadeException): + pass + + +class Timeout(_FacadeException): + pass + + +def _translate_http_error(err: httpx.HTTPStatusError) -> _FacadeException: + """Map an httpx status error to a litellm-shaped facade exception.""" + status = err.response.status_code + + # Streaming responses are not consumed; .text raises ResponseNotRead. + try: + text = err.response.text or "" + except Exception: + text = "" + body = text.lower() + message = text or str(err) + + if status == 400 and any( + token in body for token in ("context", "context_length", "maximum context") + ): + return ContextWindowExceededError(message=message, status_code=status) + + if status in (401, 403): + return AuthenticationError(message=message, status_code=status) + + if status == 404: + return NotFoundError(message=message, status_code=status) + + if status == 429: + return RateLimitError(message=message, status_code=status) + + if status >= 500: + return InternalServerError(message=message, status_code=status) + + return APIError(message=message, status_code=status) + + +# --------------------------------------------------------------------------- +# Package -> shim translation helpers +# --------------------------------------------------------------------------- + + +def _tool_call_from_pkg(index: int, tc: Any) -> ChatCompletionMessageToolCall: + """Convert a ``cecli.helpers.llms.ToolCall`` into a litellm-shaped tool call. + + Streaming tool calls arrive with ``arguments`` set to a ``{"_fragment": + }`` marker (see the domain stream parsers in ``domains/``). + The fragment text must flow through ``Function.arguments`` raw so the + downstream concatenation (``base_coder._build_tool_calls_from_chunks`` and + the facade ``stream_chunk_builder``) can reassemble it into valid JSON. + Empty fragments stay empty so the joiners skip them. + + When the domain also attaches ``arguments["_index"]`` (the provider's + original per-SSE-event tool-call index, see the anthropic stream parser), + that index is used for the shim instead of the enumerate position so + parallel tool calls do not collapse onto index 0. + """ + arguments = tc.arguments + tool_index = index + if isinstance(arguments, dict) and isinstance(arguments.get("_index"), int): + tool_index = arguments["_index"] + if isinstance(arguments, dict) and "_fragment" in arguments: + arguments_str = arguments["_fragment"] or "" + else: + arguments_str = json.dumps(arguments) if arguments else "{}" + return ChatCompletionMessageToolCall( + id=tc.id, + type="function", + index=tool_index, + function=Function( + name=tc.name, + arguments=arguments_str, + ), + ) + + +def _usage_from_pkg(usage: Any) -> Optional[Usage]: + if not usage: + return None + return Usage( + prompt_tokens=usage.prompt_tokens, + completion_tokens=usage.completion_tokens, + total_tokens=usage.total_tokens, + prompt_cache_hit_tokens=getattr(usage, "prompt_cache_hit_tokens", None), + cache_read_input_tokens=getattr(usage, "cache_read_input_tokens", None), + cache_creation_input_tokens=getattr(usage, "cache_creation_input_tokens", None), + prompt_tokens_details=getattr(usage, "prompt_tokens_details", None), + completion_tokens_details=getattr(usage, "completion_tokens_details", None), + ) + + +def _response_shim(resp: Any, model: Optional[str] = None) -> ModelResponse: + """Convert a package ``CompletionResponse`` into a litellm-shaped response.""" + finish_reason = resp.choices[0].finish_reason if resp.choices else None + pkg_message = resp.choices[0].message if resp.choices else None + message = Message( + role="assistant", + content=resp.text or None, + reasoning_content=resp.reasoning or None, + reasoning_redacted=bool(getattr(pkg_message, "reasoning_redacted", False)), + provider_specific_fields=dict(resp.provider_specific_fields or {}), + ) + message.tool_calls = [_tool_call_from_pkg(i, tc) for i, tc in enumerate(resp.tool_calls or [])] + return ModelResponse( + id=resp.id, + model=resp.model or model, + choices=[Choices(index=0, message=message, finish_reason=finish_reason)], + usage=_usage_from_pkg(resp.usage), + created=0, + ) + + +def _chunk_shim(chunk: Any, model: Optional[str] = None) -> StreamChunk: + """Convert a package ``CompletionChunk`` into a litellm-shaped stream chunk.""" + delta = Delta( + content=chunk.text or None, + reasoning_content=chunk.reasoning or None, + ) + # Honor a per-tool-call ``index`` (gemini parallel functionCall parts) and + # fall back to the enumerate position for fragmented providers. + delta.tool_calls = [ + _tool_call_from_pkg( + getattr(tc, "index", None) if getattr(tc, "index", None) is not None else i, + tc, + ) + for i, tc in enumerate(chunk.tool_calls or []) + ] + # Forward provider round-trip metadata (Anthropic thinking blocks, OpenAI + # Responses reasoning items, ...) onto the delta so + # ``base_coder.consolidate_chunks`` persists it on the stored assistant + # message for the next stateless turn. + chunk_psf = getattr(chunk, "provider_specific_fields", None) + if chunk_psf: + delta.provider_specific_fields = dict(chunk_psf) + return StreamChunk( + model=model, + choices=[StreamChoice(index=0, delta=delta, finish_reason=chunk.finish_reason)], + usage=_usage_from_pkg(chunk.usage), + created=0, + ) + + +def _accumulate_tool_call( + tool_calls_dict: Dict[int, Dict[str, Any]], tc: Optional[ChatCompletionMessageToolCall] +) -> None: + """Merge one delta tool-call into an index-keyed accumulation dict.""" + if tc is None: + return + function = tc.function + if function is None: + return + + index = tc.index + if index is None: + index = len(tool_calls_dict) + + entry = tool_calls_dict.setdefault( + index, + { + "id": None, + "name": None, + "type": "function", + "arguments": [], + "provider_specific_fields": {}, + }, + ) + entry["id"] = tc.id or entry["id"] + entry["type"] = tc.type or entry["type"] + entry["name"] = function.name or entry["name"] + if function.arguments: + entry["arguments"].append(function.arguments) + + psf = tc.provider_specific_fields + if not psf: + psf = getattr(function, "provider_specific_fields", None) + if psf and isinstance(psf, dict): + entry["provider_specific_fields"].update(psf) + + +def _finalize_tool_calls( + tool_calls_dict: Dict[int, Dict[str, Any]], +) -> List[ChatCompletionMessageToolCall]: + """Build final tool-call shims from an index-keyed accumulation dict.""" + tool_calls: List[ChatCompletionMessageToolCall] = [] + for index in sorted(tool_calls_dict.keys()): + data = tool_calls_dict[index] + if not (data["id"] and data["name"]): + continue + function = Function(arguments="".join(data["arguments"]) or "{}", name=data["name"]) + params: Dict[str, Any] = { + "id": data["id"], + "function": function, + "type": data["type"] or "function", + } + if data["provider_specific_fields"]: + params["provider_specific_fields"] = data["provider_specific_fields"] + tool_calls.append(ChatCompletionMessageToolCall(**params)) + return tool_calls + + +# --------------------------------------------------------------------------- +# The facade implementation +# --------------------------------------------------------------------------- + + +class _LiteLLMFacade: + """Lazily-built implementation object behind :class:`LazyLiteLLM`. + + Holds litellm-compatible settings/attributes and the translation methods + that delegate the actual HTTP work to :mod:`cecli.helpers.llms`. + """ + + #: litellm-compatible settings (accepted; most are no-ops after the swap). + drop_params = True + disable_streaming_logging = True + suppress_debug_info = True + set_verbose = False + + def __init__(self) -> None: + self.model_cost: Dict[str, Dict[str, Any]] = {} + self.utils = SimpleNamespace(_invalidate_model_cost_lowercase_map=lambda: None) + self.types = SimpleNamespace( + utils=SimpleNamespace( + ModelResponse=ModelResponse, + Choices=Choices, + Message=Message, + ChatCompletionMessageToolCall=ChatCompletionMessageToolCall, + Function=Function, + Delta=Delta, + ) + ) + for name in _EXCEPTION_NAMES: + setattr(self, name, globals()[name]) + + # Top-level litellm-shaped classes (``litellm.ModelResponse`` etc.). + for name in ( + "ModelResponse", + "Choices", + "Message", + "Function", + "ChatCompletionMessageToolCall", + "Usage", + "Delta", + "StreamChoice", + "StreamChunk", + ): + setattr(self, name, globals()[name]) + + # -- completions ------------------------------------------------------ + + async def acompletion(self, **kwargs: Any) -> Any: + """Send a completion through ``cecli.helpers.llms.acompletion``. + + Returns a litellm-shaped :class:`ModelResponse` (non-stream) or an + async generator of :class:`StreamChunk` (stream). + """ + from cecli.helpers.llms import acompletion as dispatch + + model = kwargs.get("model") + messages = kwargs.get("messages", []) + stream = kwargs.get("stream", False) + tools = kwargs.get("tools") + api_base = kwargs.get("api_base") + api_key = kwargs.get("api_key") + + extra_headers = dict(kwargs.get("extra_headers") or {}) + headers = kwargs.get("headers") + if headers: + extra_headers = {**headers, **extra_headers} + + passthrough: Dict[str, Any] = {} + for key in ( + "temperature", + "tool_choice", + "extra_body", + "prompt_cache_key", + "stream_options", + ): + if kwargs.get(key) is not None: + passthrough[key] = kwargs[key] + + max_tokens = kwargs.get("max_tokens") or kwargs.get("max_completion_tokens") + if max_tokens: + passthrough["max_tokens"] = max_tokens + + if stream: + gen = await dispatch( + model=model, + messages=messages, + stream=True, + tools=tools, + api_base=api_base, + api_key=api_key, + extra_headers=extra_headers, + **passthrough, + ) + return self._stream_with_errors(gen, model) + + try: + resp = await dispatch( + model=model, + messages=messages, + stream=False, + tools=tools, + api_base=api_base, + api_key=api_key, + extra_headers=extra_headers, + **passthrough, + ) + except httpx.TimeoutException as err: + raise Timeout(str(err)) from err + except httpx.HTTPStatusError as err: + raise _translate_http_error(err) from err + except httpx.HTTPError as err: + raise APIConnectionError(str(err)) from err + + return _response_shim(resp, model) + + def completion(self, **kwargs: Any) -> Any: + """Synchronous variant of :meth:`acompletion` (runs a fresh loop).""" + return asyncio.run(self.acompletion(**kwargs)) + + async def _stream_with_errors(self, gen: Any, model: Optional[str]) -> Any: + """Wrap a package stream generator, translating httpx errors.""" + try: + async for chunk in gen: + yield _chunk_shim(chunk, model) + except httpx.TimeoutException as err: + raise Timeout(str(err)) from err + except httpx.HTTPStatusError as err: + raise _translate_http_error(err) from err + except httpx.HTTPError as err: + raise APIConnectionError(str(err)) from err + + def stream_chunk_builder( + self, chunks: List[Any], messages: Optional[Any] = None, **kwargs: Any + ) -> ModelResponse: + """Reassemble streaming chunks into a single litellm-shaped response. + + Mirrors ``litellm.stream_chunk_builder``: text/reasoning are joined, + tool calls are accumulated per delta index, and finish_reason/usage + come from the last chunk that carried them. + """ + content_parts: List[str] = [] + reasoning_parts: List[str] = [] + tool_calls_dict: Dict[int, Dict[str, Any]] = {} + finish_reason: Optional[str] = None + usage: Optional[Usage] = None + response_id: Optional[str] = None + model: Optional[str] = None + message_psf: Dict[str, Any] = {} + + for chunk in chunks or []: + if chunk is None: + continue + + if getattr(chunk, "usage", None) is not None: + usage = _usage_from_pkg(chunk.usage) + + if not getattr(chunk, "choices", None): + continue + + choice = chunk.choices[0] + if getattr(choice, "finish_reason", None): + finish_reason = choice.finish_reason + + if getattr(chunk, "id", None) and response_id is None: + response_id = chunk.id + if getattr(chunk, "model", None): + model = chunk.model + + delta = getattr(choice, "delta", None) + if delta is None: + # Non-stream chunk (e.g. model_error_response): carry its message. + message = getattr(choice, "message", None) + if message is not None: + if getattr(message, "content", None): + content_parts.append(message.content) + reasoning_msg = getattr(message, "reasoning_content", None) or getattr( + message, "reasoning", None + ) + if reasoning_msg: + reasoning_parts.append(reasoning_msg) + for tc in getattr(message, "tool_calls", None) or []: + _accumulate_tool_call(tool_calls_dict, tc) + continue + + if getattr(delta, "content", None): + content_parts.append(delta.content) + reasoning_delta = getattr(delta, "reasoning_content", None) or getattr( + delta, "reasoning", None + ) + if reasoning_delta: + reasoning_parts.append(reasoning_delta) + for tc in getattr(delta, "tool_calls", None) or []: + _accumulate_tool_call(tool_calls_dict, tc) + + delta_psf = getattr(delta, "provider_specific_fields", None) + if delta_psf: + for key, value in delta_psf.items(): + if ( + key in message_psf + and isinstance(message_psf[key], list) + and isinstance(value, list) + ): + message_psf[key].extend(value) + elif ( + key in message_psf + and isinstance(message_psf[key], dict) + and isinstance(value, dict) + ): + # Merge dict-valued metadata (e.g. gemini per-call + # function_call_signatures) so parallel tool calls each + # keep their own signature across chunks. + message_psf[key].update(value) + else: + # Copy list values so the source chunk's delta is never + # mutated in place (aliasing it here corrupts the chunk + # stream and double-counts on any second aggregation). + message_psf[key] = list(value) if isinstance(value, list) else value + + message = Message( + role="assistant", + content="".join(content_parts) or None, + tool_calls=_finalize_tool_calls(tool_calls_dict), + reasoning_content="".join(reasoning_parts) or None, + provider_specific_fields=message_psf, + ) + return ModelResponse( + id=response_id, + model=model, + choices=[Choices(index=0, message=message, finish_reason=finish_reason)], + usage=usage, + created=0, + ) + + def completion_cost(self, completion_response: Optional[Any] = None, **kwargs: Any) -> float: + """Estimate cost from usage tokens and cecli's own model metadata.""" + usage = getattr(completion_response, "usage", None) + if not usage: + return 0.0 + + model = getattr(completion_response, "model", None) or kwargs.get("model") + info = self.get_model_info(model) if model else {} + input_cost = info.get("input_cost_per_token") or 0.0 + output_cost = info.get("output_cost_per_token") or 0.0 + return (usage.prompt_tokens or 0) * input_cost + ( + usage.completion_tokens or 0 + ) * output_cost + + # -- model metadata --------------------------------------------------- + + def get_model_info(self, model: Optional[str] = None, **kwargs: Any) -> Dict[str, Any]: + """Resolve model cost/context metadata from cecli's provider manager. + + The generic llm block (mode, ``supports_*`` flags, ...) is merged into + ``Model.info`` by ``Model.__init__`` from the config pipeline, so this + facade only contributes provider-managed info (costs, context windows). + Returns ``{}`` for unknown models, matching the litellm fallback + contract that ``ModelInfoManager.get_model_info`` relies on. + """ + from cecli.helpers.model_providers import ModelProviderManager + + info: Dict[str, Any] = {} + if not model: + return info + + try: + provider_info = ModelProviderManager().get_model_info(model) or {} + info.update({k: v for k, v in provider_info.items() if v is not None}) + except Exception: + pass + + return info + + def model_cost_items(self) -> List[Any]: + """Return the accumulated model-cost entries (cecli's own metadata).""" + return list(self.model_cost.items()) + + # -- token helpers ---------------------------------------------------- + + def encode( + self, model: Optional[str] = None, text: Optional[str] = None, **kwargs: Any + ) -> List[int]: + """Tokenize ``text`` with tiktoken (fallback: cl100k_base).""" + import tiktoken + + try: + enc = tiktoken.encoding_for_model(model) + except Exception: + enc = tiktoken.get_encoding("cl100k_base") + + return enc.encode(text or "") + + def token_counter( + self, model: Optional[str] = None, messages: Optional[Any] = None, **kwargs: Any + ) -> int: + """Count tokens in a list of chat messages (or a single message dict).""" + if isinstance(messages, dict): + messages = [messages] + + total = 0 + for msg in messages or []: + if isinstance(msg, dict): + content = msg.get("content") + else: + content = getattr(msg, "content", None) + + if isinstance(content, str): + total += len(self.encode(model=model, text=content)) + elif isinstance(content, list): + for part in content: + if isinstance(part, dict) and isinstance(part.get("text"), str): + total += len(self.encode(model=model, text=part["text"])) + + return total + + # -- environment validation -------------------------------------------- + + def validate_environment(self, model: Optional[str] = None, **kwargs: Any) -> Dict[str, Any]: + """Return ``{keys_in_environment, missing_keys}`` for a model.""" + from cecli.helpers.model_providers import ModelProviderManager + + if not model: + return {"keys_in_environment": True, "missing_keys": []} + + provider = model.split("/", 1)[0] if "/" in model else None + envs: List[str] = [] + + if provider: + config = ModelProviderManager().get_provider_config(provider) + if config: + envs = list(config.get("api_key_env") or []) + + if not envs and provider: + from cecli.helpers.llms.config import PROVIDER_DEFAULTS + + env = (PROVIDER_DEFAULTS.get(provider) or {}).get("api_key_env") + if env: + envs = [env] + + found = [env for env in envs if os.environ.get(env)] + if found: + return {"keys_in_environment": found, "missing_keys": []} + + return {"keys_in_environment": [], "missing_keys": envs} + + # -- audio ------------------------------------------------------------- + + def transcription( + self, + model: Optional[str] = None, + file: Optional[Any] = None, + prompt: Optional[str] = None, + language: Optional[str] = None, + **kwargs: Any, + ) -> SimpleNamespace: + """Transcribe audio via OpenAI's audio transcriptions API.""" + api_key = os.environ.get("OPENAI_API_KEY") + headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} + data = {"model": model or "whisper-1"} + if prompt: + data["prompt"] = prompt + if language: + data["language"] = language + + with httpx.Client(timeout=600) as client: + resp = client.post( + "https://api.openai.com/v1/audio/transcriptions", + headers=headers, + data=data, + files={"file": ("audio", file, "application/octet-stream")}, + ) + resp.raise_for_status() + payload = resp.json() + + return SimpleNamespace(text=payload.get("text", "")) + + # -- model-cost registry (no-ops) -------------------------------------- + + def add_known_models( + self, model_cost_map: Optional[Dict[str, Any]] = None, **kwargs: Any + ) -> None: + """No-op kept for litellm API compatibility (costs live in cecli metadata).""" + return None + + +# --------------------------------------------------------------------------- +# The lazy proxy +# --------------------------------------------------------------------------- + + +class LazyLiteLLM: + """Proxy that builds the facade lazily on first attribute access.""" + + _lazy_module = None + + def __getattr__(self, name: str) -> Any: + if name.startswith("_"): + raise AttributeError(name) + self._load_litellm() + return getattr(self._lazy_module, name) + + def __dir__(self) -> List[str]: + self._load_litellm() + names = set(dir(type(self))) + names.update(dir(self._lazy_module)) + return sorted(names) + + def _load_litellm(self) -> None: + if self._lazy_module is not None: + return + self._lazy_module = _LiteLLMFacade() + + +litellm = LazyLiteLLM() + +__all__ = ["litellm"] diff --git a/cecli/helpers/llms/pipeline.py b/cecli/helpers/llms/pipeline.py new file mode 100644 index 00000000000..31d95c2da7d --- /dev/null +++ b/cecli/helpers/llms/pipeline.py @@ -0,0 +1,129 @@ +"""acompletion() dispatcher for the llms package. + +Resolves a model's config (provider / API family / base / key), then routes to +the family adapter in :mod:`cecli.helpers.llms.domains`, applying per-provider +hooks (auth, headers, response repair) via :mod:`cecli.helpers.llms.providers`. +Returns a :class:`~cecli.helpers.llms.types.CompletionResponse` (non-stream) or +an async iterator of :class:`~cecli.helpers.llms.types.CompletionChunk` +(stream). +""" + +from __future__ import annotations + +from typing import Any, AsyncIterator, Dict, List, Optional + +from .config import resolve_model_config +from .domains import ( + anthropic_complete, + anthropic_stream, + chat_complete, + chat_stream, + gemini_complete, + gemini_stream, + responses_complete, + responses_stream, +) +from .providers import get_provider_adapter +from .types import CompletionChunk, CompletionResponse + + +async def acompletion( + model: str, + messages: List[Dict[str, Any]], + stream: bool = False, + tools: Optional[List[Dict[str, Any]]] = None, + api_base: Optional[str] = None, + api_key: Optional[str] = None, + extra_headers: Optional[Dict[str, str]] = None, + **kwargs: Any, +) -> Any: + """Send a chat-style completion through the llm-package-backed dispatcher.""" + resolved = resolve_model_config(model) + + if api_base: + resolved["api_base"] = api_base.rstrip("/") + + provider = get_provider_adapter(resolved.get("provider") or "openai") + resolved["api_base"] = provider.resolve_api_base(resolved) + key = provider.resolve_api_key(resolved, api_key) + + family = resolved["family"] + + headers = dict(resolved.get("extra_headers") or {}) + headers.update(extra_headers or {}) + headers = provider.build_headers(resolved, key, family, headers) + + if stream: + gen = _stream_family(family, resolved, messages, tools, key, headers, kwargs) + + return _apply_normalize(gen, provider, family, resolved) + + resp = await _complete_family(family, resolved, messages, tools, key, headers, kwargs) + + return provider.normalize(family, resp, resolved) + + +async def _complete_family( + family: str, + resolved: Dict[str, Any], + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]], + key: Optional[str], + headers: Dict[str, str], + kwargs: Dict[str, Any], +) -> CompletionResponse: + if family == "responses": + return await responses_complete(resolved, messages, tools, key, headers, kwargs) + + if family == "messages": + return await anthropic_complete(resolved, messages, tools, key, headers, kwargs) + + if family == "gemini": + return await gemini_complete(resolved, messages, tools, key, headers, kwargs) + + return await chat_complete(resolved, messages, tools, key, headers, kwargs) + + +async def _stream_family( + family: str, + resolved: Dict[str, Any], + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]], + key: Optional[str], + headers: Dict[str, str], + kwargs: Dict[str, Any], +) -> AsyncIterator[CompletionChunk]: + if family == "responses": + async for chunk in responses_stream(resolved, messages, tools, key, headers, kwargs): + yield chunk + + return + + if family == "messages": + async for chunk in anthropic_stream(resolved, messages, tools, key, headers, kwargs): + yield chunk + + return + + if family == "gemini": + async for chunk in gemini_stream(resolved, messages, tools, key, headers, kwargs): + yield chunk + + return + + async for chunk in chat_stream(resolved, messages, tools, key, headers, kwargs): + yield chunk + + +async def _apply_normalize( + gen: AsyncIterator[CompletionChunk], + provider: Any, + family: str, + resolved: Dict[str, Any], +) -> AsyncIterator[CompletionChunk]: + """Yield each stream chunk through the provider's ``normalize`` hook.""" + async for chunk in gen: + yield provider.normalize(family, chunk, resolved) + + +__all__ = ["acompletion"] diff --git a/cecli/helpers/llms/providers/__init__.py b/cecli/helpers/llms/providers/__init__.py new file mode 100644 index 00000000000..ef7bb5bb017 --- /dev/null +++ b/cecli/helpers/llms/providers/__init__.py @@ -0,0 +1,48 @@ +"""Per-provider custom logic for the llms package. + +Each provider module subclasses :class:`ProviderAdapter` and is registered in +the ``PROVIDER_REGISTRY`` so the pipeline can dispatch per-provider hooks +(auth, headers, response repair) while the domain adapters stay generic. +""" + +from __future__ import annotations + +from typing import Dict, Type + +from .base import ProviderAdapter + +#: Provider slug -> adapter class. Imported lazily to keep startup light +#: (mirrors the LazyLiteLLM deferral in cecli/llm.py). +_PROVIDER_CLASSES: Dict[str, Type[ProviderAdapter]] = {} + + +def _load_registry() -> Dict[str, Type[ProviderAdapter]]: + """Populate and return the provider registry (lazy imports).""" + if _PROVIDER_CLASSES: + return _PROVIDER_CLASSES + + import importlib + import pkgutil + + # Auto-discover every provider module (drop a file, it is registered); + # ``base`` is the abstract base class, not a concrete provider. + for module_info in pkgutil.iter_modules(__path__): + if module_info.name == "base": + continue + + importlib.import_module(f"{__name__}.{module_info.name}") + + for cls in ProviderAdapter.__subclasses__(): + _PROVIDER_CLASSES[cls.provider] = cls + + return _PROVIDER_CLASSES + + +def get_provider_adapter(provider: str) -> ProviderAdapter: + """Return the adapter for ``provider`` (base adapter when unregistered).""" + registry = _load_registry() + + return registry.get(provider, ProviderAdapter)() + + +__all__ = ["ProviderAdapter", "get_provider_adapter"] diff --git a/cecli/helpers/llms/providers/anthropic.py b/cecli/helpers/llms/providers/anthropic.py new file mode 100644 index 00000000000..ecc6d50cc65 --- /dev/null +++ b/cecli/helpers/llms/providers/anthropic.py @@ -0,0 +1,20 @@ +"""Anthropic provider adapter for the llms package. + +The Anthropic /v1/messages family adapter (:mod:`...domains.messages`) already +sets ``x-api-key`` + ``anthropic-version`` internally, so this adapter needs no +header overrides for the standard Anthropic path. The github_copilot provider +(same family, Bearer auth + messages-proxy headers) has its own adapter. +""" + +from __future__ import annotations + +from .base import ProviderAdapter + + +class AnthropicProvider(ProviderAdapter): + """Anthropic: auth handled by the messages domain; registration only.""" + + provider: str = "anthropic" + + +__all__ = ["AnthropicProvider"] diff --git a/cecli/helpers/llms/providers/base.py b/cecli/helpers/llms/providers/base.py new file mode 100644 index 00000000000..cd742023a6e --- /dev/null +++ b/cecli/helpers/llms/providers/base.py @@ -0,0 +1,66 @@ +"""Extensible base shape for per-provider custom logic. + +Each provider module in :mod:`cecli.helpers.llms.providers` subclasses +:class:`ProviderAdapter` and overrides only the hooks it needs (auth, header +injection, response repair, routing overrides). The default implementations +delegate to the generic family adapters in :mod:`cecli.helpers.llms.domains`. +""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + + +class ProviderAdapter: + """Base shape for per-provider request/response customization. + + Concrete providers override the hooks they need: + + - :meth:`resolve_api_base` - endpoint selection (e.g. copilot reads the + authenticated session's ``endpoints.api``). + - :meth:`resolve_api_key` - key source (env, auth cache, oauth refresh). + - :meth:`build_headers` - auth scheme + provider-specific headers. + - :meth:`normalize` - post-process a family-normalized response + (e.g. meta encrypted-reasoning marker). + """ + + #: Provider slug used by the registry (``openai``, ``github_copilot``, ...). + provider: str = "openai" + + def resolve_api_base(self, resolved: Dict[str, Any]) -> str: + """Return the api_base for a resolved config (default: as resolved).""" + return resolved["api_base"] + + def resolve_api_key(self, resolved: Dict[str, Any], api_key: Optional[str]) -> Optional[str]: + """Return the API key for a resolved config (default: env-based).""" + from ..config import get_api_key + + return get_api_key(resolved, api_key) + + def build_headers( + self, + resolved: Dict[str, Any], + key: Optional[str], + family: str, + headers: Dict[str, str], + ) -> Dict[str, str]: + """Return the merged request headers (default: Bearer + content-type).""" + merged = dict(headers) + + if key: + merged.setdefault("Authorization", f"Bearer {key}") + + merged.setdefault("Content-Type", "application/json") + return merged + + def normalize( + self, + family: str, + data: Any, + resolved: Dict[str, Any], + ) -> Any: + """Post-process a normalized response (default: no-op).""" + return data + + +__all__ = ["ProviderAdapter"] diff --git a/cecli/helpers/llms/providers/deepseek.py b/cecli/helpers/llms/providers/deepseek.py new file mode 100644 index 00000000000..9a5fb1f9d5e --- /dev/null +++ b/cecli/helpers/llms/providers/deepseek.py @@ -0,0 +1,20 @@ +"""DeepSeek provider adapter for the llms package. + +DeepSeek speaks OpenAI-compatible /v1/chat/completions with Bearer auth and +returns reasoning via ``delta.reasoning_content`` (also ``reasoning_content`` on +non-streamed choices). The generic :func:`cecli.helpers.llms.utils.extract_reasoning` +already handles that shape, so no overrides are needed here. +""" + +from __future__ import annotations + +from .base import ProviderAdapter + + +class DeepSeekProvider(ProviderAdapter): + """DeepSeek: Bearer auth + reasoning_content extraction.""" + + provider: str = "deepseek" + + +__all__ = ["DeepSeekProvider"] diff --git a/cecli/helpers/llms/providers/gemini.py b/cecli/helpers/llms/providers/gemini.py new file mode 100644 index 00000000000..a1515153b0c --- /dev/null +++ b/cecli/helpers/llms/providers/gemini.py @@ -0,0 +1,37 @@ +"""Gemini provider adapter for the llms package. + +Gemini authenticates via the ``key`` query parameter (or ``X-Goog-Api-Key`` +header), NOT via an ``Authorization: Bearer`` header. The base +:class:`ProviderAdapter` adds a Bearer header whenever a key is present, which +Google rejects with 401 for API keys, so this adapter overrides +:meth:`build_headers` to skip it (the domain adapter passes ``key`` as a query +param itself). +""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +from .base import ProviderAdapter + + +class GeminiProvider(ProviderAdapter): + """Gemini: key via query param; no Authorization header.""" + + provider: str = "gemini" + + def build_headers( + self, + resolved: Dict[str, Any], + key: Optional[str], + family: str, + headers: Dict[str, str], + ) -> Dict[str, str]: + """Return headers without an Authorization header (key is a query param).""" + merged = dict(headers) + + merged.setdefault("Content-Type", "application/json") + return merged + + +__all__ = ["GeminiProvider"] diff --git a/cecli/helpers/llms/providers/github_copilot.py b/cecli/helpers/llms/providers/github_copilot.py new file mode 100644 index 00000000000..bc927eae283 --- /dev/null +++ b/cecli/helpers/llms/providers/github_copilot.py @@ -0,0 +1,228 @@ +"""GitHub Copilot provider: OAuth device flow + disk-cached API key. + +Mirrors litellm's ``Authenticator`` (llms/github_copilot/authenticator.py): +an access token (from device flow) is exchanged for a short-lived Copilot API +key via ``https://api.github.com/copilot_internal/v2/token``; the key is cached +to disk (api-key.json) and refreshed on expiry. The tenant endpoint +(``endpoints.api``) comes from the authenticated session, never a caller- +supplied base (token-leak prevention, matching litellm). +""" + +from __future__ import annotations + +import json +import os +import time +from typing import Any, Dict, Optional +from uuid import uuid4 + +import httpx + +from .base import ProviderAdapter + +COPILOT_DEFAULT_API_BASE = "https://api.githubcopilot.com" +COPILOT_API_KEY_URL = "https://api.github.com/copilot_internal/v2/token" +COPILOT_DEVICE_CODE_URL = "https://github.com/login/device/code" +COPILOT_ACCESS_TOKEN_URL = "https://github.com/login/oauth/access_token" +COPILOT_CLIENT_ID = "Iv1.b507a08c87ecfe98" +COPILOT_TOKEN_DIR = os.path.expanduser("~/.config/litellm/github_copilot") +COPILOT_TIMEOUT = 120.0 + + +class CopilotAuthenticator: + """GitHub Copilot OAuth: device flow + disk-cached API key with refresh.""" + + def __init__(self) -> None: + self.token_dir = os.getenv("GITHUB_COPILOT_TOKEN_DIR", COPILOT_TOKEN_DIR) + self.access_token_file = os.path.join(self.token_dir, "access-token") + self.api_key_file = os.path.join(self.token_dir, "api-key.json") + + def get_api_key(self) -> Optional[str]: + """Return a valid Copilot API key, refreshing from disk if needed.""" + try: + with open(self.api_key_file) as f: + info = json.load(f) + + if info.get("expires_at", 0) > time.time(): + return info.get("token") + except (IOError, json.JSONDecodeError): + pass + + try: + info = self._refresh_api_key() + os.makedirs(self.token_dir, exist_ok=True) + + with open(self.api_key_file, "w") as f: + json.dump(info, f) + + return info.get("token") + except Exception: + return None + + def get_api_base(self) -> Optional[str]: + """Return the tenant-specific Copilot endpoint from the cached key.""" + try: + with open(self.api_key_file) as f: + info = json.load(f) + + return (info.get("endpoints") or {}).get("api") + except (IOError, json.JSONDecodeError): + return None + + def get_access_token(self) -> str: + """Return the cached GitHub access token, or run device-flow login.""" + try: + with open(self.access_token_file) as f: + token = f.read().strip() + + if token: + return token + except IOError: + pass + + return self._device_flow_login() + + def _refresh_api_key(self) -> Dict[str, Any]: + access_token = self.get_access_token() + headers = { + "accept": "application/json", + "editor-version": "vscode/1.85.1", + "editor-plugin-version": "copilot/1.155.0", + "user-agent": "GithubCopilot/1.155.0", + "authorization": f"token {access_token}", + } + resp = httpx.get(COPILOT_API_KEY_URL, headers=headers, timeout=COPILOT_TIMEOUT) + resp.raise_for_status() + data = resp.json() + + if "token" not in data: + raise RuntimeError(f"API key response missing token: {data}") + + return data + + def _device_flow_login(self) -> str: + """GitHub device flow: print verification URI + code, poll for token.""" + resp = httpx.post( + COPILOT_DEVICE_CODE_URL, + headers={"accept": "application/json"}, + json={"client_id": COPILOT_CLIENT_ID, "scope": "read:user"}, + timeout=COPILOT_TIMEOUT, + ) + resp.raise_for_status() + info = resp.json() + + print( # noqa: T201 + f"Please visit {info['verification_uri']} and enter code {info['user_code']} to authenticate.", + flush=True, + ) + + for _ in range(12): + poll = httpx.post( + COPILOT_ACCESS_TOKEN_URL, + headers={"accept": "application/json"}, + json={ + "client_id": COPILOT_CLIENT_ID, + "device_code": info["device_code"], + "grant_type": "urn:ietf:params:oauth:grant-type:device_code", + }, + timeout=COPILOT_TIMEOUT, + ) + poll.raise_for_status() + data = poll.json() + + if "access_token" in data: + os.makedirs(self.token_dir, exist_ok=True) + + with open(self.access_token_file, "w") as f: + f.write(data["access_token"]) + + return data["access_token"] + + time.sleep(5) + + raise RuntimeError("Timed out waiting for user to authorize the device") + + +#: Module-level singleton so config resolution and pipeline share one instance. +_AUTH: Optional[CopilotAuthenticator] = None + + +def _auth() -> CopilotAuthenticator: + global _AUTH + + if _AUTH is None: + _AUTH = CopilotAuthenticator() + + return _AUTH + + +def copilot_api_key() -> Optional[str]: + """Return a valid Copilot API key (refreshing on demand).""" + return _auth().get_api_key() + + +def copilot_api_base() -> str: + """Return the tenant endpoint, falling back to the default base.""" + return _auth().get_api_base() or COPILOT_DEFAULT_API_BASE + + +def copilot_headers(api_key: str, *, messages_proxy: bool = False) -> Dict[str, str]: + """Copilot request headers (Authorization Bearer + VSCode integration).""" + headers: Dict[str, str] = { + "Authorization": f"Bearer {api_key}", + "content-type": "application/json", + "copilot-integration-id": "vscode-chat", + "editor-version": "vscode/1.95.0", + "editor-plugin-version": "copilot-chat/0.26.7", + "user-agent": "GitHubCopilotChat/0.26.7", + "x-request-id": str(uuid4()), + "x-vscode-user-agent-library-version": "electron-fetch", + } + + if messages_proxy: + headers["openai-intent"] = "messages-proxy" + headers["x-interaction-type"] = "messages-proxy" + headers["x-github-api-version"] = "2026-06-01" + headers["anthropic-version"] = "2023-06-01" + else: + headers["openai-intent"] = "conversation-panel" + headers["x-github-api-version"] = "2025-04-01" + + return headers + + +class GithubCopilotProvider(ProviderAdapter): + """Provider adapter for GitHub Copilot (auth + headers + routing).""" + + provider = "github_copilot" + + def resolve_api_base(self, resolved: Dict[str, Any]) -> str: + return copilot_api_base() + + def resolve_api_key(self, resolved: Dict[str, Any], api_key: Optional[str]) -> Optional[str]: + return copilot_api_key() + + def build_headers( + self, + resolved: Dict[str, Any], + key: Optional[str], + family: str, + headers: Dict[str, str], + ) -> Dict[str, str]: + merged = dict(headers) + + if key: + for k, v in copilot_headers(key, messages_proxy=(family == "messages")).items(): + merged.setdefault(k, v) + + merged.setdefault("Content-Type", "application/json") + return merged + + +__all__ = [ + "CopilotAuthenticator", + "copilot_api_key", + "copilot_api_base", + "copilot_headers", + "GithubCopilotProvider", +] diff --git a/cecli/helpers/llms/providers/meta.py b/cecli/helpers/llms/providers/meta.py new file mode 100644 index 00000000000..7d3de2f8b76 --- /dev/null +++ b/cecli/helpers/llms/providers/meta.py @@ -0,0 +1,22 @@ +"""Meta (Muse Spark) provider adapter for the llms package. + +Meta speaks the /v1/responses family with Bearer auth. Its reasoning output is +encrypted (``reasoning.encrypted_content`` with an empty ``summary``); the +responses domain normalizer already marks that as reasoning-present (mirroring +the Anthropic encrypted-thinking handling), so this adapter is registration-only +for now. A future override of :meth:`~ProviderAdapter.normalize` could post-process +the marker without touching the domain. +""" + +from __future__ import annotations + +from .base import ProviderAdapter + + +class MetaProvider(ProviderAdapter): + """Meta: encrypted reasoning handled by the responses domain.""" + + provider: str = "meta" + + +__all__ = ["MetaProvider"] diff --git a/cecli/helpers/llms/providers/openai.py b/cecli/helpers/llms/providers/openai.py new file mode 100644 index 00000000000..b0d76106349 --- /dev/null +++ b/cecli/helpers/llms/providers/openai.py @@ -0,0 +1,20 @@ +"""OpenAI provider adapter for the llms package. + +OpenAI-compatible endpoints (/v1/chat/completions, /v1/responses) use the base +adapter's defaults: ``Authorization: Bearer`` + JSON content-type. This module +exists to document the drop-in extension point and to register the ``openai`` +slug in the provider registry. +""" + +from __future__ import annotations + +from .base import ProviderAdapter + + +class OpenAIProvider(ProviderAdapter): + """OpenAI: default Bearer auth; no overrides needed.""" + + provider: str = "openai" + + +__all__ = ["OpenAIProvider"] diff --git a/cecli/helpers/llms/providers/openrouter.py b/cecli/helpers/llms/providers/openrouter.py new file mode 100644 index 00000000000..50c8b9a4922 --- /dev/null +++ b/cecli/helpers/llms/providers/openrouter.py @@ -0,0 +1,21 @@ +"""OpenRouter provider adapter for the llms package. + +OpenRouter speaks OpenAI-compatible /v1/chat/completions with Bearer auth, so +the base adapter's defaults apply. Reasoning arrives via ``message.reasoning`` +/ ``message.reasoning_details`` (not ``reasoning_content``); the generic +:func:`cecli.helpers.llms.utils.extract_reasoning` already handles all three +shapes, so no normalize override is needed here. +""" + +from __future__ import annotations + +from .base import ProviderAdapter + + +class OpenRouterProvider(ProviderAdapter): + """OpenRouter: Bearer auth + generic reasoning extraction.""" + + provider: str = "openrouter" + + +__all__ = ["OpenRouterProvider"] diff --git a/cecli/helpers/llms/runtime.py b/cecli/helpers/llms/runtime.py new file mode 100644 index 00000000000..81ddac31438 --- /dev/null +++ b/cecli/helpers/llms/runtime.py @@ -0,0 +1,43 @@ +"""Runtime knobs for the llms package (import-light, no heavy deps). + +The dispatcher and domain adapters read :data:`VERIFY_SSL` when constructing +httpx clients so ``--no-verify-ssl`` keeps working after the litellm swap +(litellm previously patched ``client_session``/``aclient_session`` globals). +""" + +from __future__ import annotations + +import ssl +from typing import Any + +import httpx + +#: Global TLS verification flag; set False for ``--no-verify-ssl``. +VERIFY_SSL = True + + +def set_verify_ssl(verify: bool) -> None: + """Set whether outbound httpx clients verify TLS certificates.""" + global VERIFY_SSL + + VERIFY_SSL = bool(verify) + + +def make_client(timeout: float, **kwargs: Any) -> httpx.AsyncClient: + """Create an httpx AsyncClient, retrying once on the OpenSSL first-init flake. + + On some platforms (observed: WSL2 + OpenSSL 3.5 + Python 3.14) the very + first ``ssl.create_default_context(cafile=...)`` in a fresh process can + fail with ``ssl.SSLError`` (``[CONF: MODULE_INITIALIZATION_ERROR]`` / + "unknown error (0x0)") because the OpenSSL CONF module races its lazy + initialization. A second attempt succeeds. Retrying keeps per-request + httpx clients reliable whether or not truststore has been injected. + """ + try: + return httpx.AsyncClient(timeout=timeout, **kwargs) + + except ssl.SSLError: + return httpx.AsyncClient(timeout=timeout, **kwargs) + + +__all__ = ["VERIFY_SSL", "make_client", "set_verify_ssl"] diff --git a/cecli/helpers/llms/types.py b/cecli/helpers/llms/types.py new file mode 100644 index 00000000000..89b5c10c8b4 --- /dev/null +++ b/cecli/helpers/llms/types.py @@ -0,0 +1,209 @@ +"""Normalized response objects (litellm-shaped). + +These dataclasses are the stable output shape of :func:`cecli.helpers.llms.acompletion` +across all four API families. They intentionally mirror the litellm +``ModelResponse``/``ChatCompletionMessage`` surface that cecli consumes so the +``LazyLiteLLM`` swap in ``cecli/llm.py`` keeps the same public attribute shape. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + + +@dataclass +class ToolCall: + id: str + name: str + arguments: Dict[str, Any] # parsed JSON + #: Provider-specific stream index (e.g. Gemini parallel functionCall parts + #: streamed in separate SSE chunks get distinct indices so they do not + #: collapse onto index 0 in the litellm shim). None for single-call chunks. + index: Optional[int] = None + + +@dataclass +class Message: + role: str + content: Optional[str] = None + tool_calls: List[ToolCall] = field(default_factory=list) + reasoning_content: Optional[str] = None + reasoning_redacted: bool = False + provider_specific_fields: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class Choice: + index: int + message: Message + finish_reason: Optional[str] = None + + +@dataclass +class Usage: + prompt_tokens: Optional[int] = None + completion_tokens: Optional[int] = None + total_tokens: Optional[int] = None + # Provider cache/usage details preserved for token & cost logging + # (deepseek/gemini cached tokens, anthropic cache_read/creation, openai details). + prompt_cache_hit_tokens: Optional[int] = None + cache_read_input_tokens: Optional[int] = None + cache_creation_input_tokens: Optional[int] = None + prompt_tokens_details: Optional[Dict[str, Any]] = None + completion_tokens_details: Optional[Dict[str, Any]] = None + + +@dataclass +class CompletionResponse: + id: Optional[str] = None + model: Optional[str] = None + choices: List[Choice] = field(default_factory=list) + usage: Optional[Usage] = None + provider_specific_fields: Dict[str, Any] = field(default_factory=dict) + + @property + def text(self) -> str: + parts = [c.message.content or "" for c in self.choices if c.message.content] + return "\n".join(parts) + + @property + def reasoning(self) -> str: + parts = [ + c.message.reasoning_content or "" for c in self.choices if c.message.reasoning_content + ] + return "\n".join(parts) + + @property + def tool_calls(self) -> List[ToolCall]: + calls: List[ToolCall] = [] + + for c in self.choices: + calls.extend(c.message.tool_calls) + + return calls + + +@dataclass +class CompletionChunk: + """Normalized streaming chunk.""" + + text: str = "" + reasoning: str = "" + tool_calls: List[ToolCall] = field(default_factory=list) + finish_reason: Optional[str] = None + usage: Optional[Usage] = None + provider_specific_fields: Dict[str, Any] = field(default_factory=dict) + + +# --------------------------------------------------------------------------- +# Parts model — the common underlying shape across all four API families +# --------------------------------------------------------------------------- +# Mirrors simonw/LLM's parts model (llm/parts.py): every native response is +# normalized into an ordered list of Parts on a PartsMessage, so chat, +# responses, gemini and messages all expose the SAME underlying shape and can +# round-trip provider metadata statelessly (send the entire conversation each +# turn). ``provider_metadata`` carries opaque provider data that must be echoed +# back on the next request (Anthropic thinking signatures, OpenAI Responses +# encrypted_content, Gemini thoughtSignature). + + +@dataclass +class Part: + """Base class for all parts. The role lives on the enclosing PartsMessage.""" + + provider_metadata: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class TextPart(Part): + text: str = "" + + +@dataclass +class ReasoningPart(Part): + """Reasoning/thinking tokens. + + ``redacted=True, text=""`` is the opaque-reasoning marker: the provider + reports reasoning happened but withholds the content (the token total lives + on ``Usage.completion_tokens_details["reasoning_tokens"]``). + """ + + text: str = "" + redacted: bool = False + + +@dataclass +class ToolCallPart(Part): + """A request by the model to call a tool.""" + + name: str = "" + arguments: Dict[str, Any] = field(default_factory=dict) + tool_call_id: Optional[str] = None + server_executed: bool = False + + +@dataclass +class ToolResultPart(Part): + """The result of a tool call.""" + + name: str = "" + output: str = "" + tool_call_id: Optional[str] = None + server_executed: bool = False + exception: Optional[str] = None + + +@dataclass +class PartsMessage: + """A single turn in the common conversation shape: role + ordered parts.""" + + role: str + parts: List[Part] = field(default_factory=list) + provider_metadata: Dict[str, Any] = field(default_factory=dict) + + +def parts_message_to_message(pm: PartsMessage) -> Message: + """Convert a PartsMessage into the litellm-shaped Message adapter. + + The parts model is the canonical shape produced by the domain normalizers; + this adapter feeds cecli's existing litellm-shaped consumers + (``litellm_compat`` / ``base_coder``) unchanged. + """ + text = "\n".join(p.text for p in pm.parts if isinstance(p, TextPart) and p.text) + reasoning = "\n".join(p.text for p in pm.parts if isinstance(p, ReasoningPart) and p.text) + redacted = any(isinstance(p, ReasoningPart) and p.redacted for p in pm.parts) + tool_calls = [ + ToolCall( + id=p.tool_call_id or f"call_{i}", + name=p.name, + arguments=p.arguments, + ) + for i, p in enumerate(pm.parts) + if isinstance(p, ToolCallPart) + ] + return Message( + role=pm.role, + content=text or None, + tool_calls=tool_calls, + reasoning_content=reasoning or None, + reasoning_redacted=redacted, + provider_specific_fields=dict(pm.provider_metadata), + ) + + +__all__ = [ + "ToolCall", + "Message", + "Choice", + "Usage", + "CompletionResponse", + "CompletionChunk", + "Part", + "TextPart", + "ReasoningPart", + "ToolCallPart", + "ToolResultPart", + "PartsMessage", + "parts_message_to_message", +] diff --git a/cecli/helpers/llms/utils.py b/cecli/helpers/llms/utils.py new file mode 100644 index 00000000000..81c1a84f785 --- /dev/null +++ b/cecli/helpers/llms/utils.py @@ -0,0 +1,76 @@ +"""Shared helpers for the llms package. + +SSE parsing, system-prompt extraction, and reasoning-text extraction are used +by multiple domain adapters, so they live here rather than being duplicated. +""" + +from __future__ import annotations + +import json +from typing import Any, AsyncIterator, Dict, List, Optional + +import httpx + + +async def sse_json_lines(resp: httpx.Response) -> AsyncIterator[Dict[str, Any]]: + """Yield parsed JSON from ``data:`` lines of an SSE stream.""" + buffer = "" + + async for raw in resp.aiter_lines(): + line = raw.strip() + + if not line.startswith("data:"): + continue + + payload = line[5:].strip() + + if payload == "[DONE]": + continue + + try: + yield json.loads(payload) + except json.JSONDecodeError: + buffer += payload + + try: + yield json.loads(buffer) + except json.JSONDecodeError: + continue + + +def system_prompt(messages: List[Dict[str, Any]]) -> Optional[str]: + """Join all system messages into a single prompt (or None).""" + systems = [m["content"] for m in messages if m.get("role") == "system" and m.get("content")] + return "\n\n".join(systems) if systems else None + + +def extract_reasoning(msg: Dict[str, Any]) -> str: + """Extract reasoning text from a chat message or delta. + + Handles three shapes seen in the wild: + - ``reasoning_content`` (str) - deepseek-style + - ``reasoning`` (str) - openrouter + - ``reasoning_details`` (list of {"type": "reasoning.text", "text": ...}) + """ + parts: List[str] = [] + + for key in ("reasoning_content", "reasoning"): + val = msg.get(key) + + if isinstance(val, str) and val.strip(): + parts.append(val) + + details = msg.get("reasoning_details") or msg.get("reasoning_content_details") + + if isinstance(details, list): + for item in details: + if isinstance(item, dict): + text = item.get("text") + + if isinstance(text, str) and text.strip(): + parts.append(text) + + return "\n".join(parts) + + +__all__ = ["sse_json_lines", "system_prompt", "extract_reasoning"] diff --git a/cecli/helpers/model_providers.py b/cecli/helpers/model_providers.py index 50fc45e9957..46ae7243ba3 100644 --- a/cecli/helpers/model_providers.py +++ b/cecli/helpers/model_providers.py @@ -3,7 +3,7 @@ Historically cecli kept separate modules per provider (OpenRouter vs OpenAI-like). Those grew unwieldy and duplicated caching, request, and normalization logic. This helper centralizes that behavior so every OpenAI-compatible endpoint defines -a small config blob and inherits the same cache + LiteLLM registration plumbing. +a small config blob and inherits the same cache + routing plumbing. Provider configs remain curated via ``scripts/generate_providers.py`` and the static per-model fallback metadata is still cleaned up with ``clean_metadata.py``. """ @@ -17,24 +17,13 @@ import time from copy import deepcopy from pathlib import Path -from typing import Any, Dict, Optional +from typing import Dict, Optional import requests from cecli.helpers.file_searcher import handle_core_files RESOURCE_FILE = "providers.json" -_PROVIDERS_REGISTERED = False -_CUSTOM_HANDLERS: Dict[str, "Any"] = {} - - -def _coerce_str(value): - """Return the first string representation that litellm expects.""" - if isinstance(value, str): - return value - if isinstance(value, list) and value: - return value[0] - return None def _first_env_value(names): @@ -52,198 +41,6 @@ def _first_env_value(names): return None -def _get_json_openai_handler(slug: str, config: Dict) -> Any: - """Create a custom handler for OpenAI-compatible providers, lazily importing litellm.""" - try: - from litellm.llms.openai_like.chat.handler import OpenAILikeChatHandler - except Exception: - return None - - class _JSONOpenAIProvider(OpenAILikeChatHandler): - """CustomLLM wrapper that routes OpenAI-compatible providers through LiteLLM.""" - - def __init__(self, slug: str, config: Dict): - try: - from litellm.llms.custom_llm import CustomLLM - except Exception: - CustomLLM = None - - if CustomLLM is None: - raise RuntimeError("litellm custom handler support unavailable") - - super().__init__() - self.slug = slug - self.config = config - - def _resolve_api_base(self, api_base: Optional[str]) -> str: - base = ( - api_base - or _first_env_value(self.config.get("base_url_env")) - or self.config.get("api_base") - ) - if not base: - try: - from litellm.llms.custom_llm import CustomLLMError - except Exception: - CustomLLMError = Exception - - raise CustomLLMError(500, f"{self.slug} missing base URL") - return base.rstrip("/") - - def _resolve_api_key(self, api_key: Optional[str]) -> Optional[str]: - if api_key: - return api_key - env_val = _first_env_value(self.config.get("api_key_env")) - return env_val - - def _apply_special_handling(self, messages): - special = self.config.get("special_handling") or {} - if special.get("convert_content_list_to_string"): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - handle_messages_with_content_list_to_str_conversion, - ) - - return handle_messages_with_content_list_to_str_conversion(messages) - return messages - - def _inject_headers(self, headers): - defaults = self.config.get("default_headers") or {} - combined = dict(defaults) - combined.update(headers or {}) - return combined - - def _normalize_model_name(self, model: str) -> str: - if not isinstance(model, str): - return model - trimmed = model - if trimmed.startswith(f"{self.slug}/"): - trimmed = trimmed.split("/", 1)[1] - hf_namespace = self.config.get("hf_namespace") - if hf_namespace and not trimmed.startswith("hf:"): - trimmed = f"hf:{trimmed}" - return trimmed - - def _build_request_params(self, optional_params, stream: bool): - params = dict(optional_params or {}) - default_headers = dict(self.config.get("default_headers") or {}) - headers = params.setdefault("extra_headers", default_headers) - if headers is default_headers and default_headers: - params["extra_headers"] = dict(default_headers) - if stream: - params["stream"] = True - return params - - def completion(self, *args, **kwargs): - kwargs["api_base"] = self._resolve_api_base(kwargs.get("api_base", None)) - kwargs["api_key"] = self._resolve_api_key(kwargs.get("api_key", None)) - kwargs["headers"] = self._inject_headers(kwargs.get("headers", None)) - kwargs["optional_params"] = self._build_request_params( - kwargs.get("optional_params", None), False - ) - kwargs["messages"] = self._apply_special_handling(kwargs.get("messages", [])) - kwargs["model"] = self._normalize_model_name(kwargs.get("model", None)) - kwargs["custom_llm_provider"] = "openai" - return super().completion(*args, **kwargs) - - async def acompletion(self, *args, **kwargs): - kwargs["api_base"] = self._resolve_api_base(kwargs.get("api_base", None)) - kwargs["api_key"] = self._resolve_api_key(kwargs.get("api_key", None)) - kwargs["headers"] = self._inject_headers(kwargs.get("headers", None)) - kwargs["optional_params"] = self._build_request_params( - kwargs.get("optional_params", None), False - ) - kwargs["messages"] = self._apply_special_handling(kwargs.get("messages", [])) - kwargs["model"] = self._normalize_model_name(kwargs.get("model", None)) - kwargs["custom_llm_provider"] = "openai" - kwargs["acompletion"] = True - return await super().completion(*args, **kwargs) - - def streaming(self, *args, **kwargs): - kwargs["api_base"] = self._resolve_api_base(kwargs.get("api_base", None)) - kwargs["api_key"] = self._resolve_api_key(kwargs.get("api_key", None)) - kwargs["headers"] = self._inject_headers(kwargs.get("headers", None)) - kwargs["optional_params"] = self._build_request_params( - kwargs.get("optional_params", None), True - ) - kwargs["messages"] = self._apply_special_handling(kwargs.get("messages", [])) - kwargs["model"] = self._normalize_model_name(kwargs.get("model", None)) - kwargs["custom_llm_provider"] = "openai" - response = super().completion(*args, **kwargs) - for chunk in response: - yield self.get_generic_chunk(chunk) - - async def astreaming(self, *args, **kwargs): - kwargs["api_base"] = self._resolve_api_base(kwargs.get("api_base", None)) - kwargs["api_key"] = self._resolve_api_key(kwargs.get("api_key", None)) - kwargs["headers"] = self._inject_headers(kwargs.get("headers", None)) - kwargs["optional_params"] = self._build_request_params( - kwargs.get("optional_params", None), True - ) - kwargs["messages"] = self._apply_special_handling(kwargs.get("messages", [])) - kwargs["model"] = self._normalize_model_name(kwargs.get("model", None)) - kwargs["custom_llm_provider"] = "openai" - kwargs["acompletion"] = True - response = await super().completion(*args, **kwargs) - async for chunk in response: - yield self.get_generic_chunk(chunk) - - def get_generic_chunk(self, chunk): - choice = chunk.choices[0] if chunk.choices else None - delta = choice.delta if choice else None - text_content = delta.content if delta and delta.content else "" - tool_calls = delta.tool_calls if delta and delta.tool_calls else None - if tool_calls and len(tool_calls): - tool_calls = tool_calls[0] - usage_data = getattr(chunk, "usage", None) - if hasattr(usage_data, "model_dump"): - usage_dict = usage_data.model_dump() - elif isinstance(usage_data, dict): - usage_dict = usage_data - else: - usage_dict = {"completion_tokens": 0, "prompt_tokens": 0, "total_tokens": 0} - generic_chunk = { - "finish_reason": choice.finish_reason if choice else None, - "index": choice.index if choice else 0, - "is_finished": bool(choice.finish_reason) if choice else False, - "text": text_content, - "tool_use": tool_calls, - "usage": usage_dict, - } - return generic_chunk - - return _JSONOpenAIProvider(slug, config) - - -def _register_provider_with_litellm(slug: str, config: Dict) -> None: - """Register provider metadata and custom handlers with LiteLLM.""" - try: - from litellm.llms.openai_like.json_loader import JSONProviderRegistry - except Exception: - return - JSONProviderRegistry.load() - base_url = config.get("api_base") - api_key_env = _coerce_str(config.get("api_key_env")) - if not base_url or not api_key_env: - return - try: - import litellm - except Exception: - return - handler = _CUSTOM_HANDLERS.get(slug) - if handler is None: - handler = _get_json_openai_handler(slug, config) - _CUSTOM_HANDLERS[slug] = handler - if handler is None: - return - already_present = any(item.get("provider") == slug for item in litellm.custom_provider_map) - if not already_present: - litellm.custom_provider_map.append({"provider": slug, "custom_handler": handler}) - try: - litellm.custom_llm_setup() - except Exception: - pass - - def _deep_merge(base: Dict, override: Dict) -> Dict: """Recursively merge override dict into base without mutating inputs.""" result = deepcopy(base) @@ -290,7 +87,12 @@ def set_verify_ssl(self, verify_ssl: bool) -> None: self.verify_ssl = verify_ssl def merge_provider_configs(self, user_configs: Dict[str, Dict]) -> None: - """Merge user-defined provider configs into the existing provider configs.""" + """Merge user-defined provider configs into the existing provider configs. + + Merges into both this instance and the module-level ``PROVIDER_CONFIGS`` + so freshly-constructed managers (e.g. the llms dispatcher's config + resolver) see the same user providers without sharing instance state. + """ for slug, cfg in user_configs.items(): if slug in self.provider_configs: self.provider_configs[slug] = _deep_merge(self.provider_configs[slug], cfg) @@ -299,6 +101,11 @@ def merge_provider_configs(self, user_configs: Dict[str, Dict]) -> None: self._provider_cache[slug] = None self._cache_loaded[slug] = False + if slug in PROVIDER_CONFIGS: + PROVIDER_CONFIGS[slug] = _deep_merge(PROVIDER_CONFIGS[slug], cfg) + else: + PROVIDER_CONFIGS[slug] = deepcopy(cfg) + def supports_provider(self, provider: Optional[str]) -> bool: return bool(provider and provider in self.provider_configs) @@ -581,23 +388,7 @@ def _get_account_id(self, provider: str) -> Optional[str]: return None -def register_user_providers_with_litellm(user_configs: Dict[str, Dict]) -> None: - """Register user-defined providers with LiteLLM for custom handler support.""" - for slug, cfg in user_configs.items(): - _register_provider_with_litellm(slug, cfg) - - -def ensure_litellm_providers_registered() -> None: - """One-time registration guard for LiteLLM provider metadata.""" - global _PROVIDERS_REGISTERED - if _PROVIDERS_REGISTERED: - return - for slug, cfg in PROVIDER_CONFIGS.items(): - _register_provider_with_litellm(slug, cfg) - _PROVIDERS_REGISTERED = True - - -_NUMBER_RE = re.compile("-?(?:\\d+(?:\\.\\d*)?|\\.\\d+)(?:[eE][+-]?\\d+)?") +_NUMBER_RE = re.compile(r"-?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?") def _cost_per_token(val: Optional[str | float | int]) -> Optional[float]: diff --git a/cecli/helpers/responses.py b/cecli/helpers/responses.py index 68d69330718..4eefc575393 100644 --- a/cecli/helpers/responses.py +++ b/cecli/helpers/responses.py @@ -9,9 +9,11 @@ from cecli import utils from cecli.helpers import nested +from cecli.llm import litellm if TYPE_CHECKING: - from litellm.types.utils import ChatCompletionMessageToolCall, Function # noqa + ChatCompletionMessageToolCall = litellm.types.utils.ChatCompletionMessageToolCall + Function = litellm.types.utils.Function def preprocess_json(response: str) -> str: @@ -39,7 +41,8 @@ def extract_tools_from_content_json(content: str) -> Optional[List[ChatCompletio Simple extraction of JSON-like structures that look like tool calls. This handles models that write JSON in text instead of using native calling. """ - from litellm.types.utils import ChatCompletionMessageToolCall, Function # noqa + ChatCompletionMessageToolCall = litellm.types.utils.ChatCompletionMessageToolCall + Function = litellm.types.utils.Function if not content or ("{" not in content and "[" not in content): return None @@ -117,7 +120,8 @@ def extract_tools_from_content_xml(content: str) -> Optional[List[ChatCompletion """ - from litellm.types.utils import ChatCompletionMessageToolCall, Function # noqa + ChatCompletionMessageToolCall = litellm.types.utils.ChatCompletionMessageToolCall + Function = litellm.types.utils.Function if not content or ("= 1.62.0 removed the _logging module - pass - - # Make sure JSON-based OpenAI-compatible providers are registered - ensure_litellm_providers_registered() - - # Patch GLOBAL_LOGGING_WORKER to avoid event loop binding issues - # See: https://github.com/BerriAI/litellm/issues/16518 - # See: https://github.com/BerriAI/litellm/issues/14521 - try: - # Use importlib for lazy loading - logging_worker = importlib.import_module("litellm.litellm_core_utils.logging_worker") - except ImportError: - # Module didn't exist before litellm 1.76.0 - # https://github.com/BerriAI/litellm/pull/13905 - pass - else: - - class NoOpLoggingWorker: - """No-op worker that executes callbacks immediately without queuing.""" - - def start(self) -> None: - pass - - def enqueue(self, coroutine: Coroutine) -> None: - # Execute immediately in current loop instead of queueing, - # and do nothing if there's no current loop - with contextlib.suppress(RuntimeError): - # This logging task is fire-and-forget - asyncio.create_task(coroutine) - - def ensure_initialized_and_enqueue(self, async_coroutine: Coroutine) -> None: - self.enqueue(async_coroutine) - - async def stop(self) -> None: - pass - - async def flush(self) -> None: - pass - - async def clear_queue(self) -> None: - pass - - logging_worker.GLOBAL_LOGGING_WORKER = NoOpLoggingWorker() - - # Patch Delta.__init__ to support 'reasoning_text' -> 'reasoning_content' mapping - _original_delta_init = self._lazy_module.types.utils.Delta.__init__ - - def _patched_delta_init(self_delta, *args, **kwargs): - # Intercept and map 'reasoning_text' to 'reasoning_content' - if kwargs.get("reasoning_content") is None and "reasoning_text" in kwargs: - kwargs["reasoning_content"] = kwargs.pop("reasoning_text", None) - - if kwargs.get("reasoning_text", None) is not None: - if kwargs.get("provider_specific_fields") is None: - kwargs["provider_specific_fields"] = {} - - kwargs["provider_specific_fields"]["reasoning_text"] = kwargs.pop( - "reasoning_text", None - ) - - if kwargs.get("reasoning_opaque", None) is not None: - if kwargs.get("provider_specific_fields") is None: - kwargs["provider_specific_fields"] = {} - - kwargs["provider_specific_fields"]["reasoning_opaque"] = kwargs.pop( - "reasoning_opaque", None - ) - - if kwargs.get("reasoning_items", None) is not None: - if kwargs.get("provider_specific_fields") is None: - kwargs["provider_specific_fields"] = {} - - kwargs["provider_specific_fields"]["reasoning_items"] = kwargs.pop( - "reasoning_items", None - ) - - # Pass the modified kwargs to the original __init__ - _original_delta_init(self_delta, *args, **kwargs) - - self._lazy_module.types.utils.Delta.__init__ = _patched_delta_init - - -litellm = LazyLiteLLM() - -__all__ = [litellm] +__all__ = ["litellm"] diff --git a/cecli/main.py b/cecli/main.py index 8e76c2ea411..fe5e887310b 100644 --- a/cecli/main.py +++ b/cecli/main.py @@ -577,7 +577,6 @@ async def main_async( from cecli.history import ChatSummary from cecli.hooks import HookService from cecli.io import InputOutput - from cecli.llm import litellm from cecli.mcp import McpServerManager, load_mcp_servers from cecli.models import ModelSettings from cecli.onboarding import offer_openrouter_oauth, select_default_model @@ -771,12 +770,9 @@ async def main_async( if git is None: args.git = False if not args.verify_ssl: - import httpx + from cecli.helpers.llms import set_verify_ssl - os.environ["LITELLM_LOCAL_MODEL_COST"] = "true" - litellm._load_litellm() - litellm._lazy_module.client_session = httpx.Client(verify=False) - litellm._lazy_module.aclient_session = httpx.AsyncClient(verify=False) + set_verify_ssl(False) models.model_info_manager.set_verify_ssl(False) if args.timeout: models.request_timeout = args.timeout @@ -983,16 +979,17 @@ def get_io(pretty): await check_and_load_imports(io, is_first_run, verbose=args.verbose) register_models(git_root, args.model_settings_file, io, verbose=args.verbose) register_litellm_models(git_root, args.model_metadata_file, io, verbose=args.verbose) + + # Release transient garbage from the (heavy) model/litellm imports now that + # registration is done, so import-time bloat doesn't carry into the session. + from cecli.helpers.memory_control import trim_memory + + trim_memory() if args.model_providers: try: user_providers = json.loads(args.model_providers) if isinstance(user_providers, dict): models.model_info_manager.provider_manager.merge_provider_configs(user_providers) - from cecli.helpers.model_providers import ( - register_user_providers_with_litellm, - ) - - register_user_providers_with_litellm(user_providers) if args.verbose: io.tool_output(f"Loaded {len(user_providers)} custom model provider(s):") for slug in user_providers: diff --git a/cecli/models.py b/cecli/models.py index 6e51c7adec6..510c6d8fe1f 100644 --- a/cecli/models.py +++ b/cecli/models.py @@ -11,7 +11,6 @@ from uuid import uuid4 as generate_unique_id import yaml -from PIL import Image from cecli import __version__ from cecli.decoding import safe_open @@ -728,8 +727,6 @@ def _apply_structured_kwargs(self, config, model_name): litellm.model_cost[model_name] = {} litellm.model_cost[model_name].update(self.info) - litellm.utils._invalidate_model_cost_lowercase_map() - litellm.add_known_models(model_cost_map=litellm.model_cost) elif isinstance(value, dict) and isinstance(self.extra_params.get(key), dict): self.extra_params[key] = {**self.extra_params[key], **value} @@ -1051,6 +1048,8 @@ def get_image_size(self, fname): :param fname: The filename of the image. :return: A tuple (width, height) representing the image size in pixels. """ + from PIL import Image + with Image.open(fname) as img: return img.size diff --git a/requirements.txt b/requirements.txt index 58e6b26c294..d1073e4095a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,5 @@ # This file was autogenerated by uv via the following command: # uv pip compile --no-strip-extras --constraint=requirements/common-constraints.txt --output-file=tmp.requirements.txt requirements/requirements.in -aiohappyeyeballs==2.6.1 - # via - # -c requirements/common-constraints.txt - # aiohttp -aiohttp==3.13.2 - # via - # -c requirements/common-constraints.txt - # litellm -aiosignal==1.4.0 - # via - # -c requirements/common-constraints.txt - # aiohttp annotated-types==0.7.0 # via # -c requirements/common-constraints.txt @@ -28,7 +16,6 @@ anyio==4.11.0 attrs==25.4.0 # via # -c requirements/common-constraints.txt - # aiohttp # jsonschema # referencing beautifulsoup4==4.14.2 @@ -59,8 +46,16 @@ charset-normalizer==3.4.9 click==8.3.1 # via # -c requirements/common-constraints.txt - # litellm + # click-default-group + # llm + # sqlite-utils # uvicorn +click-default-group==1.2.4 + # via + # llm + # sqlite-utils +condense-json==1.1 + # via llm configargparse==1.7.1 # via # -c requirements/common-constraints.txt @@ -82,23 +77,6 @@ distro==1.9.0 # via # -c requirements/common-constraints.txt # openai -fastuuid==0.14.0 - # via - # -c requirements/common-constraints.txt - # litellm -filelock==3.20.0 - # via - # -c requirements/common-constraints.txt - # huggingface-hub -frozenlist==1.8.0 - # via - # -c requirements/common-constraints.txt - # aiohttp - # aiosignal -fsspec==2025.10.0 - # via - # -c requirements/common-constraints.txt - # huggingface-hub gitdb==4.0.12 # via # -c requirements/common-constraints.txt @@ -112,10 +90,6 @@ h11==0.16.0 # -c requirements/common-constraints.txt # httpcore # uvicorn -hf-xet==1.2.0 - # via - # -c requirements/common-constraints.txt - # huggingface-hub httpcore==1.0.9 # via # -c requirements/common-constraints.txt @@ -123,37 +97,26 @@ httpcore==1.0.9 httpx==0.28.1 # via # -c requirements/common-constraints.txt - # litellm # mcp # openai httpx-sse==0.4.3 # via # -c requirements/common-constraints.txt # mcp -huggingface-hub==0.36.0 - # via - # -c requirements/common-constraints.txt - # tokenizers idna==3.11 # via # -c requirements/common-constraints.txt # anyio # httpx # requests - # yarl importlib-metadata==8.7.0 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in - # litellm importlib-resources==6.5.2 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in -jinja2==3.1.6 - # via - # -c requirements/common-constraints.txt - # litellm jiter==0.12.0 # via # -c requirements/common-constraints.txt @@ -166,7 +129,6 @@ jsonschema==4.25.1 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in - # litellm # mcp jsonschema-specifications==2025.9.1 # via @@ -176,10 +138,8 @@ linkify-it-py==2.0.3 # via # -c requirements/common-constraints.txt # markdown-it-py -litellm==1.81.11 - # via - # -c requirements/common-constraints.txt - # -r requirements/requirements.in +llm==0.32 + # via -r requirements/requirements.in marisa-trie==1.4.1 # via # -c requirements/common-constraints.txt @@ -190,10 +150,6 @@ markdown-it-py[linkify]==4.0.0 # mdit-py-plugins # rich # textual -markupsafe==3.0.3 - # via - # -c requirements/common-constraints.txt - # jinja2 mcp==1.25.0 # via # -c requirements/common-constraints.txt @@ -210,11 +166,6 @@ mslex==1.3.0 # via # -c requirements/common-constraints.txt # oslex -multidict==6.7.0 - # via - # -c requirements/common-constraints.txt - # aiohttp - # yarl ngram==4.0.3 # via # -c requirements/common-constraints.txt @@ -223,12 +174,11 @@ numpy==2.3.5 # via # -c requirements/common-constraints.txt # rustworkx - # scipy # soundfile -openai==2.8.1 +openai==2.53.0 # via # -c requirements/common-constraints.txt - # litellm + # llm orjson==3.11.9 # via # -c requirements/common-constraints.txt @@ -241,7 +191,6 @@ packaging==25.0 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in - # huggingface-hub pathspec==0.12.1 # via # -c requirements/common-constraints.txt @@ -254,19 +203,24 @@ pillow==12.0.0 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in +pip==25.3 + # via + # -c requirements/common-constraints.txt + # llm + # sqlite-utils platformdirs==4.5.0 # via # -c requirements/common-constraints.txt # textual -prompt-toolkit==3.0.52 +pluggy==1.6.0 # via # -c requirements/common-constraints.txt - # -r requirements/requirements.in -propcache==0.4.1 + # llm + # sqlite-utils +prompt-toolkit==3.0.52 # via # -c requirements/common-constraints.txt - # aiohttp - # yarl + # -r requirements/requirements.in psutil==7.1.3 # via # -c requirements/common-constraints.txt @@ -275,6 +229,8 @@ ptyprocess==0.7.0 # via # -c requirements/common-constraints.txt # pexpect +puremagic==2.2.0 + # via llm py-cymbal==0.2.1 # via # -c requirements/common-constraints.txt @@ -286,7 +242,7 @@ pycparser==2.23 pydantic==2.12.4 # via # -c requirements/common-constraints.txt - # litellm + # llm # mcp # openai # pydantic-settings @@ -319,20 +275,25 @@ pyperclip==1.11.0 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in +python-dateutil==2.9.0.post0 + # via + # -c requirements/common-constraints.txt + # sqlite-utils python-dotenv==1.2.2 # via # -c requirements/common-constraints.txt - # litellm # pydantic-settings python-multipart==0.0.20 # via # -c requirements/common-constraints.txt # mcp +python-ulid==4.0.1 + # via llm pyyaml==6.0.3 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in - # huggingface-hub + # llm rapidfuzz==3.14.5 # via # -c requirements/common-constraints.txt @@ -349,7 +310,6 @@ regex==2025.11.3 requests==2.32.5 # via # -c requirements/common-constraints.txt - # huggingface-hub # tiktoken rich==14.2.0 # via @@ -365,14 +325,18 @@ rustworkx==0.17.1 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in -scipy==1.16.3 +setuptools==80.9.0 # via # -c requirements/common-constraints.txt - # -r requirements/requirements.in + # llm shtab==1.8.0 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in +six==1.17.0 + # via + # -c requirements/common-constraints.txt + # python-dateutil smmap==5.0.2 # via # -c requirements/common-constraints.txt @@ -398,6 +362,10 @@ soupsieve==2.8 # via # -c requirements/common-constraints.txt # beautifulsoup4 +sqlite-fts4==1.0.3 + # via sqlite-utils +sqlite-utils==4.1.1 + # via llm sse-starlette==3.0.3 # via # -c requirements/common-constraints.txt @@ -406,18 +374,16 @@ starlette==0.50.0 # via # -c requirements/common-constraints.txt # mcp +tabulate==0.10.0 + # via sqlite-utils textual==8.2.8 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in -tiktoken==0.12.0 - # via - # -c requirements/common-constraints.txt - # litellm -tokenizers==0.22.1 +tiktoken==0.13.0 # via # -c requirements/common-constraints.txt - # litellm + # -r requirements/requirements.in tomlkit==0.14.0 # via # -c requirements/common-constraints.txt @@ -425,13 +391,12 @@ tomlkit==0.14.0 tqdm==4.67.1 # via # -c requirements/common-constraints.txt - # huggingface-hub # openai +tree-sitter==0.25.2 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in # tree-sitter-language-pack - # tree-sitter-languages tree-sitter-c-sharp==0.23.5 # via # -c requirements/common-constraints.txt @@ -444,10 +409,6 @@ tree-sitter-language-pack==0.13.0 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in -tree-sitter-languages==1.10.2 - # via - # -c requirements/common-constraints.txt - # -r requirements/requirements.in tree-sitter-yaml==0.7.2 # via # -c requirements/common-constraints.txt @@ -459,16 +420,11 @@ truststore==0.10.4 typing-extensions==4.15.0 # via # -c requirements/common-constraints.txt - # aiosignal - # anyio # beautifulsoup4 - # huggingface-hub # mcp # openai # pydantic # pydantic-core - # referencing - # starlette # textual # typing-inspection typing-inspection==0.4.2 @@ -505,14 +461,7 @@ xxhash==3.6.0 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in -yarl==1.22.0 - # via - # -c requirements/common-constraints.txt - # aiohttp zipp==3.23.0 # via # -c requirements/common-constraints.txt # importlib-metadata - -tree-sitter==0.23.2; python_version < "3.10" -tree-sitter>=0.25.1; python_version >= "3.10" diff --git a/requirements/common-constraints.txt b/requirements/common-constraints.txt index 306a605a58a..bad8d696db4 100644 --- a/requirements/common-constraints.txt +++ b/requirements/common-constraints.txt @@ -300,7 +300,7 @@ nvidia-nvtx-cu12==12.8.90 # via torch objgraph==3.6.2 # via -r requirements/requirements-dev.in -openai==2.8.1 +openai>=2.32.0 # via litellm orjson==3.11.9 # via -r requirements/requirements.in @@ -467,7 +467,6 @@ scikit-learn==1.7.2 # via sentence-transformers scipy==1.16.3 # via - # -r requirements/requirements.in # scikit-learn # sentence-transformers semver==3.0.4 @@ -515,7 +514,7 @@ textual==8.2.8 # memray threadpoolctl==3.6.0 # via scikit-learn -tiktoken==0.12.0 +tiktoken>=0.13.0 # via # litellm # llama-index-core diff --git a/requirements/requirements.in b/requirements/requirements.in index c69f47393a0..ac3f3a49dfa 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -13,7 +13,8 @@ GitPython>=3.1.45 pathspec>=0.12.1 # communication -litellm>=1.80.11,!=1.82.7,!=1.82.8 +llm>=0.32 +tiktoken>=0.13.0 mcp>=1.24.0 socksio>=1.0.0 truststore @@ -22,7 +23,8 @@ truststore cryptography>=42.0.0 xxhash>=3.6.0 -# scraping +# scraping (OPTIONAL: only used by the scrape command; bs4 already imported lazily in cecli/scrape.py) +# TODO: split into a `[scrape]` extra (requirements-scrape.in) after making pypandoc lazy beautifulsoup4>=4.13.4 pypandoc>=1.15 @@ -42,7 +44,8 @@ diff-match-patch>=20241021 # copying and pasting pyperclip>=1.9.0 -# audio +# audio (OPTIONAL: only used by the voice command; imported lazily in cecli/voice.py) +# TODO: split into a `[voice]` extra (requirements-voice.in) - safe because imports are lazy pydub>=0.25.1 sounddevice>=0.5.2 soundfile>=0.13.1 @@ -59,7 +62,8 @@ textual>=8.2.8 blinker>=1.9.0 websockets>=16.1.1 -# file system lookup aids +# file system lookup aids (OPTIONAL: only used for fuzzy model-name search; imported lazily in cecli/models.py) +# TODO: split into a `[search]` extra (requirements-search.in) - safe because imports are lazy marisa-trie>=1.0 ngram>=4.0.3 rapidfuzz>=3.0 @@ -67,10 +71,10 @@ rapidfuzz>=3.0 # replaced networkx with rustworkx for better performance in repomap rustworkx>=0.15.0 -# scipy is still needed for other parts of the codebase -scipy>=1.15.3 +# NOTE: scipy was removed - unused across the whole repo (grep for `scipy` finds nothing). +# It previously pulled in the numpy stack (~100 modules) for zero usage. -# dependencies added because litellm is sometimes really mid and doesn't declare all of its deps properly +# fast JSON serialization (kept pinned for tooling/performance) orjson>=3.11.6 # tool helpers diff --git a/tests/basic/test_exceptions.py b/tests/basic/test_exceptions.py index 004f58f2b13..d85a6c52cd6 100644 --- a/tests/basic/test_exceptions.py +++ b/tests/basic/test_exceptions.py @@ -1,4 +1,5 @@ from cecli.exceptions import ExInfo, LiteLLMExceptions +from cecli.llm import litellm def test_litellm_exceptions_load(): @@ -19,7 +20,7 @@ def test_get_ex_info(): ex = LiteLLMExceptions() # Test with a known exception type - from litellm import AuthenticationError + AuthenticationError = litellm.AuthenticationError auth_error = AuthenticationError( message="Invalid API key", llm_provider="openai", model="gpt-4" @@ -45,7 +46,7 @@ class UnknownError(Exception): def test_rate_limit_error(): """Test specific handling of RateLimitError""" ex = LiteLLMExceptions() - from litellm import RateLimitError + RateLimitError = litellm.RateLimitError rate_error = RateLimitError(message="Rate limit exceeded", llm_provider="openai", model="gpt-4") ex_info = ex.get_ex_info(rate_error) @@ -56,7 +57,7 @@ def test_rate_limit_error(): def test_bad_gateway_error(): """Test specific handling of BadGatewayError""" ex = LiteLLMExceptions() - from litellm import BadGatewayError + BadGatewayError = litellm.BadGatewayError bad_gateway_error = BadGatewayError(message="Bad Gateway", llm_provider="openai", model="gpt-4") ex_info = ex.get_ex_info(bad_gateway_error) @@ -67,7 +68,7 @@ def test_bad_gateway_error(): def test_context_window_error(): """Test specific handling of ContextWindowExceededError""" ex = LiteLLMExceptions() - from litellm import ContextWindowExceededError + ContextWindowExceededError = litellm.ContextWindowExceededError ctx_error = ContextWindowExceededError( message="Context length exceeded", model="gpt-4", llm_provider="openai" @@ -79,7 +80,7 @@ def test_context_window_error(): def test_openrouter_error(): """Test specific handling of OpenRouter API errors""" ex = LiteLLMExceptions() - from litellm import APIConnectionError + APIConnectionError = litellm.APIConnectionError # Create an APIConnectionError with OpenrouterException message openrouter_error = APIConnectionError( diff --git a/tests/basic/test_main.py b/tests/basic/test_main.py index 13b92cb74d2..87b47b9a4f9 100644 --- a/tests/basic/test_main.py +++ b/tests/basic/test_main.py @@ -1083,7 +1083,11 @@ def test_reasoning_effort_option(dummy_io, git_temp_dir): **dummy_io, return_coder=True, ) - assert coder.main_model.extra_params.get("extra_body", {}).get("reasoning_effort") == "3" + # OpenRouter / responses-mode models store the effort as the nested + # ``reasoning.effort``; everything else uses the flat ``reasoning_effort``. + extra_body = coder.main_model.extra_params.get("extra_body", {}) + effort = extra_body.get("reasoning_effort") or extra_body.get("reasoning", {}).get("effort") + assert effort == "3" def test_thinking_tokens_option(dummy_io, git_temp_dir): diff --git a/tests/basic/test_reasoning.py b/tests/basic/test_reasoning.py index c08279ac2b9..86186b2ce71 100644 --- a/tests/basic/test_reasoning.py +++ b/tests/basic/test_reasoning.py @@ -228,7 +228,7 @@ async def async_chunks(): with ( patch.object(model, "send_completion", return_value=(mock_hash, async_chunks())), patch.object(model, "token_count", return_value=10), - patch("litellm.stream_chunk_builder", return_value=None), + patch("cecli.llm.litellm.stream_chunk_builder", return_value=None), ): # Mock token count and stream_chunk_builder to avoid serialization issues # Set mdstream directly on the coder object coder.mdstream = mock_mdstream @@ -384,7 +384,7 @@ async def async_chunks(): # Mock the model's send_completion to return the hash and completion with ( patch.object(model, "send_completion", return_value=(mock_hash, async_chunks())), - patch("litellm.stream_chunk_builder", return_value=None), + patch("cecli.llm.litellm.stream_chunk_builder", return_value=None), ): # Set mdstream directly on the coder object coder.mdstream = mock_mdstream @@ -570,7 +570,7 @@ async def async_chunks(): with ( patch.object(model, "send_completion", return_value=(mock_hash, async_chunks())), patch.object(model, "token_count", return_value=10), - patch("litellm.stream_chunk_builder", return_value=None), + patch("cecli.llm.litellm.stream_chunk_builder", return_value=None), ): # Mock token count and stream_chunk_builder to avoid serialization issues # Set mdstream directly on the coder object coder.mdstream = mock_mdstream diff --git a/tests/basic/test_sendchat.py b/tests/basic/test_sendchat.py index 46394e653f4..9b544597260 100644 --- a/tests/basic/test_sendchat.py +++ b/tests/basic/test_sendchat.py @@ -21,7 +21,7 @@ def test_litellm_exceptions(self): litellm_ex = LiteLLMExceptions() litellm_ex._load(strict=True) - @patch("litellm.acompletion") + @patch("cecli.llm.litellm.acompletion") @patch("builtins.print") async def test_simple_send_with_retries_rate_limit_error(self, mock_print, mock_completion): mock = MagicMock() @@ -45,7 +45,7 @@ async def test_simple_send_with_retries_rate_limit_error(self, mock_print, mock_ await model.simple_send_with_retries(self.mock_messages) assert mock_print.call_count > 0 - @patch("litellm.acompletion") + @patch("cecli.llm.litellm.acompletion") async def test_send_completion_basic(self, mock_completion): # Setup mock response mock_response = MagicMock() @@ -59,7 +59,7 @@ async def test_send_completion_basic(self, mock_completion): assert response == mock_response mock_completion.assert_called_once() - @patch("litellm.acompletion") + @patch("cecli.llm.litellm.acompletion") async def test_send_completion_with_functions(self, mock_completion): mock_function = {"name": "test_function", "parameters": {"type": "object"}} @@ -72,7 +72,7 @@ async def test_send_completion_with_functions(self, mock_completion): assert "tools" in called_kwargs assert called_kwargs["tools"][0]["function"] == mock_function - @patch("litellm.acompletion") + @patch("cecli.llm.litellm.acompletion") async def test_simple_send_with_retries_passes_tools_from_coder(self, mock_completion): # Setup mock response mock_response = MagicMock() @@ -102,7 +102,7 @@ async def test_simple_send_with_retries_passes_tools_from_coder(self, mock_compl # send_completion sorts tools deterministically by function name assert called_kwargs["tools"] == sorted(tools, key=lambda x: x["function"]["name"]) - @patch("litellm.acompletion") + @patch("cecli.llm.litellm.acompletion") async def test_simple_send_attribute_error(self, mock_completion): # Setup mock to raise AttributeError mock_completion.return_value = MagicMock() @@ -112,7 +112,7 @@ async def test_simple_send_attribute_error(self, mock_completion): result = await Model(self.mock_model).simple_send_with_retries(self.mock_messages) assert result is None - @patch("litellm.acompletion") + @patch("cecli.llm.litellm.acompletion") @patch("builtins.print") async def test_simple_send_non_retryable_error(self, mock_print, mock_completion): # Test with an error that shouldn't trigger retries diff --git a/tests/coders/test_copypaste_coder.py b/tests/coders/test_copypaste_coder.py index acb9d3414dc..16d8759aa68 100644 --- a/tests/coders/test_copypaste_coder.py +++ b/tests/coders/test_copypaste_coder.py @@ -110,9 +110,14 @@ class DummyModelResponse: def __init__(self, **kwargs): self.kwargs = kwargs + class DummyUsage(dict): + def __init__(self, **kwargs): + super().__init__(**kwargs) + monkeypatch.setattr("cecli.coders.copypaste_coder.litellm.Message", DummyMessage) monkeypatch.setattr("cecli.coders.copypaste_coder.litellm.Choices", DummyChoices) monkeypatch.setattr("cecli.coders.copypaste_coder.litellm.ModelResponse", DummyModelResponse) + monkeypatch.setattr("cecli.coders.copypaste_coder.litellm.Usage", DummyUsage) class ModelStub: name = "cp:gpt-4o" diff --git a/tests/coders/test_tool_call_consolidation.py b/tests/coders/test_tool_call_consolidation.py index 5fedb4feac1..ab5fd8f19b4 100644 --- a/tests/coders/test_tool_call_consolidation.py +++ b/tests/coders/test_tool_call_consolidation.py @@ -9,35 +9,26 @@ 3. Tool-call deltas that start at a non-zero index were mishandled. """ -from litellm.types.utils import ( - ChatCompletionDeltaToolCall, - Delta, - Function, - ModelResponseStream, - StreamingChoices, -) - from cecli.coders.base_coder import Coder +from cecli.llm import litellm def mk_chunk(delta_kwargs, finish_reason=None, usage=None, cid="cmpl-1", created=1000): - delta = Delta(**delta_kwargs) - choices = [StreamingChoices(finish_reason=finish_reason, index=0, delta=delta, logprobs=None)] - return ModelResponseStream( + delta = litellm.Delta(**delta_kwargs) + choices = [litellm.StreamChoice(finish_reason=finish_reason, index=0, delta=delta)] + return litellm.StreamChunk( id=cid, created=created, model="gpt-test", - object="chat.completion.chunk", - system_fingerprint=None, choices=choices, usage=usage, ) def tc(index, id=None, name=None, arguments=None): - return ChatCompletionDeltaToolCall( + return litellm.ChatCompletionMessageToolCall( id=id, - function=Function(arguments=arguments or "", name=name), + function=litellm.Function(arguments=arguments or "", name=name), type="function", index=index, ) From 240cbec83ee53739aa9f249e5a7627c85b5032a8 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 10 Aug 2026 23:47:58 -0400 Subject: [PATCH 11/30] Fix models switching parameter dropping --- cecli/helpers/llms/domains/chat.py | 58 +++++++++++++++++- cecli/helpers/llms/domains/gemini.py | 80 ++++++++++++++++++++----- cecli/helpers/llms/domains/responses.py | 50 +++++++++++++--- 3 files changed, 165 insertions(+), 23 deletions(-) diff --git a/cecli/helpers/llms/domains/chat.py b/cecli/helpers/llms/domains/chat.py index 3622f0475b1..5bf4d658e03 100644 --- a/cecli/helpers/llms/domains/chat.py +++ b/cecli/helpers/llms/domains/chat.py @@ -38,9 +38,10 @@ def chat_payload( stream: bool, kwargs: Dict[str, Any], ) -> Dict[str, Any]: + api_block = resolved.get("api_block") or {} payload: Dict[str, Any] = { "model": resolved["route"], - "messages": messages, + "messages": _coerce_reasoning_content(messages, api_block), "stream": stream, } @@ -48,7 +49,6 @@ def chat_payload( payload["tools"] = tools payload["tool_choice"] = kwargs.get("tool_choice", "auto") - api_block = resolved.get("api_block") or {} if api_block.get("reasoning_effort"): payload["reasoning_effort"] = api_block["reasoning_effort"] @@ -284,6 +284,60 @@ def _usage_from_raw(usage_raw: Optional[Dict[str, Any]]) -> Optional[Usage]: ) +def _coerce_reasoning_content( + messages: List[Dict[str, Any]], api_block: Dict[str, Any] +) -> List[Dict[str, Any]]: + """Ensure assistant messages carry ``reasoning_content`` for thinking-mode chat. + + DeepSeek (and other OpenAI-compatible providers in thinking mode) require + the ``reasoning_content`` of a prior assistant turn to be echoed back when + that message is replayed; a missing/``None`` value is rejected with ``The + reasoning_content in the thinking mode must be passed back to the API``. + Messages produced by other providers (gemini / anthropic / copilot) do not + carry it, so we loosely map any captured reasoning (anthropic thinking + blocks, gemini thought parts) into ``reasoning_content`` and otherwise send + an empty string, which DeepSeek accepts. + """ + if not (api_block.get("reasoning_effort") or api_block.get("thinking")): + return messages + + out: List[Dict[str, Any]] = [] + changed = False + + for msg in messages: + if msg.get("role") == "assistant" and msg.get("reasoning_content") is None: + copied = dict(msg) + copied["reasoning_content"] = _map_reasoning_content(msg) or "" + out.append(copied) + changed = True + else: + out.append(msg) + + return out if changed else messages + + +def _map_reasoning_content(msg: Dict[str, Any]) -> str: + """Loosely map foreign-provider reasoning into a ``reasoning_content`` string.""" + psf = msg.get("provider_specific_fields") or {} + texts: List[str] = [] + + for block in psf.get("anthropic") or []: + if isinstance(block, dict) and block.get("type") == "thinking" and block.get("thinking"): + texts.append(block["thinking"]) + + for tp in psf.get("thought_parts") or []: + if isinstance(tp, dict) and tp.get("text"): + texts.append(tp["text"]) + + for key in ("reasoning_text", "reasoning"): + value = psf.get(key) + + if isinstance(value, str) and value: + texts.append(value) + + return "\n".join(texts) + + __all__ = [ "chat_payload", "chat_complete", diff --git a/cecli/helpers/llms/domains/gemini.py b/cecli/helpers/llms/domains/gemini.py index d19337e51fa..3e3e778637b 100644 --- a/cecli/helpers/llms/domains/gemini.py +++ b/cecli/helpers/llms/domains/gemini.py @@ -51,9 +51,13 @@ def gemini_payload( # text part makes the model re-invoke the tool instead of consuming the # result. call_meta = _build_call_meta(messages) + # Gemini 3.x requires a native ``thoughtSignature`` on every replayed + # ``functionCall`` part; messages from other providers cannot provide one, + # so their calls are dropped to text (see :func:`_requires_thought_signatures`). + require_signatures = _requires_thought_signatures(resolved) payload: Dict[str, Any] = { - "contents": _encode_contents(messages, call_meta), + "contents": _encode_contents(messages, call_meta, require_signatures), } if system: @@ -110,7 +114,9 @@ def gemini_thinking_config(resolved: Dict[str, Any], effort: str) -> Dict[str, A def gemini_content( - msg: Dict[str, Any], name_by_call_id: Optional[Dict[str, Any]] = None + msg: Dict[str, Any], + name_by_call_id: Optional[Dict[str, Any]] = None, + require_signatures: bool = False, ) -> Dict[str, Any]: """Encode one chat message as a Gemini ``Content`` dict. @@ -119,12 +125,15 @@ def gemini_content( tool results become ``functionResponse`` parts that echo the original call id/signature when available. Assistant turns replay prior thought parts (with their ``thoughtSignature``) for multi-turn reasoning. + ``require_signatures`` is set for Gemini 3.x targets, which reject + ``functionCall`` parts without a native signature (see + :func:`_requires_thought_signatures`). """ if msg.get("role") == "tool": - return _tool_content(msg, name_by_call_id) + return _tool_content(msg, name_by_call_id, require_signatures) if msg.get("role") == "assistant": - return _model_content(msg) + return _model_content(msg, require_signatures) # user (and any non-assistant role, e.g. system, which folds into user). content = msg.get("content") @@ -416,6 +425,18 @@ def _gemini_usage(usage_raw: Dict[str, Any]) -> Optional[Usage]: ) +def _requires_thought_signatures(resolved: Dict[str, Any]) -> bool: + """Whether the target model requires a signature on replayed functionCall parts. + + Gemini 3.x demands the native ``thoughtSignature`` (an opaque encrypted + blob) on every ``functionCall`` part replayed in history; without it the + API rejects the request with a 400. Older models (e.g. gemini-2.5-pro) + accept signature-less replays. Messages from other providers never carry a + Gemini signature, so for 3.x targets the encoder drops those calls to text. + """ + return "gemini-3" in (resolved.get("route") or "").lower() + + def _build_call_meta(messages: List[Dict[str, Any]]) -> Dict[str, Any]: """Map assistant tool-call ids to ``{"name", "signature"}`` metadata. @@ -460,12 +481,22 @@ def _call_meta(name_by_call_id: Optional[Dict[str, Any]], call_id: Optional[str] return {"name": entry or ""} -def _tool_content(msg: Dict[str, Any], name_by_call_id: Optional[Dict[str, Any]]) -> Dict[str, Any]: - """Encode a tool-result message as a ``functionResponse`` user Content.""" +def _tool_content( + msg: Dict[str, Any], + name_by_call_id: Optional[Dict[str, Any]], + require_signatures: bool = False, +) -> Dict[str, Any]: + """Encode a tool-result message as a ``functionResponse`` user Content. + + On Gemini 3.x targets the matching ``functionCall`` was dropped to text + when no signature was available, so the result is downgraded to a plain + text part too (a ``functionResponse`` without its ``functionCall`` would + be rejected or ignored). + """ meta = _call_meta(name_by_call_id, msg.get("tool_call_id")) name = meta.get("name") or "" - if name: + if name and (meta.get("signature") or not require_signatures): fr: Dict[str, Any] = { "name": name, "response": {"output": msg.get("content") or ""}, @@ -477,16 +508,17 @@ def _tool_content(msg: Dict[str, Any], name_by_call_id: Optional[Dict[str, Any]] part_dict: Dict[str, Any] = {"functionResponse": fr} - if meta.get("signature"): + if require_signatures and meta.get("signature"): part_dict["thoughtSignature"] = meta["signature"] return {"role": "user", "parts": [part_dict]} - # No matching functionCall recorded -- fall back to plain text. + # No matching functionCall recorded (or it was dropped for a + # signature-requiring model) -- fall back to plain text. return {"role": "user", "parts": [{"text": msg.get("content") or ""}]} -def _model_content(msg: Dict[str, Any]) -> Dict[str, Any]: +def _model_content(msg: Dict[str, Any], require_signatures: bool = False) -> Dict[str, Any]: """Encode an assistant message, replaying prior thought parts. Gemini requires the full thinking turn (thought parts + ``thoughtSignature``) @@ -495,6 +527,11 @@ def _model_content(msg: Dict[str, Any]) -> Dict[str, Any]: live on ``provider_specific_fields``; when an upstream consumer dropped them, fall back to reconstructing a single thought part from ``reasoning_content``. + + Gemini 3.x also requires a native ``thoughtSignature`` on every replayed + ``functionCall`` part. Foreign messages never carry one, so on those models + the call is dropped and surfaced as a text part instead of failing with a + 400 (see :func:`_requires_thought_signatures`). """ psf = msg.get("provider_specific_fields") or {} thought_parts = psf.get("thought_parts") or [] @@ -540,15 +577,24 @@ def _model_content(msg: Dict[str, Any]) -> Dict[str, Any]: except json.JSONDecodeError: args = {} + sig = call_signatures.get(tc["id"]) if tc.get("id") else None + + # A signature-requiring model cannot replay a call without its native + # signature (foreign messages never have one). Drop the functionCall + # and surface the call as text so the turn stays coherent; the + # matching tool result is downgraded to text by _tool_content too. + if require_signatures and not sig: + parts.append({"text": f"[tool call: {fn.get('name', '')}({json.dumps(args)})]"}) + continue + fc: Dict[str, Any] = {"name": fn.get("name", ""), "args": args} if tc.get("id"): fc["id"] = tc["id"] part_dict: Dict[str, Any] = {"functionCall": fc} - sig = call_signatures.get(tc["id"]) if tc.get("id") else None - if sig: + if require_signatures and sig: part_dict["thoughtSignature"] = sig parts.append(part_dict) @@ -563,7 +609,9 @@ def _model_content(msg: Dict[str, Any]) -> Dict[str, Any]: def _encode_contents( - messages: List[Dict[str, Any]], name_by_call_id: Optional[Dict[str, Any]] + messages: List[Dict[str, Any]], + name_by_call_id: Optional[Dict[str, Any]], + require_signatures: bool = False, ) -> List[Dict[str, Any]]: """Encode the message list, merging consecutive same-role Contents. @@ -574,7 +622,11 @@ def _encode_contents( Merging keeps e.g. multiple tool results (plus a following user text turn) in ONE Content right after the assistant tool-call turn. """ - encoded = [gemini_content(m, name_by_call_id) for m in messages if m.get("role") != "system"] + encoded = [ + gemini_content(m, name_by_call_id, require_signatures) + for m in messages + if m.get("role") != "system" + ] merged: List[Dict[str, Any]] = [] for content in encoded: diff --git a/cecli/helpers/llms/domains/responses.py b/cecli/helpers/llms/domains/responses.py index 6830afac4a9..a3a8cf9f6ad 100644 --- a/cecli/helpers/llms/domains/responses.py +++ b/cecli/helpers/llms/domains/responses.py @@ -49,7 +49,7 @@ def responses_payload( ) -> Dict[str, Any]: payload: Dict[str, Any] = { "model": resolved["route"], - "input": to_responses_input(messages), + "input": to_responses_input(messages, resolved.get("model")), "stream": stream, "store": False, } @@ -81,8 +81,17 @@ def responses_payload( return payload -def to_responses_input(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """Convert OpenAI chat messages to responses-API input items.""" +def to_responses_input( + messages: List[Dict[str, Any]], current_model: Optional[str] = None +) -> List[Dict[str, Any]]: + """Convert OpenAI chat messages to responses-API input items. + + ``current_model`` gates the encrypted-reasoning replay: reasoning items are + only replayed when the model that produced them (recorded at stash time in + ``provider_specific_fields["reasoning_items_origin"]``) matches the current + target. Foreign encrypted reasoning (ids + ciphertext are provider/model + specific) is dropped instead of replayed, which would 400. + """ items: List[Dict[str, Any]] = [] for msg in messages: @@ -94,9 +103,9 @@ def to_responses_input(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: if role == "assistant": # Replay stashed reasoning items BEFORE the assistant message item - # so the provider can continue the encrypted reasoning state + # so the provider can continue its OWN encrypted reasoning state # (stateless round-trip: the whole conversation is re-sent). - for r_item in _stashed_reasoning_items(msg): + for r_item in _stashed_reasoning_items(msg, current_model): items.append(_reasoning_input_item(r_item)) # Assistant turns must use ``output_text`` content blocks; Copilot / @@ -198,6 +207,10 @@ async def responses_stream( hdrs = {"Authorization": f"Bearer {key}", "Content-Type": "application/json", **headers} _reset_stream_state() + # Tag reasoning items stashed during this stream with the producing model + # so the request encoder replays them only to the same model (a switch to + # another provider/model 400s on foreign reasoning ids/ciphertext). + _stream_state["model"] = resolved["model"] async with make_client(timeout=DEFAULT_TIMEOUT, verify=VERIFY_SSL) as client: async with client.stream("POST", url, json=payload, headers=hdrs) as resp: @@ -232,6 +245,10 @@ def normalize_responses_response(data: Dict[str, Any], model: str) -> Completion # redacted instead of fabricating placeholder text. if item.get("encrypted_content"): provider_fields.setdefault("reasoning_items", []).append(item) + # Record which model produced this encrypted reasoning so the + # request encoder only replays it to the SAME model (ids and + # ciphertext are model-specific; a switch 400s otherwise). + provider_fields["reasoning_items_origin"] = model parts.append(ReasoningPart(redacted=True)) elif item_type == "function_call": @@ -326,7 +343,8 @@ def parse_responses_chunk(data: Dict[str, Any]) -> Optional[CompletionChunk]: # per-event ciphertext differs, so only the final items are emitted. if _stream_state["reasoning_items"]: chunk.provider_specific_fields = { - "reasoning_items": list(_stream_state["reasoning_items"].values()) + "reasoning_items": list(_stream_state["reasoning_items"].values()), + "reasoning_items_origin": _stream_state.get("model"), } chunk.usage = _build_usage(resp.get("usage") or {}) @@ -339,6 +357,7 @@ def _reset_stream_state() -> None: """Clear per-stream correlation state before a new SSE loop.""" _stream_state["tool_items"] = {} _stream_state["reasoning_items"] = {} + _stream_state["model"] = None def _on_output_item_added(item: Dict[str, Any], output_index: Optional[int] = None) -> None: @@ -406,18 +425,35 @@ def _capture_final_reasoning(resp: Dict[str, Any]) -> None: _stream_state["reasoning_items"] = {item.get("id"): item for item in items} -def _stashed_reasoning_items(msg: Dict[str, Any]) -> List[Dict[str, Any]]: +def _stashed_reasoning_items( + msg: Dict[str, Any], current_model: Optional[str] = None +) -> List[Dict[str, Any]]: """Collect stashed reasoning items from a stored assistant message. The normalizer stores whole ``reasoning`` items under ``provider_specific_fields["reasoning_items"]``; ``helpers.requests`` may hoist them to the message top level (``msg["reasoning_items"]``) before this point, so both locations are honoured. + + Encrypted reasoning is model-specific: the ``id`` format and ciphertext are + only valid for the model that produced them. When the current target model + differs from the recorded origin (or the origin is unknown/absent -- a + legacy or foreign stash), the items are dropped instead of being replayed + verbatim; replaying foreign items 400s ("Invalid reasoning item id + format"). """ raw = (msg.get("provider_specific_fields") or {}).get("reasoning_items") if raw is None: raw = msg.get("reasoning_items") + if not raw: + return [] + + origin = (msg.get("provider_specific_fields") or {}).get("reasoning_items_origin") + + if not current_model or not origin or origin != current_model: + return [] + if isinstance(raw, dict): return [raw] From 5a1517f6344d43d9f37aca09af272612c465fe86 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 11 Aug 2026 00:11:40 -0400 Subject: [PATCH 12/30] Add reasoning effort configuration to new LLM sub system --- cecli/helpers/llms/domains/gemini.py | 28 +- cecli/helpers/llms/domains/messages.py | 22 +- cecli/helpers/llms/domains/responses.py | 9 +- cecli/helpers/llms/litellm_compat.py | 15 + tests/helpers/test_llms_reasoning_config.py | 287 ++++++++++++++++++++ 5 files changed, 355 insertions(+), 6 deletions(-) create mode 100644 tests/helpers/test_llms_reasoning_config.py diff --git a/cecli/helpers/llms/domains/gemini.py b/cecli/helpers/llms/domains/gemini.py index 3e3e778637b..e1768793bc0 100644 --- a/cecli/helpers/llms/domains/gemini.py +++ b/cecli/helpers/llms/domains/gemini.py @@ -69,7 +69,23 @@ def gemini_payload( api_block = resolved.get("api_block") or {} gen_config = payload.setdefault("generationConfig", {}) - if api_block.get("reasoning_effort"): + # Caller/system overrides ride in ``kwargs["extra_body"]`` (models.py's + # ``set_reasoning_effort`` / ``set_thinking_tokens``). Map them onto + # ``generationConfig.thinkingConfig`` and keep the generic keys out of the + # top level (Gemini rejects unknown fields). + override_body = kwargs.get("extra_body") or {} + override_effort = override_body.get("reasoning_effort") + override_thinking = override_body.get("thinking") + + # The explicit ``set_thinking_tokens`` budget wins over the config-default + # ``reasoning_effort`` that rides in the same channel: gemini configs default + # to ``reasoning_effort`` (never ``thinking``), so an override ``thinking`` + # only appears when the user explicitly requested a thinking budget. + if isinstance(override_thinking, dict) and override_thinking.get("budget_tokens"): + gen_config["thinkingConfig"] = {"thinkingBudget": override_thinking["budget_tokens"]} + elif override_effort: + gen_config["thinkingConfig"] = gemini_thinking_config(resolved, override_effort) + elif api_block.get("reasoning_effort"): gen_config["thinkingConfig"] = gemini_thinking_config( resolved, api_block["reasoning_effort"] ) @@ -85,8 +101,14 @@ def gemini_payload( temperature = kwargs.get("temperature") if temperature is not None: payload.setdefault("generationConfig", {})["temperature"] = temperature - payload.update(resolved.get("extra_body") or {}) - payload.update(kwargs.get("extra_body") or {}) + + # Apply extra_body passthrough without leaking the generic reasoning keys + # (they are consumed above into thinkingConfig). + extra_body = dict(resolved.get("extra_body") or {}) + extra_body.update(kwargs.get("extra_body") or {}) + extra_body.pop("reasoning_effort", None) + extra_body.pop("thinking", None) + payload.update(extra_body) return payload diff --git a/cecli/helpers/llms/domains/messages.py b/cecli/helpers/llms/domains/messages.py index 94d2ffcc2e5..9a98a3f5b36 100644 --- a/cecli/helpers/llms/domains/messages.py +++ b/cecli/helpers/llms/domains/messages.py @@ -59,6 +59,18 @@ def anthropic_payload( api_block = resolved.get("api_block") or {} + # Caller/system overrides ride in ``kwargs["extra_body"]`` (models.py's + # ``set_reasoning_effort`` / ``set_thinking_tokens``). Merge them into the + # api_block so the generation-gated dispatcher below emits the right wire + # field (output_config.effort on Claude 5+, thinking block pre-5). + override_body = kwargs.get("extra_body") or {} + + if override_body.get("reasoning_effort"): + api_block = {**api_block, "reasoning_effort": override_body["reasoning_effort"]} + + if override_body.get("thinking"): + api_block = {**api_block, "thinking": override_body["thinking"]} + # Claude 5+ uses adaptive thinking via ``output_config.effort``; pre-5 # Claude uses the ``thinking`` block. Gate on the model generation so # e.g. claude-haiku-4-5 (no effort support) never receives output_config. @@ -70,8 +82,14 @@ def anthropic_payload( if temperature is not None: payload["temperature"] = temperature - payload.update(resolved.get("extra_body") or {}) - payload.update(kwargs.get("extra_body") or {}) + # Apply extra_body passthrough without leaking the generic reasoning keys + # (they are consumed above by format_thinking; Anthropic rejects unknown + # params). + extra_body = dict(resolved.get("extra_body") or {}) + extra_body.update(kwargs.get("extra_body") or {}) + extra_body.pop("reasoning_effort", None) + extra_body.pop("thinking", None) + payload.update(extra_body) return payload diff --git a/cecli/helpers/llms/domains/responses.py b/cecli/helpers/llms/domains/responses.py index a3a8cf9f6ad..6c1f34bc8d6 100644 --- a/cecli/helpers/llms/domains/responses.py +++ b/cecli/helpers/llms/domains/responses.py @@ -76,8 +76,15 @@ def responses_payload( if temperature is not None: payload["temperature"] = temperature + extra_body = dict(kwargs.get("extra_body") or {}) + # The Responses API controls reasoning via the nested ``reasoning.effort`` + # field (already handled above); a generic top-level ``thinking`` budget + # (or flat ``reasoning_effort``) has no wire equivalent and would be + # rejected as an unknown field. + extra_body.pop("reasoning_effort", None) + extra_body.pop("thinking", None) payload.update(resolved.get("extra_body") or {}) - payload.update(kwargs.get("extra_body") or {}) + payload.update(extra_body) return payload diff --git a/cecli/helpers/llms/litellm_compat.py b/cecli/helpers/llms/litellm_compat.py index aa9976e7513..b07af409a4f 100644 --- a/cecli/helpers/llms/litellm_compat.py +++ b/cecli/helpers/llms/litellm_compat.py @@ -765,6 +765,21 @@ async def acompletion(self, **kwargs: Any) -> Any: if kwargs.get(key) is not None: passthrough[key] = kwargs[key] + # The model-config pipeline formatters (helpers.format_reasoning / + # helpers.format_thinking) lift reasoning_effort/thinking OUT of + # extra_body into top-level kwargs (models.py set_reasoning_effort / + # set_thinking_tokens). Forward them on the same extra_body channel the + # domain payload builders consume; a top-level value wins over an + # extra_body copy (it is the post-format value). + extra_body = dict(kwargs.get("extra_body") or {}) + + for key in ("reasoning_effort", "thinking"): + if kwargs.get(key) is not None: + extra_body[key] = kwargs[key] + + if extra_body: + passthrough["extra_body"] = extra_body + max_tokens = kwargs.get("max_tokens") or kwargs.get("max_completion_tokens") if max_tokens: passthrough["max_tokens"] = max_tokens diff --git a/tests/helpers/test_llms_reasoning_config.py b/tests/helpers/test_llms_reasoning_config.py new file mode 100644 index 00000000000..2b19fdd352f --- /dev/null +++ b/tests/helpers/test_llms_reasoning_config.py @@ -0,0 +1,287 @@ +"""Reasoning-config path tests: Model settings -> shim -> domain wire payload. + +Covers the per-provider mapping of ``reasoning_effort`` (low/medium/high) and the +thinking budget (``set_thinking_tokens``) onto each domain's wire payload: + +- chat: flat ``reasoning_effort`` / ``thinking`` keys +- gemini: ``generationConfig.thinkingConfig`` (``thinkingLevel`` or + ``thinkingBudget``), with an explicit budget winning over the config-default + effort +- anthropic: ``output_config.effort`` (Claude 5+) or the ``thinking`` block + (pre-5) +- responses: nested ``reasoning.effort`` (generic keys stripped) + +Plus the shim's forwarding of the model-config formatters' top-level kwargs +(``LazyLiteLLM.acompletion``) into the ``extra_body`` channel the builders +consume. No network: the package dispatch is monkeypatched. +""" + +import asyncio + +import cecli.helpers.llms as llms_pkg +from cecli.helpers.llms.config import resolve_model_config +from cecli.helpers.llms.domains.chat import chat_payload +from cecli.helpers.llms.domains.gemini import gemini_payload +from cecli.helpers.llms.domains.messages import anthropic_payload +from cecli.helpers.llms.domains.responses import responses_payload +from cecli.helpers.llms.litellm_compat import litellm +from cecli.helpers.llms.types import Choice, CompletionResponse, Message + +MSGS = [{"role": "user", "content": "hi"}] + + +def _build(family, resolved, extra_body): + """Call the family payload builder with ``extra_body`` overrides.""" + kwargs = {"extra_body": dict(extra_body or {})} + if family == "chat": + return chat_payload(resolved, MSGS, None, False, kwargs) + if family == "gemini": + return gemini_payload(resolved, MSGS, None, kwargs) + if family == "anthropic": + return anthropic_payload(resolved, MSGS, None, False, kwargs) + return responses_payload(resolved, MSGS, None, False, kwargs) + + +def _wire(payload): + """Extract the reasoning-related wire fields from a built payload.""" + out = {} + gen = payload.get("generationConfig") or {} + if gen.get("thinkingConfig"): + out["thinkingConfig"] = gen["thinkingConfig"] + for key in ("reasoning_effort", "thinking", "reasoning", "output_config"): + if key in payload: + out[key] = payload[key] + return out + + +def _patch_dispatch(monkeypatch, captured): + """Monkeypatch the package dispatch, capturing the kwargs it receives.""" + + async def fake_dispatch(**kwargs): + captured.clear() + captured.update(kwargs) + return CompletionResponse( + id="x", + model=kwargs.get("model"), + choices=[Choice(index=0, message=Message(role="assistant", content="hi"))], + ) + + monkeypatch.setattr(llms_pkg, "acompletion", fake_dispatch) + + +# --------------------------------------------------------------------------- +# Builder-level wire mapping +# --------------------------------------------------------------------------- + + +def test_chat_flat_reasoning_effort(): + resolved = resolve_model_config("deepseek/deepseek-v4-flash") + payload = _build("chat", resolved, {"reasoning_effort": "high"}) + assert payload["reasoning_effort"] == "high" + + +def test_chat_flat_thinking_budget(): + resolved = resolve_model_config("deepseek/deepseek-v4-flash") + thinking = {"type": "enabled", "budget_tokens": 4096} + payload = _build("chat", resolved, {"thinking": thinking}) + assert payload["thinking"] == thinking + + +def test_gemini_effort_maps_to_thinking_level(): + resolved = resolve_model_config("gemini/gemini-3-flash-preview") + for effort, level in (("low", "low"), ("medium", "medium"), ("high", "high")): + payload = _build("gemini", resolved, {"reasoning_effort": effort}) + assert payload["generationConfig"]["thinkingConfig"] == { + "thinkingLevel": level, + "includeThoughts": True, + } + + +def test_gemini_thinking_budget(): + resolved = resolve_model_config("gemini/gemini-3-flash-preview") + payload = _build("gemini", resolved, {"thinking": {"type": "enabled", "budget_tokens": 4096}}) + assert payload["generationConfig"]["thinkingConfig"] == {"thinkingBudget": 4096} + + +def test_gemini_thinking_budget_wins_over_config_effort(): + """Regression: an explicit set_thinking_tokens budget beats the config-default + reasoning_effort that rides in the same channel.""" + resolved = resolve_model_config("gemini/gemini-3-flash-preview") + payload = _build( + "gemini", + resolved, + { + "reasoning_effort": "medium", + "thinking": {"type": "enabled", "budget_tokens": 4096}, + }, + ) + assert payload["generationConfig"]["thinkingConfig"] == {"thinkingBudget": 4096} + + +def test_gemini_generic_reasoning_keys_not_leaked(): + resolved = resolve_model_config("gemini/gemini-3-flash-preview") + payload = _build( + "gemini", + resolved, + { + "reasoning_effort": "high", + "thinking": {"type": "enabled", "budget_tokens": 4096}, + "other": 1, + }, + ) + assert "reasoning_effort" not in payload + assert "thinking" not in payload + assert payload["other"] == 1 + + +def test_anthropic_5_effort_to_output_config(): + resolved = resolve_model_config("claude-sonnet-5") + payload = _build("anthropic", resolved, {"reasoning_effort": "high"}) + assert payload["output_config"] == {"effort": "high"} + + +def test_anthropic_5_thinking_block_dropped(): + """Claude 5+ cannot use thinking.type.enabled; the block must not be sent.""" + resolved = resolve_model_config("claude-sonnet-5") + payload = _build( + "anthropic", resolved, {"thinking": {"type": "enabled", "budget_tokens": 4096}} + ) + assert "thinking" not in payload + assert "reasoning_effort" not in payload + + +def test_anthropic_pre5_thinking_budget(): + resolved = resolve_model_config("anthropic/claude-haiku-4-5-20251001") + payload = _build( + "anthropic", resolved, {"thinking": {"type": "enabled", "budget_tokens": 4096}} + ) + assert payload["thinking"] == {"type": "enabled", "budget_tokens": 4096} + + +def test_anthropic_pre5_effort_ignored(): + """Haiku 4.x has no effort support; the thinking block stays at the default.""" + resolved = resolve_model_config("anthropic/claude-haiku-4-5-20251001") + payload = _build("anthropic", resolved, {"reasoning_effort": "high"}) + assert "output_config" not in payload + assert "reasoning_effort" not in payload + assert payload["thinking"]["type"] == "enabled" + + +def test_responses_effort_override(): + resolved = resolve_model_config("meta/muse-spark-1.2-contributor") + payload = _build("responses", resolved, {"reasoning": {"effort": "low"}}) + assert payload["reasoning"]["effort"] == "low" + + +def test_responses_generic_reasoning_keys_stripped(): + resolved = resolve_model_config("meta/muse-spark-1.2-contributor") + payload = _build( + "responses", + resolved, + { + "reasoning_effort": "low", + "thinking": {"type": "enabled", "budget_tokens": 4096}, + }, + ) + assert "reasoning_effort" not in payload + assert "thinking" not in payload + + +# --------------------------------------------------------------------------- +# Shim forwarding of top-level reasoning kwargs +# --------------------------------------------------------------------------- + + +def test_shim_forwards_top_level_reasoning_kwargs(monkeypatch): + """Top-level reasoning_effort/thinking kwargs reach the dispatch extra_body.""" + captured = {} + _patch_dispatch(monkeypatch, captured) + + asyncio.run( + litellm.acompletion( + model="deepseek/deepseek-v4-flash", + messages=MSGS, + stream=False, + reasoning_effort="high", + thinking={"type": "enabled", "budget_tokens": 4096}, + ) + ) + + extra_body = captured.get("extra_body") or {} + assert extra_body["reasoning_effort"] == "high" + assert extra_body["thinking"] == {"type": "enabled", "budget_tokens": 4096} + + +# --------------------------------------------------------------------------- +# Full path: Model.set_* -> shim -> builder -> wire +# --------------------------------------------------------------------------- + + +def test_model_settings_reach_wire(monkeypatch): + """The program settings path carries low/medium/high + budget to each wire.""" + from cecli.models import Model + + captured = {} + _patch_dispatch(monkeypatch, captured) + + cases = [ + # (model, family, setter, value, expected wire reasoning fields) + ( + "deepseek/deepseek-v4-flash", + "chat", + "set_reasoning_effort", + "high", + {"reasoning_effort": "high"}, + ), + ( + "gemini/gemini-3-flash-preview", + "gemini", + "set_reasoning_effort", + "high", + {"thinkingConfig": {"thinkingLevel": "high", "includeThoughts": True}}, + ), + ( + "gemini/gemini-3-flash-preview", + "gemini", + "set_thinking_tokens", + "4k", + {"thinkingConfig": {"thinkingBudget": 4096}}, + ), + ( + "claude-sonnet-5", + "anthropic", + "set_reasoning_effort", + "low", + {"output_config": {"effort": "low"}}, + ), + ( + "anthropic/claude-haiku-4-5-20251001", + "anthropic", + "set_thinking_tokens", + "4k", + {"thinking": {"type": "enabled", "budget_tokens": 4096}}, + ), + ( + "meta/muse-spark-1.2-contributor", + "responses", + "set_reasoning_effort", + "low", + {"reasoning": {"effort": "low"}}, + ), + ] + + for label, family, setter, value, expected in cases: + captured.clear() + model = Model(label) + getattr(model, setter)(value) + kwargs = { + "model": model.name, + "stream": False, + "messages": MSGS, + **dict(model.extra_params or {}), + } + asyncio.run(litellm.acompletion(**kwargs)) + extra_body = captured.get("extra_body") or {} + payload = _build(family, resolve_model_config(label), extra_body) + wire = _wire(payload) + assert wire == expected, f"{label} {setter}({value!r}): got {wire}, expected {expected}" From 7a2fc1b6d5121889dcf411531e9c538796e57396 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 11 Aug 2026 00:25:56 -0400 Subject: [PATCH 13/30] Add tests for setting api base --- tests/helpers/test_llms_api_base.py | 117 ++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 tests/helpers/test_llms_api_base.py diff --git a/tests/helpers/test_llms_api_base.py b/tests/helpers/test_llms_api_base.py new file mode 100644 index 00000000000..dac5f461d28 --- /dev/null +++ b/tests/helpers/test_llms_api_base.py @@ -0,0 +1,117 @@ +"""api_base / base_url override tests for the llms package. + +Mirrors litellm's ``api_base`` override: a per-request ``api_base`` kwarg (via +the shim or directly on ``pipeline.acompletion``) overrides the provider's +default endpoint, ``{PROVIDER}_API_BASE`` env vars override globally at config +resolution, and each family adapter builds its request URL from the resolved +base. No network: the family adapter / package dispatch are monkeypatched. +""" + +import asyncio + +import cecli.helpers.llms as llms_pkg +import cecli.helpers.llms.pipeline as pipeline +from cecli.helpers.llms.config import resolve_model_config +from cecli.helpers.llms.litellm_compat import litellm +from cecli.helpers.llms.types import Choice, CompletionResponse, Message + +MSGS = [{"role": "user", "content": "hi"}] + + +def _fake_response(model): + return CompletionResponse( + id="x", + model=model, + choices=[Choice(index=0, message=Message(role="assistant", content="hi"))], + ) + + +# --------------------------------------------------------------------------- +# Config-level precedence +# --------------------------------------------------------------------------- + + +def test_default_api_base_from_provider_defaults(): + resolved = resolve_model_config("deepseek/deepseek-v4-flash") + assert resolved["api_base"] == "https://api.deepseek.com/v1" + + +def test_env_api_base_overrides_default(monkeypatch): + monkeypatch.setenv("DEEPSEEK_API_BASE", "https://env.example.com/v1") + resolved = resolve_model_config("deepseek/deepseek-v4-flash") + assert resolved["api_base"] == "https://env.example.com/v1" + + +def test_env_api_base_trailing_slash_stripped(monkeypatch): + monkeypatch.setenv("DEEPSEEK_API_BASE", "https://env.example.com/v1/") + resolved = resolve_model_config("deepseek/deepseek-v4-flash") + assert resolved["api_base"] == "https://env.example.com/v1" + + +# --------------------------------------------------------------------------- +# Per-request override via the pipeline +# --------------------------------------------------------------------------- + + +def test_pipeline_api_base_override_reaches_adapter(monkeypatch): + """pipeline.acompletion(api_base=...) overrides the resolved base (rstripped).""" + captured = {} + + async def fake_chat_complete(resolved, messages, tools, key, headers, kwargs): + captured["api_base"] = resolved["api_base"] + captured["family"] = resolved["family"] + return _fake_response(resolved["model"]) + + monkeypatch.setattr(pipeline, "chat_complete", fake_chat_complete) + + asyncio.run( + pipeline.acompletion( + model="deepseek/deepseek-v4-flash", + messages=MSGS, + api_base="https://my-proxy.example.com/v1/", + ) + ) + + assert captured["api_base"] == "https://my-proxy.example.com/v1" + assert captured["family"] == "chat" + + +def test_pipeline_without_api_base_uses_default(monkeypatch): + captured = {} + + async def fake_chat_complete(resolved, messages, tools, key, headers, kwargs): + captured["api_base"] = resolved["api_base"] + return _fake_response(resolved["model"]) + + monkeypatch.setattr(pipeline, "chat_complete", fake_chat_complete) + + asyncio.run(pipeline.acompletion(model="deepseek/deepseek-v4-flash", messages=MSGS)) + + assert captured["api_base"] == "https://api.deepseek.com/v1" + + +# --------------------------------------------------------------------------- +# Shim forwarding +# --------------------------------------------------------------------------- + + +def test_shim_forwards_api_base_to_dispatch(monkeypatch): + """litellm.acompletion(api_base=...) reaches the package dispatch.""" + captured = {} + + async def fake_dispatch(**kwargs): + captured.update(kwargs) + return _fake_response(kwargs.get("model")) + + monkeypatch.setattr(llms_pkg, "acompletion", fake_dispatch) + + asyncio.run( + litellm.acompletion( + model="deepseek/deepseek-v4-flash", + messages=MSGS, + stream=False, + api_base="https://my-proxy.example.com/v1", + ) + ) + + assert captured.get("api_base") == "https://my-proxy.example.com/v1" From da16bb53b9285478f7b85c3001475e22cf98c264 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 11 Aug 2026 00:36:10 -0400 Subject: [PATCH 14/30] Add images support for all providers --- cecli/helpers/llms/domains/gemini.py | 47 ++++++- cecli/helpers/llms/domains/messages.py | 46 ++++++- cecli/helpers/llms/domains/responses.py | 45 ++++++ cecli/helpers/llms/utils.py | 25 +++- tests/helpers/test_llms_vision.py | 176 ++++++++++++++++++++++++ 5 files changed, 335 insertions(+), 4 deletions(-) create mode 100644 tests/helpers/test_llms_vision.py diff --git a/cecli/helpers/llms/domains/gemini.py b/cecli/helpers/llms/domains/gemini.py index e1768793bc0..b267ced2dcc 100644 --- a/cecli/helpers/llms/domains/gemini.py +++ b/cecli/helpers/llms/domains/gemini.py @@ -25,7 +25,7 @@ Usage, parts_message_to_message, ) -from ..utils import sse_json_lines, system_prompt +from ..utils import split_data_url, sse_json_lines, system_prompt DEFAULT_TIMEOUT = 120.0 @@ -163,6 +163,9 @@ def gemini_content( if isinstance(content, str): return {"role": "user", "parts": [{"text": content}]} + if isinstance(content, list): + return {"role": "user", "parts": _gemini_user_parts(content)} + parts: List[Dict[str, Any]] = [] if content: @@ -696,6 +699,48 @@ def _gemini_schema(schema: Dict[str, Any]) -> Dict[str, Any]: return cleaned +def _gemini_user_parts(content: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Translate OpenAI-style user content parts into Gemini ``parts``. + + - ``text`` parts become ``{"text": ...}`` + - ``image_url`` data URLs become ``{"inlineData": {"mimeType", "data"}}`` + - ``image_url`` http(s) URLs become ``{"fileData": {"fileUri"}}`` + - anything else is JSON-serialized into a text part (never dropped) + """ + parts: List[Dict[str, Any]] = [] + + for part in content: + if not isinstance(part, dict): + parts.append({"text": json.dumps(part)}) + + continue + + if part.get("type") == "text" and isinstance(part.get("text"), str): + parts.append({"text": part["text"]}) + + continue + + if part.get("type") == "image_url": + image_url = part.get("image_url") + url = image_url.get("url") if isinstance(image_url, dict) else None + parsed = split_data_url(url) + + if parsed: + mime, data = parsed + parts.append({"inlineData": {"mimeType": mime, "data": data}}) + + continue + + if isinstance(url, str) and url.startswith(("http://", "https://")): + parts.append({"fileData": {"fileUri": url}}) + + continue + + parts.append({"text": json.dumps(part)}) + + return parts + + __all__ = [ "gemini_payload", "gemini_thinking_config", diff --git a/cecli/helpers/llms/domains/messages.py b/cecli/helpers/llms/domains/messages.py index 9a98a3f5b36..412e5e7bea9 100644 --- a/cecli/helpers/llms/domains/messages.py +++ b/cecli/helpers/llms/domains/messages.py @@ -29,7 +29,7 @@ Usage, parts_message_to_message, ) -from ..utils import sse_json_lines, system_prompt +from ..utils import split_data_url, sse_json_lines, system_prompt DEFAULT_TIMEOUT = 120.0 @@ -144,6 +144,9 @@ def anthropic_message(msg: Dict[str, Any]) -> Dict[str, Any]: return {"role": "assistant", "content": blocks} content = msg.get("content") + if isinstance(content, list): + return {"role": role, "content": _anthropic_user_blocks(content)} + return {"role": role, "content": content if isinstance(content, str) else json.dumps(content)} @@ -542,6 +545,47 @@ def _anthropic_message_content(msg: Dict[str, Any]) -> Optional[List[Dict[str, A return content or None +def _anthropic_user_blocks(content: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Translate OpenAI-style user content parts into Anthropic content blocks. + + - ``text`` parts become ``{"type": "text", ...}`` + - ``image_url`` base64 data URLs become ``{"type": "image", "source": {base64}}`` + - anything else is JSON-serialized into a text block (never dropped) + """ + blocks: List[Dict[str, Any]] = [] + + for part in content: + if not isinstance(part, dict): + blocks.append({"type": "text", "text": json.dumps(part)}) + + continue + + if part.get("type") == "text" and isinstance(part.get("text"), str): + blocks.append({"type": "text", "text": part["text"]}) + + continue + + if part.get("type") == "image_url": + image_url = part.get("image_url") + url = image_url.get("url") if isinstance(image_url, dict) else None + parsed = split_data_url(url) + + if parsed: + mime, data = parsed + blocks.append( + { + "type": "image", + "source": {"type": "base64", "media_type": mime, "data": data}, + } + ) + + continue + + blocks.append({"type": "text", "text": json.dumps(part)}) + + return blocks + + __all__ = [ "anthropic_payload", "anthropic_message", diff --git a/cecli/helpers/llms/domains/responses.py b/cecli/helpers/llms/domains/responses.py index 6c1f34bc8d6..0566d8ea597 100644 --- a/cecli/helpers/llms/domains/responses.py +++ b/cecli/helpers/llms/domains/responses.py @@ -152,6 +152,17 @@ def to_responses_input( ) continue + if isinstance(content, list): + items.append( + { + "type": "message", + "role": role, + "content": _responses_user_content(content), + } + ) + + continue + text = content if isinstance(content, str) else json.dumps(content) items.append( {"type": "message", "role": role, "content": [{"type": "input_text", "text": text}]} @@ -499,6 +510,40 @@ def _build_usage(usage_raw: Dict[str, Any]) -> Usage: ) +def _responses_user_content(content: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Translate OpenAI-style user content parts into responses-API content parts. + + - ``text`` parts become ``{"type": "input_text", ...}`` + - ``image_url`` parts become ``{"type": "input_image", "image_url": }`` + - anything else is JSON-serialized into an ``input_text`` part (never dropped) + """ + out: List[Dict[str, Any]] = [] + + for part in content: + if not isinstance(part, dict): + out.append({"type": "input_text", "text": json.dumps(part)}) + + continue + + if part.get("type") == "text" and isinstance(part.get("text"), str): + out.append({"type": "input_text", "text": part["text"]}) + + continue + + if part.get("type") == "image_url": + image_url = part.get("image_url") + url = image_url.get("url") if isinstance(image_url, dict) else None + + if isinstance(url, str): + out.append({"type": "input_image", "image_url": url}) + + continue + + out.append({"type": "input_text", "text": json.dumps(part)}) + + return out + + __all__ = [ "responses_payload", "to_responses_input", diff --git a/cecli/helpers/llms/utils.py b/cecli/helpers/llms/utils.py index 81c1a84f785..d498f31f0fc 100644 --- a/cecli/helpers/llms/utils.py +++ b/cecli/helpers/llms/utils.py @@ -7,7 +7,8 @@ from __future__ import annotations import json -from typing import Any, AsyncIterator, Dict, List, Optional +import re +from typing import Any, AsyncIterator, Dict, List, Optional, Tuple import httpx @@ -73,4 +74,24 @@ def extract_reasoning(msg: Dict[str, Any]) -> str: return "\n".join(parts) -__all__ = ["sse_json_lines", "system_prompt", "extract_reasoning"] +_DATA_URL_RE = re.compile(r"data:([^;,]+)(;base64)?,(.*)", re.DOTALL) + + +def split_data_url(url: Any) -> Optional[Tuple[str, str]]: + """Parse a ``data:;base64,`` URL into ``(mime_type, data)``. + + Returns None for non-data URLs (https://..., gs://...) or non-base64 + payloads so callers can fall back to fileData / text placeholders. + """ + if not isinstance(url, str): + return None + + match = _DATA_URL_RE.match(url) + + if not match or not match.group(2): + return None + + return (match.group(1) or "application/octet-stream", match.group(3)) + + +__all__ = ["sse_json_lines", "system_prompt", "extract_reasoning", "split_data_url"] diff --git a/tests/helpers/test_llms_vision.py b/tests/helpers/test_llms_vision.py new file mode 100644 index 00000000000..75dfcb851ad --- /dev/null +++ b/tests/helpers/test_llms_vision.py @@ -0,0 +1,176 @@ +"""Vision / image-input mapping tests: OpenAI-style content parts -> domain wire. + +Covers the translation of multimodal user messages (``[{"type": "text", ...}, +{"type": "image_url", ...}]``) onto each family's wire payload: + +- chat: content lists pass through verbatim (OpenAI-style ``image_url`` parts) +- gemini: base64 data URLs -> ``inlineData {mimeType, data}``; http(s) URLs -> + ``fileData {fileUri}`` +- anthropic: base64 data URLs -> ``{"type": "image", "source": {base64}}`` + blocks; http(s) URLs fall back to JSON text (Anthropic only accepts base64) +- responses: ``image_url`` parts -> ``{"type": "input_image", "image_url"}`` + items + +Unknown / malformed parts are JSON-serialized into a text part (never dropped). +No network: only the offline payload builders are exercised. +""" + +from cecli.helpers.llms.config import resolve_model_config +from cecli.helpers.llms.domains.chat import chat_payload +from cecli.helpers.llms.domains.gemini import gemini_payload +from cecli.helpers.llms.domains.messages import anthropic_payload +from cecli.helpers.llms.domains.responses import responses_payload +from cecli.helpers.llms.utils import split_data_url + +DATA_URL = "data:image/png;base64,iVBORw0KGgo=" +HTTP_URL = "https://example.com/pic.png" + +#: OpenAI-style multimodal user message (the shape cecli stores in history). +MULTIMODAL = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "what is in this image?"}, + {"type": "image_url", "image_url": {"url": DATA_URL}}, + {"type": "image_url", "image_url": {"url": HTTP_URL}}, + ], + }, +] + + +def _gemini_parts(): + resolved = resolve_model_config("gemini/gemini-3-flash-preview") + payload = gemini_payload(resolved, MULTIMODAL, None, {}) + return payload["contents"][0]["parts"] + + +def _anthropic_blocks(): + resolved = resolve_model_config("claude-sonnet-5") + payload = anthropic_payload(resolved, MULTIMODAL, None, False, {}) + return payload["messages"][0]["content"] + + +def _responses_content(): + resolved = resolve_model_config("openai/gpt-5.6-luna") + payload = responses_payload(resolved, MULTIMODAL, None, False, {}) + return payload["input"][0]["content"] + + +def _chat_content(): + resolved = resolve_model_config("deepseek/deepseek-v4-flash") + payload = chat_payload(resolved, MULTIMODAL, None, False, {}) + return payload["messages"][0]["content"] + + +# --------------------------------------------------------------------------- +# split_data_url unit checks +# --------------------------------------------------------------------------- + + +def test_split_data_url_parses_base64_data_url(): + assert split_data_url(DATA_URL) == ("image/png", "iVBORw0KGgo=") + + +def test_split_data_url_rejects_non_base64_and_http(): + assert split_data_url("data:text/plain,hello") is None + assert split_data_url(HTTP_URL) is None + assert split_data_url(None) is None + + +# --------------------------------------------------------------------------- +# gemini: inlineData / fileData parts +# --------------------------------------------------------------------------- + + +def test_gemini_data_url_becomes_inline_data(): + parts = _gemini_parts() + assert parts[0] == {"text": "what is in this image?"} + assert parts[1] == {"inlineData": {"mimeType": "image/png", "data": "iVBORw0KGgo="}} + + +def test_gemini_http_url_becomes_file_data(): + parts = _gemini_parts() + assert parts[2] == {"fileData": {"fileUri": HTTP_URL}} + + +def test_gemini_unknown_part_falls_back_to_text(): + resolved = resolve_model_config("gemini/gemini-3-flash-preview") + msgs = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "hi"}, + {"type": "audio_url", "audio_url": {"url": "https://example.com/a.mp3"}}, + "not-a-dict", + ], + } + ] + payload = gemini_payload(resolved, msgs, None, {}) + parts = payload["contents"][0]["parts"] + assert parts[0] == {"text": "hi"} + assert parts[1]["text"].startswith("{") + assert parts[2] == {"text": '"not-a-dict"'} + + +# --------------------------------------------------------------------------- +# anthropic: base64 image blocks (http falls back to text) +# --------------------------------------------------------------------------- + + +def test_anthropic_data_url_becomes_image_block(): + blocks = _anthropic_blocks() + assert blocks[0] == {"type": "text", "text": "what is in this image?"} + assert blocks[1] == { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgo="}, + } + + +def test_anthropic_http_url_falls_back_to_text(): + blocks = _anthropic_blocks() + assert blocks[2]["type"] == "text" + assert HTTP_URL in blocks[2]["text"] + + +# --------------------------------------------------------------------------- +# responses: input_image items +# --------------------------------------------------------------------------- + + +def test_responses_image_url_becomes_input_image(): + content = _responses_content() + assert content[0] == {"type": "input_text", "text": "what is in this image?"} + assert content[1] == {"type": "input_image", "image_url": DATA_URL} + assert content[2] == {"type": "input_image", "image_url": HTTP_URL} + + +# --------------------------------------------------------------------------- +# chat: pass-through unchanged +# --------------------------------------------------------------------------- + + +def test_chat_multimodal_passes_through_verbatim(): + content = _chat_content() + assert content == MULTIMODAL[0]["content"] + + +# --------------------------------------------------------------------------- +# malformed image_url never drops the part +# --------------------------------------------------------------------------- + + +def test_malformed_image_url_falls_back_to_text(): + resolved = resolve_model_config("openai/gpt-5.6-luna") + msgs = [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": "not-a-dict"}, + {"type": "image_url"}, + ], + } + ] + payload = responses_payload(resolved, msgs, None, False, {}) + content = payload["input"][0]["content"] + assert len(content) == 2 + assert all(item["type"] == "input_text" for item in content) From f2703ee96f7db276d422cfe794fc9505652ff6ff Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 11 Aug 2026 00:47:37 -0400 Subject: [PATCH 15/30] Make sure providers.json is actually laoded --- tests/helpers/test_llms_providers_json.py | 63 +++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 tests/helpers/test_llms_providers_json.py diff --git a/tests/helpers/test_llms_providers_json.py b/tests/helpers/test_llms_providers_json.py new file mode 100644 index 00000000000..e7d234ae05f --- /dev/null +++ b/tests/helpers/test_llms_providers_json.py @@ -0,0 +1,63 @@ +"""providers.json coverage: every OpenAI-compatible provider resolves. + +``cecli/resources/providers.json`` lists the OpenAI-compatible providers cecli +ships with (base URL + API-key env var). This locks in that each one: + +- is loaded by ``ModelProviderManager`` (supports_provider == True) +- resolves through ``resolve_model_config`` to the ``chat`` family (the + OpenAI-compatible /chat/completions wire, which is what these providers + advertise) +- keeps its configured base URL and key env var +- routes through the base OpenAI-style provider adapter (Bearer auth), since + none of them register a custom adapter + +No network: resolution and adapter dispatch are offline. +""" + +import importlib.resources as importlib_resources +import json + +from cecli.helpers.llms.config import resolve_model_config +from cecli.helpers.llms.providers import ProviderAdapter, get_provider_adapter +from cecli.helpers.model_providers import ModelProviderManager + +RESOURCE_FILE = "providers.json" + + +def _providers_json() -> dict: + resource = importlib_resources.files("cecli.resources").joinpath(RESOURCE_FILE) + return json.loads(resource.read_text()) + + +def test_every_providers_json_entry_is_supported(): + providers = _providers_json() + assert providers, "providers.json should not be empty" + mpm = ModelProviderManager() + + for name, cfg in providers.items(): + assert mpm.supports_provider(name), f"{name} should be supported" + pcfg = mpm.get_provider_config(name) or {} + assert pcfg.get("api_base") == cfg["api_base"] + assert pcfg.get("api_key_env") == cfg["api_key_env"] + + +def test_every_provider_resolves_to_chat_family_with_base_and_key(): + providers = _providers_json() + + for name, cfg in providers.items(): + resolved = resolve_model_config(f"{name}/sample-model") + assert resolved["family"] == "chat", f"{name} should use chat completions" + assert resolved["provider"] == name + assert resolved["api_base"] == cfg["api_base"].rstrip("/") + assert resolved["api_key_env"] in cfg["api_key_env"] + + +def test_unregistered_providers_use_openai_style_base_adapter(): + registry_names = set(_providers_json()) + + # None of the providers.json entries register a dedicated adapter, so + # dispatch falls back to the base OpenAI-style adapter (Bearer auth). + for name in registry_names: + adapter = get_provider_adapter(name) + assert isinstance(adapter, ProviderAdapter) + assert adapter.provider == "openai" From 9bee4da7bea1c75a375a2d0456487b971becca3d Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 11 Aug 2026 00:56:43 -0400 Subject: [PATCH 16/30] Fix for models providing both reasoning and reasoning_details --- cecli/helpers/llms/utils.py | 30 ++++++---- tests/helpers/test_llms_reasoning_extract.py | 58 ++++++++++++++++++++ 2 files changed, 78 insertions(+), 10 deletions(-) create mode 100644 tests/helpers/test_llms_reasoning_extract.py diff --git a/cecli/helpers/llms/utils.py b/cecli/helpers/llms/utils.py index d498f31f0fc..7d2b1dee4e0 100644 --- a/cecli/helpers/llms/utils.py +++ b/cecli/helpers/llms/utils.py @@ -52,24 +52,34 @@ def extract_reasoning(msg: Dict[str, Any]) -> str: - ``reasoning_content`` (str) - deepseek-style - ``reasoning`` (str) - openrouter - ``reasoning_details`` (list of {"type": "reasoning.text", "text": ...}) - """ - parts: List[str] = [] - - for key in ("reasoning_content", "reasoning"): - val = msg.get(key) - - if isinstance(val, str) and val.strip(): - parts.append(val) + OpenRouter (and minimax via OpenRouter) sends BOTH the flat ``reasoning`` + string AND a ``reasoning_details`` list holding the *same* incremental text + on every delta; combining them doubled the reasoning. The structured list + is authoritative when present; the flat string is only a fallback. + """ details = msg.get("reasoning_details") or msg.get("reasoning_content_details") - if isinstance(details, list): + if isinstance(details, list) and details: + texts: List[str] = [] + for item in details: if isinstance(item, dict): text = item.get("text") if isinstance(text, str) and text.strip(): - parts.append(text) + texts.append(text) + + if texts: + return "\n".join(texts) + + parts: List[str] = [] + + for key in ("reasoning_content", "reasoning"): + val = msg.get(key) + + if isinstance(val, str) and val.strip(): + parts.append(val) return "\n".join(parts) diff --git a/tests/helpers/test_llms_reasoning_extract.py b/tests/helpers/test_llms_reasoning_extract.py new file mode 100644 index 00000000000..0cb8ec69a5f --- /dev/null +++ b/tests/helpers/test_llms_reasoning_extract.py @@ -0,0 +1,58 @@ +"""extract_reasoning unit tests: shape handling + no double-counting. + +``cecli.helpers.llms.utils.extract_reasoning`` reads reasoning from the three +wild shapes (``reasoning_content`` string, ``reasoning`` string, +``reasoning_details`` list). OpenRouter (and minimax via OpenRouter) sends BOTH +the flat ``reasoning`` string AND a ``reasoning_details`` list holding the same +incremental text on every delta; the extractor must use the structured list +authoritatively and NOT combine both (that doubled every reasoning fragment). +""" + +from cecli.helpers.llms.utils import extract_reasoning + + +def test_reasoning_string_only(): + assert extract_reasoning({"reasoning": "think think"}) == "think think" + + +def test_reasoning_content_string_only(): + assert extract_reasoning({"reasoning_content": "deepseek thinks"}) == "deepseek thinks" + + +def test_reasoning_details_list_only(): + delta = { + "reasoning_details": [ + {"type": "reasoning.text", "text": "first"}, + {"type": "reasoning.text", "text": "second"}, + ] + } + assert extract_reasoning(delta) == "first\nsecond" + + +def test_both_string_and_details_not_doubled(): + # The exact minimax/openrouter regression: same text in both fields. + delta = { + "reasoning": "The user asks hello", + "reasoning_details": [{"type": "reasoning.text", "text": "The user asks hello"}], + } + assert extract_reasoning(delta) == "The user asks hello" + + +def test_details_wins_when_string_is_shorter_suffix(): + # Incremental deltas: the details list is authoritative even when the + # flat string differs in length (it is the same content by construction). + delta = { + "reasoning": " suffix", + "reasoning_details": [{"type": "reasoning.text", "text": "prefix suffix"}], + } + assert extract_reasoning(delta) == "prefix suffix" + + +def test_empty_details_falls_back_to_string(): + delta = {"reasoning": "fallback", "reasoning_details": []} + assert extract_reasoning(delta) == "fallback" + + +def test_no_reasoning_returns_empty(): + assert extract_reasoning({"content": "hi"}) == "" + assert extract_reasoning({}) == "" From 2493fa65fd6829fcb72235732e5f02ec211d8dc5 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 11 Aug 2026 01:18:38 -0400 Subject: [PATCH 17/30] Add optimistic refreshing for copilot tokens --- .../helpers/llms/providers/github_copilot.py | 75 +++- .../helpers/test_llms_github_copilot_auth.py | 419 ++++++++++++++++++ 2 files changed, 488 insertions(+), 6 deletions(-) create mode 100644 tests/helpers/test_llms_github_copilot_auth.py diff --git a/cecli/helpers/llms/providers/github_copilot.py b/cecli/helpers/llms/providers/github_copilot.py index bc927eae283..68bed22cbf0 100644 --- a/cecli/helpers/llms/providers/github_copilot.py +++ b/cecli/helpers/llms/providers/github_copilot.py @@ -3,9 +3,10 @@ Mirrors litellm's ``Authenticator`` (llms/github_copilot/authenticator.py): an access token (from device flow) is exchanged for a short-lived Copilot API key via ``https://api.github.com/copilot_internal/v2/token``; the key is cached -to disk (api-key.json) and refreshed on expiry. The tenant endpoint -(``endpoints.api``) comes from the authenticated session, never a caller- -supplied base (token-leak prevention, matching litellm). +to disk (api-key.json) and refreshed on expiry, within a leeway of it, and +periodically for long-lived keys (see the ``COPILOT_*`` refresh constants). +The tenant endpoint (``endpoints.api``) comes from the authenticated session, +never a caller-supplied base (token-leak prevention, matching litellm). """ from __future__ import annotations @@ -28,6 +29,17 @@ COPILOT_TOKEN_DIR = os.path.expanduser("~/.config/litellm/github_copilot") COPILOT_TIMEOUT = 120.0 +#: Refresh the Copilot API key when it is within this many seconds of +#: expiring, so a key never dies mid-request. +COPILOT_REFRESH_LEEWAY = 300.0 # 5 minutes + +#: Keys whose total lifetime exceeds this are rotated at most every +#: ``COPILOT_MAX_REFRESH_INTERVAL`` instead of waiting for natural expiry. +COPILOT_LONG_LIVED_THRESHOLD = 24 * 3600.0 # 1 day + +#: Maximum age of a long-lived key before it is refreshed. +COPILOT_MAX_REFRESH_INTERVAL = 4 * 3600.0 # 4 hours + class CopilotAuthenticator: """GitHub Copilot OAuth: device flow + disk-cached API key with refresh.""" @@ -38,25 +50,43 @@ def __init__(self) -> None: self.api_key_file = os.path.join(self.token_dir, "api-key.json") def get_api_key(self) -> Optional[str]: - """Return a valid Copilot API key, refreshing from disk if needed.""" + """Return a valid Copilot API key, refreshing from disk if needed. + + The cached key is refreshed when it is expired, when it is within + ``COPILOT_REFRESH_LEEWAY`` of expiring, or when a long-lived key + (lifetime > ``COPILOT_LONG_LIVED_THRESHOLD``) has been held for more + than ``COPILOT_MAX_REFRESH_INTERVAL``. A failed refresh keeps serving + a cached key that is still unexpired instead of failing the request. + """ + cached = None + expired = False + try: with open(self.api_key_file) as f: info = json.load(f) - if info.get("expires_at", 0) > time.time(): - return info.get("token") + cached = info.get("token") + + if not self._should_refresh(info): + return cached + + expired = info.get("expires_at", 0) <= time.time() except (IOError, json.JSONDecodeError): pass try: info = self._refresh_api_key() os.makedirs(self.token_dir, exist_ok=True) + info["refreshed_at"] = time.time() with open(self.api_key_file, "w") as f: json.dump(info, f) return info.get("token") except Exception: + if cached and not expired: + return cached + return None def get_api_base(self) -> Optional[str]: @@ -142,6 +172,39 @@ def _device_flow_login(self) -> str: raise RuntimeError("Timed out waiting for user to authorize the device") + def _should_refresh(self, info: Dict[str, Any]) -> bool: + """True when the cached key should be replaced. + + Refreshes when the key is expired, within ``COPILOT_REFRESH_LEEWAY`` + of expiring, or when a long-lived key has been held for more than + ``COPILOT_MAX_REFRESH_INTERVAL`` (periodic rotation). Rotation + metadata (``refreshed_at``) is stamped when this version caches a + key; legacy caches fall back to the natural-expiry rules. + """ + now = time.time() + expires_at = info.get("expires_at", 0) + + if expires_at <= now: + return True + + if expires_at - now <= COPILOT_REFRESH_LEEWAY: + return True + + refreshed_at = info.get("refreshed_at") or info.get("issued_at") + + if not refreshed_at: + return False + + lifetime = expires_at - refreshed_at + + if ( + lifetime > COPILOT_LONG_LIVED_THRESHOLD + and now - refreshed_at >= COPILOT_MAX_REFRESH_INTERVAL + ): + return True + + return False + #: Module-level singleton so config resolution and pipeline share one instance. _AUTH: Optional[CopilotAuthenticator] = None diff --git a/tests/helpers/test_llms_github_copilot_auth.py b/tests/helpers/test_llms_github_copilot_auth.py new file mode 100644 index 00000000000..188c2e872bf --- /dev/null +++ b/tests/helpers/test_llms_github_copilot_auth.py @@ -0,0 +1,419 @@ +"""GitHub Copilot auth key flow tests (offline; no network). + +Drives :mod:`cecli.helpers.llms.providers.github_copilot` through its full +auth lifecycle without touching the network or the real +``~/.config/litellm/github_copilot`` cache: + +- cached ``api-key.json`` hit path (no HTTP at all) +- expired/missing key -> refresh from the GitHub token endpoint +- ``access-token`` cache + device-flow login (code print + poll) +- api base resolution from the cached session ``endpoints.api`` +- ``copilot_headers()`` shape (messages-proxy vs conversation-panel) +- wiring through ``resolve_model_config`` / ``get_api_key`` and the + ``GithubCopilotProvider`` adapter, down to the family adapter + +``httpx.get`` / ``httpx.post`` are monkeypatched and +``GITHUB_COPILOT_TOKEN_DIR`` points at a tmp dir, so every path is hermetic. +""" + +import asyncio +import json +import time + +import pytest + +import cecli.helpers.llms.pipeline as pipeline +from cecli.helpers.llms import config as llms_config +from cecli.helpers.llms.providers import github_copilot as copilot +from cecli.helpers.llms.providers.github_copilot import ( + COPILOT_ACCESS_TOKEN_URL, + COPILOT_DEFAULT_API_BASE, + COPILOT_DEVICE_CODE_URL, + CopilotAuthenticator, + GithubCopilotProvider, + copilot_api_base, + copilot_api_key, + copilot_headers, +) + +SAMPLE_KEY = { + "token": "sk-cached", + "expires_at": time.time() + 3600, + "endpoints": {"api": "https://tenant.githubcopilot.com"}, +} + +FRESH_KEY = { + "token": "sk-fresh", + "expires_at": time.time() + 3600, + "endpoints": {"api": "https://tenant.githubcopilot.com"}, +} + +DEVICE_INFO = { + "verification_uri": "https://github.com/login/device", + "user_code": "ABCD-1234", + "device_code": "dev-1", +} + +MSGS = [{"role": "user", "content": "hi"}] + + +class _FakeResponse: + """Minimal httpx.Response stand-in (raise_for_status + json).""" + + def __init__(self, payload, *, ok=True): + self._payload = payload + self._ok = ok + + def raise_for_status(self): + if not self._ok: + raise RuntimeError("http error") + + def json(self): + return self._payload + + +@pytest.fixture +def auth(tmp_path, monkeypatch): + """Fresh authenticator rooted at a tmp dir; module singleton reset.""" + monkeypatch.setenv("GITHUB_COPILOT_TOKEN_DIR", str(tmp_path)) + monkeypatch.setattr(copilot, "_AUTH", None) + + return CopilotAuthenticator() + + +def _seed_api_key(token_dir, payload=SAMPLE_KEY): + (token_dir / "api-key.json").write_text(json.dumps(payload)) + + +def _seed_expired_key(token_dir): + _seed_api_key(token_dir, dict(SAMPLE_KEY, expires_at=time.time() - 100)) + + +def _key_payload(*, token="sk-cached", expires_at=None, refreshed_at=None): + """Build an api-key.json payload with optional rotation metadata.""" + + payload = { + "token": token, + "expires_at": expires_at if expires_at is not None else time.time() + 3600, + "endpoints": {"api": "https://tenant.githubcopilot.com"}, + } + + if refreshed_at is not None: + payload["refreshed_at"] = refreshed_at + + return payload + + +def _seed_access_token(token_dir, token="tok-gh"): + (token_dir / "access-token").write_text(token) + + +def _no_network(monkeypatch): + def fail(*args, **kwargs): + raise AssertionError("unexpected network call") + + monkeypatch.setattr(copilot.httpx, "get", fail) + monkeypatch.setattr(copilot.httpx, "post", fail) + + +def _fake_refresh(monkeypatch, payload): + monkeypatch.setattr(copilot.httpx, "get", lambda *a, **k: _FakeResponse(payload)) + + +def _fake_device_flow(monkeypatch, poll_payloads): + """Stub httpx.post for the device-code and token-poll endpoints.""" + calls = [] + + def fake_post(url, **kwargs): + calls.append(url) + + if url == COPILOT_DEVICE_CODE_URL: + return _FakeResponse(DEVICE_INFO) + + if poll_payloads: + return _FakeResponse(poll_payloads.pop(0)) + + return _FakeResponse({}) + + monkeypatch.setattr(copilot.httpx, "post", fake_post) + monkeypatch.setattr(copilot.time, "sleep", lambda *a: None) + + return calls + + +# --------------------------------------------------------------------------- +# Cached api-key.json hit path (no network) +# --------------------------------------------------------------------------- + + +def test_get_api_key_returns_valid_cached_key(auth, tmp_path, monkeypatch): + _seed_api_key(tmp_path) + _no_network(monkeypatch) + + assert auth.get_api_key() == "sk-cached" + + +def test_copilot_api_key_module_function(auth, tmp_path): + _seed_api_key(tmp_path) + + assert copilot_api_key() == "sk-cached" + + +def test_get_api_base_returns_session_endpoint(auth, tmp_path): + _seed_api_key(tmp_path) + + assert auth.get_api_base() == "https://tenant.githubcopilot.com" + + +def test_get_api_base_none_when_no_cache(auth): + assert auth.get_api_base() is None + + +def test_copilot_api_base_falls_back_to_default(auth): + assert copilot_api_base() == COPILOT_DEFAULT_API_BASE + + +def test_get_access_token_from_cache(auth, tmp_path, monkeypatch): + _seed_access_token(tmp_path) + _no_network(monkeypatch) + + assert auth.get_access_token() == "tok-gh" + + +# --------------------------------------------------------------------------- +# Refresh path (expired/missing key -> GitHub token endpoint) +# --------------------------------------------------------------------------- + + +def test_get_api_key_refreshes_when_expired(auth, tmp_path, monkeypatch): + _seed_expired_key(tmp_path) + _seed_access_token(tmp_path) + _fake_refresh(monkeypatch, FRESH_KEY) + + assert auth.get_api_key() == "sk-fresh" + + cached = json.loads((tmp_path / "api-key.json").read_text()) + assert cached["token"] == "sk-fresh" + + +def test_get_api_key_refreshes_when_cache_missing(auth, tmp_path, monkeypatch): + _seed_access_token(tmp_path) + _fake_refresh(monkeypatch, FRESH_KEY) + + assert auth.get_api_key() == "sk-fresh" + + +def test_get_api_key_none_on_refresh_http_error(auth, tmp_path, monkeypatch): + _seed_expired_key(tmp_path) + _seed_access_token(tmp_path) + monkeypatch.setattr(copilot.httpx, "get", lambda *a, **k: _FakeResponse({}, ok=False)) + + assert auth.get_api_key() is None + + +def test_get_api_key_none_when_response_missing_token(auth, tmp_path, monkeypatch): + _seed_access_token(tmp_path) + _fake_refresh(monkeypatch, {"foo": "bar"}) + + assert auth.get_api_key() is None + + +def test_get_api_key_refreshes_when_within_leeway(auth, tmp_path, monkeypatch): + _seed_api_key(tmp_path, _key_payload(expires_at=time.time() + 60)) + _seed_access_token(tmp_path) + _fake_refresh(monkeypatch, FRESH_KEY) + + assert auth.get_api_key() == "sk-fresh" + + cached = json.loads((tmp_path / "api-key.json").read_text()) + assert cached["token"] == "sk-fresh" + assert cached["refreshed_at"] > 0 + + +def test_get_api_key_does_not_refresh_outside_leeway(auth, tmp_path, monkeypatch): + _seed_api_key(tmp_path, _key_payload(expires_at=time.time() + 600)) + _no_network(monkeypatch) + + assert auth.get_api_key() == "sk-cached" + + +def test_get_api_key_rotates_long_lived_key_when_due(auth, tmp_path, monkeypatch): + refreshed_at = time.time() - 4 * 3600 + _seed_api_key( + tmp_path, _key_payload(expires_at=refreshed_at + 25 * 3600, refreshed_at=refreshed_at) + ) + _seed_access_token(tmp_path) + _fake_refresh(monkeypatch, FRESH_KEY) + + assert auth.get_api_key() == "sk-fresh" + + +def test_get_api_key_keeps_long_lived_key_before_interval(auth, tmp_path, monkeypatch): + refreshed_at = time.time() - 1 * 3600 + _seed_api_key( + tmp_path, _key_payload(expires_at=refreshed_at + 25 * 3600, refreshed_at=refreshed_at) + ) + _no_network(monkeypatch) + + assert auth.get_api_key() == "sk-cached" + + +def test_get_api_key_short_lived_key_not_rotated_periodically(auth, tmp_path, monkeypatch): + refreshed_at = time.time() - 10 * 3600 + _seed_api_key( + tmp_path, _key_payload(expires_at=refreshed_at + 12 * 3600, refreshed_at=refreshed_at) + ) + _no_network(monkeypatch) + + assert auth.get_api_key() == "sk-cached" + + +def test_get_api_key_refresh_failure_keeps_valid_cached_key(auth, tmp_path, monkeypatch): + refreshed_at = time.time() - 4 * 3600 + _seed_api_key( + tmp_path, _key_payload(expires_at=refreshed_at + 25 * 3600, refreshed_at=refreshed_at) + ) + _seed_access_token(tmp_path) + monkeypatch.setattr(copilot.httpx, "get", lambda *a, **k: _FakeResponse({}, ok=False)) + + assert auth.get_api_key() == "sk-cached" + + +# --------------------------------------------------------------------------- +# Device flow (no access-token cached) +# --------------------------------------------------------------------------- + + +def test_device_flow_login_writes_access_token(auth, tmp_path, monkeypatch, capsys): + _fake_device_flow(monkeypatch, [{"access_token": "tok-device"}]) + + assert auth.get_access_token() == "tok-device" + assert (tmp_path / "access-token").read_text() == "tok-device" + + out = capsys.readouterr().out + assert DEVICE_INFO["verification_uri"] in out + assert DEVICE_INFO["user_code"] in out + + +def test_device_flow_times_out_after_12_polls(auth, monkeypatch): + calls = _fake_device_flow(monkeypatch, [{}] * 12) + + with pytest.raises(RuntimeError, match="Timed out"): + auth.get_access_token() + + poll_count = sum(1 for url in calls if url == COPILOT_ACCESS_TOKEN_URL) + assert poll_count == 12 + + +# --------------------------------------------------------------------------- +# copilot_headers() +# --------------------------------------------------------------------------- + + +def test_copilot_headers_messages_proxy(): + headers = copilot_headers("sk-test", messages_proxy=True) + + assert headers["Authorization"] == "Bearer sk-test" + assert headers["openai-intent"] == "messages-proxy" + assert headers["x-interaction-type"] == "messages-proxy" + assert headers["x-github-api-version"] == "2026-06-01" + assert headers["anthropic-version"] == "2023-06-01" + assert headers["x-request-id"] + + +def test_copilot_headers_conversation_panel(): + headers = copilot_headers("sk-test") + + assert headers["openai-intent"] == "conversation-panel" + assert headers["x-github-api-version"] == "2025-04-01" + assert "anthropic-version" not in headers + + +# --------------------------------------------------------------------------- +# Wiring through config resolution / get_api_key / the provider adapter +# --------------------------------------------------------------------------- + + +def test_resolve_model_config_uses_session_endpoint(auth, tmp_path): + _seed_api_key(tmp_path) + + resolved = llms_config.resolve_model_config("github_copilot/gpt-5") + + assert resolved["provider"] == "github_copilot" + assert resolved["api_base"] == "https://tenant.githubcopilot.com" + assert resolved["family"] == "responses" + + +def test_resolve_model_config_claude_family_messages(auth, tmp_path): + _seed_api_key(tmp_path) + + # github_copilot/claude-sonnet-4.5 is the bundled copilot record; the + # bare claude-sonnet-5 key resolves to the anthropic provider. + resolved = llms_config.resolve_model_config("github_copilot/claude-sonnet-4.5") + + assert resolved["provider"] == "github_copilot" + assert resolved["family"] == "messages" + + +def test_get_api_key_wiring_uses_cached_token(auth, tmp_path): + _seed_api_key(tmp_path) + resolved = llms_config.resolve_model_config("github_copilot/gpt-5") + + assert llms_config.get_api_key(resolved, None) == "sk-cached" + + +def test_get_api_key_wiring_explicit_key_wins(auth, tmp_path): + _seed_api_key(tmp_path) + resolved = llms_config.resolve_model_config("github_copilot/gpt-5") + + assert llms_config.get_api_key(resolved, "explicit-key") == "explicit-key" + + +def test_provider_adapter_resolves_from_session(auth, tmp_path): + _seed_api_key(tmp_path) + provider = GithubCopilotProvider() + resolved = llms_config.resolve_model_config("github_copilot/gpt-5") + + assert provider.resolve_api_base(resolved) == "https://tenant.githubcopilot.com" + assert provider.resolve_api_key(resolved, None) == "sk-cached" + # Token-leak prevention: the copilot adapter ignores caller-supplied keys + # and always serves the authenticated session key. + assert provider.resolve_api_key(resolved, "explicit") == "sk-cached" + + +def test_provider_adapter_build_headers_messages_family(): + provider = GithubCopilotProvider() + resolved = {"family": "messages"} + + headers = provider.build_headers(resolved, "sk-test", "messages", {}) + + assert headers["Authorization"] == "Bearer sk-test" + assert headers["openai-intent"] == "messages-proxy" + assert headers["Content-Type"] == "application/json" + + +def test_provider_adapter_build_headers_conversation_family(): + provider = GithubCopilotProvider() + resolved = {"family": "responses"} + + headers = provider.build_headers(resolved, "sk-test", "responses", {}) + + assert headers["openai-intent"] == "conversation-panel" + + +def test_pipeline_uses_copilot_key_and_headers(auth, tmp_path, monkeypatch): + """End to end: cached key flows from config -> adapter -> family adapter.""" + _seed_api_key(tmp_path) + captured = {} + + async def fake_responses_complete(resolved, messages, tools, key, headers, kwargs): + captured["key"] = key + captured["headers"] = headers + return None + + monkeypatch.setattr(pipeline, "responses_complete", fake_responses_complete) + asyncio.run(pipeline.acompletion(model="github_copilot/gpt-5", messages=MSGS)) + + assert captured["key"] == "sk-cached" + assert captured["headers"]["Authorization"] == "Bearer sk-cached" + assert captured["headers"]["openai-intent"] == "conversation-panel" From 1876ec70030f350d227ea24926328ac40c8914b4 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 11 Aug 2026 01:44:41 -0400 Subject: [PATCH 18/30] Add all OpenAI compatible providers from model-metadata.json --- cecli/resources/providers.json | 360 +++++++++++++++++++++- tests/helpers/test_llms_providers_json.py | 8 +- 2 files changed, 361 insertions(+), 7 deletions(-) diff --git a/cecli/resources/providers.json b/cecli/resources/providers.json index 45b717c8cb7..7faca9d876c 100644 --- a/cecli/resources/providers.json +++ b/cecli/resources/providers.json @@ -1,4 +1,18 @@ { + "ai21": { + "api_base": "https://api.ai21.com/studio/v1", + "api_key_env": [ + "AI21_API_KEY" + ], + "display_name": "ai21" + }, + "anyscale": { + "api_base": "https://api.endpoints.anyscale.com/v1", + "api_key_env": [ + "ANYSCALE_API_KEY" + ], + "display_name": "anyscale" + }, "apertis": { "api_base": "https://api.stima.tech/v1", "api_key_env": [ @@ -6,6 +20,34 @@ ], "display_name": "apertis" }, + "azure_ai": { + "api_base": "https://{resource}.services.ai.azure.com", + "api_key_env": [ + "AZURE_AI_API_KEY" + ], + "display_name": "azure_ai" + }, + "baseten": { + "api_base": "https://inference.baseten.co/v1", + "api_key_env": [ + "BASETEN_API_KEY" + ], + "display_name": "baseten" + }, + "cerebras": { + "api_base": "https://api.cerebras.ai/v1", + "api_key_env": [ + "CEREBRAS_API_KEY" + ], + "display_name": "cerebras" + }, + "chatgpt": { + "api_base": "https://chatgpt.com/backend-api/codex", + "api_key_env": [ + "CHATGPT_API_KEY" + ], + "display_name": "chatgpt" + }, "chutes": { "api_base": "https://llm.chutes.ai/v1/", "api_key_env": [ @@ -13,6 +55,103 @@ ], "display_name": "chutes" }, + "cloudflare": { + "account_id_env": "CLOUDFLARE_ACCOUNT_ID", + "api_base": "https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/v1", + "api_key_env": [ + "CLOUDFLARE_API_KEY" + ], + "display_name": "cloudflare", + "models_url": "https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/v1/models" + }, + "codestral": { + "api_base": "https://codestral.mistral.ai/v1", + "api_key_env": [ + "CODESTRAL_API_KEY" + ], + "display_name": "codestral" + }, + "crusoe": { + "api_base": "https://managed-inference-api-proxy.crusoecloud.com/v1", + "api_key_env": [ + "CRUSOE_API_KEY" + ], + "base_url_env": [ + "CRUSOE_API_BASE" + ], + "display_name": "crusoe" + }, + "darkbloom": { + "api_base": "https://api.darkbloom.dev/v1", + "api_key_env": [ + "DARKBLOOM_API_KEY" + ], + "base_url_env": [ + "DARKBLOOM_API_BASE" + ], + "display_name": "darkbloom" + }, + "dashscope": { + "api_base": "https://dashscope.aliyuncs.com/compatible-mode/v1", + "api_key_env": [ + "DASHSCOPE_API_KEY" + ], + "display_name": "dashscope" + }, + "databricks": { + "api_base": "https://{workspace}.cloud.databricks.com", + "api_key_env": [ + "DATABRICKS_API_KEY" + ], + "base_url_env": [ + "DATABRICKS_API_BASE" + ], + "display_name": "databricks" + }, + "deepinfra": { + "api_base": "https://api.deepinfra.com/v1/openai", + "api_key_env": [ + "DEEPINFRA_API_KEY" + ], + "display_name": "deepinfra" + }, + "featherless_ai": { + "api_base": "https://api.featherless.ai/v1", + "api_key_env": [ + "FEATHERLESS_AI_API_KEY" + ], + "display_name": "featherless_ai" + }, + "fireworks_ai": { + "account_id_env": "FIREWORKS_AI_ACCOUNT_ID", + "api_base": "https://api.fireworks.ai/inference/v1", + "api_key_env": [ + "FIREWORKS_AI_API_KEY" + ], + "display_name": "fireworks_ai", + "models_url": "https://api.fireworks.ai/v1/accounts/{account_id}/models" + }, + "friendliai": { + "api_base": "https://api.friendli.ai/serverless/v1", + "api_key_env": [ + "FRIENDLIAI_API_KEY" + ], + "display_name": "friendliai" + }, + "gmi": { + "api_base": "https://api.gmi-serving.com/v1", + "api_key_env": [ + "GMI_API_KEY" + ], + "display_name": "gmi" + }, + "groq": { + "api_base": "https://api.groq.com/openai/v1", + "api_key_env": [ + "GROQ_API_KEY" + ], + "display_name": "groq" + }, "helicone": { "api_base": "https://ai-gateway.helicone.ai/", "api_key_env": [ @@ -20,6 +159,82 @@ ], "display_name": "helicone" }, + "hyperbolic": { + "api_base": "https://api.hyperbolic.xyz/v1", + "api_key_env": [ + "HYPERBOLIC_API_KEY" + ], + "display_name": "hyperbolic" + }, + "inception": { + "api_base": "https://api.inceptionlabs.ai/v1", + "api_key_env": [ + "INCEPTION_API_KEY" + ], + "display_name": "inception" + }, + "lambda_ai": { + "api_base": "https://api.lambda.ai/v1", + "api_key_env": [ + "LAMBDA_API_KEY" + ], + "display_name": "lambda_ai" + }, + "lemonade": { + "api_base": "http://localhost:8000/api/v1", + "api_key_env": [ + "LEMONADE_API_KEY" + ], + "base_url_env": [ + "LEMONADE_API_BASE" + ], + "display_name": "lemonade" + }, + "libertai": { + "api_base": "https://api.libertai.io/v1", + "api_key_env": [ + "LIBERTAI_API_KEY" + ], + "base_url_env": [ + "LIBERTAI_API_BASE" + ], + "display_name": "libertai" + }, + "llamagate": { + "api_base": "https://api.llamagate.dev/v1", + "api_key_env": [ + "LLAMAGATE_API_KEY" + ], + "display_name": "llamagate" + }, + "meta_llama": { + "api_base": "https://api.llama.com/compat/v1", + "api_key_env": [ + "LLAMA_API_KEY" + ], + "display_name": "meta_llama" + }, + "mistral": { + "api_base": "https://api.mistral.ai/v1", + "api_key_env": [ + "MISTRAL_API_KEY" + ], + "display_name": "mistral" + }, + "moonshot": { + "api_base": "https://api.moonshot.ai/v1", + "api_key_env": [ + "MOONSHOT_API_KEY" + ], + "display_name": "moonshot" + }, + "morph": { + "api_base": "https://api.morphllm.com/v1", + "api_key_env": [ + "MORPH_API_KEY" + ], + "display_name": "morph" + }, "nano-gpt": { "api_base": "https://nano-gpt.com/api/v1", "api_key_env": [ @@ -27,6 +242,61 @@ ], "display_name": "nano-gpt" }, + "nebius": { + "api_base": "https://api.studio.nebius.ai/v1", + "api_key_env": [ + "NEBIUS_API_KEY" + ], + "display_name": "nebius" + }, + "novita": { + "api_base": "https://api.novita.ai/v3/openai", + "api_key_env": [ + "NOVITA_API_KEY" + ], + "display_name": "novita" + }, + "nscale": { + "api_base": "https://inference.api.nscale.com/v1", + "api_key_env": [ + "NSCALE_API_KEY" + ], + "display_name": "nscale" + }, + "ollama": { + "api_base": "http://localhost:11434/v1", + "api_key_env": [ + "OLLAMA_API_KEY" + ], + "base_url_env": [ + "OLLAMA_API_BASE" + ], + "display_name": "ollama" + }, + "ovhcloud": { + "api_base": "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1", + "api_key_env": [ + "OVHCLOUD_API_KEY" + ], + "display_name": "ovhcloud" + }, + "perplexity": { + "api_base": "https://api.perplexity.ai", + "api_key_env": [ + "PERPLEXITY_API_KEY" + ], + "display_name": "perplexity" + }, + "pinstripes": { + "api_base": "https://pinstripes.io/v1", + "api_key_env": [ + "PINSTRIPES_API_KEY" + ], + "base_url_env": [ + "PINSTRIPES_API_BASE" + ], + "display_name": "pinstripes" + }, "poe": { "api_base": "https://api.poe.com/v1", "api_key_env": [ @@ -41,6 +311,27 @@ ], "display_name": "publicai" }, + "sambanova": { + "api_base": "https://api.sambanova.ai/v1", + "api_key_env": [ + "SAMBANOVA_API_KEY" + ], + "display_name": "sambanova" + }, + "sarvam": { + "api_base": "https://api.sarvam.ai/v1", + "api_key_env": [ + "SARVAM_API_KEY" + ], + "display_name": "sarvam" + }, + "scaleway": { + "api_base": "https://api.scaleway.ai/v1", + "api_key_env": [ + "SCW_SECRET_KEY" + ], + "display_name": "scaleway" + }, "synthetic": { "api_base": "https://api.synthetic.new/openai/v1", "api_key_env": [ @@ -50,6 +341,37 @@ "hf_namespace": true, "supports_stream": false }, + "tencent": { + "api_base": "https://tokenhub-intl.tencentcloudmaas.com/v1", + "api_key_env": [ + "TENCENT_API_KEY" + ], + "display_name": "tencent" + }, + "tensormesh": { + "api_base": "https://serverless.tensormesh.ai/v1", + "api_key_env": [ + "TENSORMESH_INFERENCE_API_KEY" + ], + "base_url_env": [ + "TENSORMESH_SERVERLESS_BASE_URL" + ], + "display_name": "tensormesh" + }, + "together_ai": { + "api_base": "https://api.together.xyz/v1", + "api_key_env": [ + "TOGETHER_AI_API_KEY" + ], + "display_name": "together_ai" + }, + "v0": { + "api_base": "https://api.v0.dev/v1", + "api_key_env": [ + "V0_API_KEY" + ], + "display_name": "v0" + }, "veniceai": { "api_base": "https://api.venice.ai/api/v1", "api_key_env": [ @@ -57,14 +379,33 @@ ], "display_name": "veniceai" }, - "fireworks_ai": { - "api_base": "https://api.fireworks.ai/inference/v1", + "vercel_ai_gateway": { + "api_base": "https://ai-gateway.vercel.sh/v1", "api_key_env": [ - "FIREWORKS_AI_API_KEY" + "VERCEL_AI_GATEWAY_API_KEY" ], - "display_name": "fireworks_ai", - "models_url": "https://api.fireworks.ai/v1/accounts/{account_id}/models", - "account_id_env": "FIREWORKS_AI_ACCOUNT_ID" + "display_name": "vercel_ai_gateway" + }, + "volcengine": { + "api_base": "https://ark.cn-beijing.volces.com/api/v3", + "api_key_env": [ + "VOLCENGINE_API_KEY" + ], + "display_name": "volcengine" + }, + "wandb": { + "api_base": "https://api.inference.wandb.ai/v1", + "api_key_env": [ + "WANDB_API_KEY" + ], + "display_name": "wandb" + }, + "xai": { + "api_base": "https://api.x.ai/v1", + "api_key_env": [ + "XAI_API_KEY" + ], + "display_name": "xai" }, "xiaomi_mimo": { "api_base": "https://api.xiaomimimo.com/v1", @@ -72,5 +413,12 @@ "XIAOMI_MIMO_API_KEY" ], "display_name": "xiaomi_mimo" + }, + "zai": { + "api_base": "https://api.z.ai/api/paas/v4", + "api_key_env": [ + "ZAI_API_KEY" + ], + "display_name": "zai" } } diff --git a/tests/helpers/test_llms_providers_json.py b/tests/helpers/test_llms_providers_json.py index e7d234ae05f..81b690aad37 100644 --- a/tests/helpers/test_llms_providers_json.py +++ b/tests/helpers/test_llms_providers_json.py @@ -5,6 +5,9 @@ - is loaded by ``ModelProviderManager`` (supports_provider == True) - resolves through ``resolve_model_config`` to the ``chat`` family (the + OpenAI-compatible /chat/completions wire, which is what these providers + advertise); ``chatgpt`` is the exception and uses OpenAI's newer + ``responses`` family OpenAI-compatible /chat/completions wire, which is what these providers advertise) - keeps its configured base URL and key env var @@ -46,7 +49,10 @@ def test_every_provider_resolves_to_chat_family_with_base_and_key(): for name, cfg in providers.items(): resolved = resolve_model_config(f"{name}/sample-model") - assert resolved["family"] == "chat", f"{name} should use chat completions" + if name == "chatgpt": # ChatGPT subscription routes via /v1/responses + assert resolved["family"] == "responses", f"{name} should use responses" + else: + assert resolved["family"] == "chat", f"{name} should use chat completions" assert resolved["provider"] == name assert resolved["api_base"] == cfg["api_base"].rstrip("/") assert resolved["api_key_env"] in cfg["api_key_env"] From 9a9f83727026581348d0b69d820b0abc1313f7d4 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 11 Aug 2026 01:47:22 -0400 Subject: [PATCH 19/30] Update model metadata json file --- cecli/resources/model-metadata.json | 499 ++++++++++++++++++++++++---- 1 file changed, 427 insertions(+), 72 deletions(-) diff --git a/cecli/resources/model-metadata.json b/cecli/resources/model-metadata.json index 38281944f96..c2048a93d1a 100644 --- a/cecli/resources/model-metadata.json +++ b/cecli/resources/model-metadata.json @@ -1468,7 +1468,7 @@ "mode": "chat" }, "azure/eu/gpt-4o-2024-08-06": { - "deprecation_date": "2026-02-27", + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 0.000001375, "input_cost_per_token": 0.00000275, "litellm_provider": "azure", @@ -1485,7 +1485,7 @@ "supports_vision": true }, "azure/eu/gpt-4o-2024-11-20": { - "deprecation_date": "2026-03-01", + "deprecation_date": "2027-04-14", "cache_creation_input_token_cost": 0.00000138, "input_cost_per_token": 0.00000275, "litellm_provider": "azure", @@ -1502,6 +1502,7 @@ }, "azure/eu/gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 8.3e-8, + "deprecation_date": "2027-04-14", "input_cost_per_token": 1.65e-7, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -1518,6 +1519,7 @@ }, "azure/eu/gpt-5-2025-08-07": { "cache_read_input_token_cost": 1.375e-7, + "deprecation_date": "2027-02-09", "input_cost_per_token": 0.000001375, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -1550,6 +1552,7 @@ }, "azure/eu/gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.75e-8, + "deprecation_date": "2027-02-09", "input_cost_per_token": 2.75e-7, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -1582,6 +1585,7 @@ }, "azure/eu/gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5.5e-9, + "deprecation_date": "2027-02-09", "input_cost_per_token": 5.5e-8, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -1648,6 +1652,7 @@ }, "azure/eu/gpt-5.1-chat": { "cache_read_input_token_cost": 1.4e-7, + "deprecation_date": "2026-06-29", "input_cost_per_token": 0.00000138, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -1718,6 +1723,7 @@ "azure/eu/gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.8e-7, "cache_read_input_token_cost_priority": 5.5e-7, + "deprecation_date": "2027-09-02", "input_cost_per_token": 0.00000275, "input_cost_per_token_priority": 0.0000055, "output_cost_per_token": 0.0000165, @@ -1877,6 +1883,7 @@ "cache_read_input_token_cost": 2.2e-8, "cache_read_input_token_cost_above_272k_tokens": 4.4e-8, "cache_read_input_token_cost_priority": 5.5e-8, + "deprecation_date": "2028-01-11", "input_cost_per_token": 2.2e-7, "input_cost_per_token_above_272k_tokens": 4.4e-7, "input_cost_per_token_priority": 5.5e-7, @@ -1919,6 +1926,7 @@ "cache_read_input_token_cost": 5.5e-7, "cache_read_input_token_cost_above_272k_tokens": 0.0000011, "cache_read_input_token_cost_priority": 0.000001375, + "deprecation_date": "2028-01-11", "input_cost_per_token": 0.0000055, "input_cost_per_token_above_272k_tokens": 0.000011, "input_cost_per_token_priority": 0.00001375, @@ -1961,6 +1969,7 @@ "cache_read_input_token_cost": 2.2e-7, "cache_read_input_token_cost_above_272k_tokens": 4.4e-7, "cache_read_input_token_cost_priority": 5.5e-7, + "deprecation_date": "2028-01-11", "input_cost_per_token": 0.0000022, "input_cost_per_token_above_272k_tokens": 0.0000044, "input_cost_per_token_priority": 0.0000055, @@ -2001,6 +2010,7 @@ }, "azure/eu/o1-2024-12-17": { "cache_read_input_token_cost": 0.00000825, + "deprecation_date": "2026-10-21", "input_cost_per_token": 0.0000165, "litellm_provider": "azure", "max_input_tokens": 200000, @@ -2046,6 +2056,7 @@ }, "azure/eu/o3-mini-2025-01-31": { "cache_read_input_token_cost": 6.05e-7, + "deprecation_date": "2026-10-01", "input_cost_per_token": 0.00000121, "input_cost_per_token_batches": 6.05e-7, "litellm_provider": "azure", @@ -2062,7 +2073,7 @@ }, "azure/global-standard/gpt-4o-2024-08-06": { "cache_read_input_token_cost": 0.00000125, - "deprecation_date": "2026-02-27", + "deprecation_date": "2027-04-14", "input_cost_per_token": 0.0000025, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -2079,7 +2090,7 @@ }, "azure/global-standard/gpt-4o-2024-11-20": { "cache_read_input_token_cost": 0.00000125, - "deprecation_date": "2026-03-01", + "deprecation_date": "2027-04-14", "input_cost_per_token": 0.0000025, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -2108,7 +2119,7 @@ "supports_vision": true }, "azure/global/gpt-4o-2024-08-06": { - "deprecation_date": "2026-02-27", + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 0.00000125, "input_cost_per_token": 0.0000025, "litellm_provider": "azure", @@ -2125,7 +2136,7 @@ "supports_vision": true }, "azure/global/gpt-4o-2024-11-20": { - "deprecation_date": "2026-03-01", + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 0.00000125, "input_cost_per_token": 0.0000025, "litellm_provider": "azure", @@ -2177,6 +2188,7 @@ }, "azure/global/gpt-5.1-chat": { "cache_read_input_token_cost": 1.25e-7, + "deprecation_date": "2026-06-29", "input_cost_per_token": 0.00000125, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -2427,7 +2439,7 @@ "supports_web_search": false }, "azure/gpt-4.1-2025-04-14": { - "deprecation_date": "2026-11-04", + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 5e-7, "input_cost_per_token": 0.000002, "input_cost_per_token_batches": 0.000001, @@ -2494,7 +2506,7 @@ "supports_web_search": false }, "azure/gpt-4.1-mini-2025-04-14": { - "deprecation_date": "2026-11-04", + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1e-7, "input_cost_per_token": 4e-7, "input_cost_per_token_batches": 2e-7, @@ -2560,7 +2572,7 @@ "supports_vision": true }, "azure/gpt-4.1-nano-2025-04-14": { - "deprecation_date": "2026-11-04", + "deprecation_date": "2026-10-14", "cache_read_input_token_cost": 2.5e-8, "input_cost_per_token": 1e-7, "input_cost_per_token_batches": 5e-8, @@ -2628,6 +2640,7 @@ "supports_vision": true }, "azure/gpt-4o-2024-05-13": { + "deprecation_date": "2026-10-01", "input_cost_per_token": 0.000005, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -2642,7 +2655,7 @@ "supports_vision": true }, "azure/gpt-4o-2024-08-06": { - "deprecation_date": "2026-02-27", + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 0.00000125, "input_cost_per_token": 0.0000025, "litellm_provider": "azure", @@ -2659,7 +2672,7 @@ "supports_vision": true }, "azure/gpt-4o-2024-11-20": { - "deprecation_date": "2026-03-01", + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 0.00000125, "input_cost_per_token": 0.00000275, "litellm_provider": "azure", @@ -2724,6 +2737,7 @@ }, "azure/gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 7.5e-8, + "deprecation_date": "2027-04-14", "input_cost_per_token": 1.65e-7, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -2803,6 +2817,7 @@ }, "azure/gpt-5-2025-08-07": { "cache_read_input_token_cost": 1.25e-7, + "deprecation_date": "2027-02-09", "input_cost_per_token": 0.00000125, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -2835,6 +2850,7 @@ }, "azure/gpt-5-chat": { "cache_read_input_token_cost": 1.25e-7, + "deprecation_date": "2026-06-29", "input_cost_per_token": 0.00000125, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -2868,6 +2884,7 @@ }, "azure/gpt-5-chat-latest": { "cache_read_input_token_cost": 1.25e-7, + "deprecation_date": "2026-06-29", "input_cost_per_token": 0.00000125, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -2932,6 +2949,7 @@ }, "azure/gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.5e-8, + "deprecation_date": "2027-02-09", "input_cost_per_token": 2.5e-7, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -2996,6 +3014,7 @@ }, "azure/gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5e-9, + "deprecation_date": "2027-02-09", "input_cost_per_token": 5e-8, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -3063,6 +3082,7 @@ "azure/gpt-5.1-2025-11-13": { "cache_read_input_token_cost": 1.25e-7, "cache_read_input_token_cost_priority": 2.5e-7, + "deprecation_date": "2027-05-15", "input_cost_per_token": 0.00000125, "input_cost_per_token_priority": 0.0000025, "litellm_provider": "azure", @@ -3099,6 +3119,7 @@ }, "azure/gpt-5.1-chat": { "cache_read_input_token_cost": 1.25e-7, + "deprecation_date": "2026-06-29", "input_cost_per_token": 0.00000125, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -3134,6 +3155,7 @@ "azure/gpt-5.1-chat-2025-11-13": { "cache_read_input_token_cost": 1.25e-7, "cache_read_input_token_cost_priority": 2.5e-7, + "deprecation_date": "2026-06-29", "input_cost_per_token": 0.00000125, "input_cost_per_token_priority": 0.0000025, "litellm_provider": "azure", @@ -3203,6 +3225,7 @@ "azure/gpt-5.2-2025-12-11": { "cache_read_input_token_cost": 1.75e-7, "cache_read_input_token_cost_priority": 3.5e-7, + "deprecation_date": "2027-06-08", "input_cost_per_token": 0.00000175, "input_cost_per_token_priority": 0.0000035, "litellm_provider": "azure", @@ -3239,6 +3262,7 @@ "azure/gpt-5.2-chat": { "cache_read_input_token_cost": 1.75e-7, "cache_read_input_token_cost_priority": 3.5e-7, + "deprecation_date": "2026-06-29", "input_cost_per_token": 0.00000175, "input_cost_per_token_priority": 0.0000035, "litellm_provider": "azure", @@ -3273,6 +3297,7 @@ "azure/gpt-5.2-chat-2025-12-11": { "cache_read_input_token_cost": 1.75e-7, "cache_read_input_token_cost_priority": 3.5e-7, + "deprecation_date": "2026-05-13", "input_cost_per_token": 0.00000175, "input_cost_per_token_priority": 0.0000035, "litellm_provider": "azure", @@ -3307,6 +3332,7 @@ "azure/gpt-5.3-chat": { "cache_read_input_token_cost": 1.75e-7, "cache_read_input_token_cost_priority": 3.5e-7, + "deprecation_date": "2026-06-29", "input_cost_per_token": 0.00000175, "input_cost_per_token_priority": 0.0000035, "litellm_provider": "azure", @@ -3384,6 +3410,7 @@ "cache_read_input_token_cost_above_272k_tokens": 5e-7, "cache_read_input_token_cost_priority": 5e-7, "cache_read_input_token_cost_above_272k_tokens_priority": 0.000001, + "deprecation_date": "2027-09-02", "input_cost_per_token": 0.0000025, "input_cost_per_token_above_272k_tokens": 0.000005, "input_cost_per_token_priority": 0.000005, @@ -3457,6 +3484,7 @@ }, "azure/gpt-5.4-mini-2026-03-17": { "cache_read_input_token_cost": 7.5e-8, + "deprecation_date": "2027-09-21", "input_cost_per_token": 7.5e-7, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -3527,6 +3555,7 @@ }, "azure/gpt-5.4-nano-2026-03-17": { "cache_read_input_token_cost": 2e-8, + "deprecation_date": "2027-09-21", "input_cost_per_token": 2e-7, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -3697,6 +3726,7 @@ "cache_read_input_token_cost_above_272k_tokens": 4e-8, "cache_read_input_token_cost_priority": 4e-8, "cache_read_input_token_cost_above_272k_tokens_priority": 8e-8, + "deprecation_date": "2028-01-11", "input_cost_per_token": 2e-7, "input_cost_per_token_above_272k_tokens": 4e-7, "input_cost_per_token_priority": 4e-7, @@ -3742,6 +3772,7 @@ "cache_read_input_token_cost_above_272k_tokens": 0.000001, "cache_read_input_token_cost_priority": 0.000001, "cache_read_input_token_cost_above_272k_tokens_priority": 0.000002, + "deprecation_date": "2028-01-11", "input_cost_per_token": 0.000005, "input_cost_per_token_above_272k_tokens": 0.00001, "input_cost_per_token_priority": 0.00001, @@ -3787,6 +3818,7 @@ "cache_read_input_token_cost_above_272k_tokens": 4e-7, "cache_read_input_token_cost_priority": 4e-7, "cache_read_input_token_cost_above_272k_tokens_priority": 8e-7, + "deprecation_date": "2028-01-11", "input_cost_per_token": 0.000002, "input_cost_per_token_above_272k_tokens": 0.000004, "input_cost_per_token_priority": 0.000004, @@ -3828,6 +3860,7 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-audio-1.5-2026-02-23": { + "deprecation_date": "2027-08-24", "input_cost_per_audio_token": 0.00004, "input_cost_per_token": 0.0000025, "litellm_provider": "azure", @@ -3859,6 +3892,7 @@ "supports_vision": false }, "azure/gpt-audio-2025-08-28": { + "deprecation_date": "2027-03-02", "input_cost_per_audio_token": 0.00004, "input_cost_per_token": 0.0000025, "litellm_provider": "azure", @@ -3890,6 +3924,7 @@ "supports_vision": false }, "azure/gpt-audio-mini-2025-10-06": { + "deprecation_date": "2027-04-06", "input_cost_per_audio_token": 0.00001, "input_cost_per_token": 6e-7, "litellm_provider": "azure", @@ -3956,6 +3991,7 @@ }, "azure/o1-2024-12-17": { "cache_read_input_token_cost": 0.0000075, + "deprecation_date": "2026-10-21", "input_cost_per_token": 0.000015, "litellm_provider": "azure", "max_input_tokens": 200000, @@ -4061,7 +4097,7 @@ "supports_vision": true }, "azure/o3-2025-04-16": { - "deprecation_date": "2026-04-16", + "deprecation_date": "2026-10-21", "cache_read_input_token_cost": 5e-7, "input_cost_per_token": 0.000002, "litellm_provider": "azure", @@ -4092,6 +4128,7 @@ }, "azure/o3-deep-research": { "cache_read_input_token_cost": 0.0000025, + "deprecation_date": "2026-12-26", "input_cost_per_token": 0.00001, "litellm_provider": "azure", "max_input_tokens": 200000, @@ -4139,6 +4176,7 @@ }, "azure/o3-mini-2025-01-31": { "cache_read_input_token_cost": 5.5e-7, + "deprecation_date": "2026-10-01", "input_cost_per_token": 0.0000011, "litellm_provider": "azure", "max_input_tokens": 200000, @@ -4182,6 +4220,7 @@ "supports_vision": true }, "azure/o3-pro-2025-06-10": { + "deprecation_date": "2026-12-17", "input_cost_per_token": 0.00002, "input_cost_per_token_batches": 0.00001, "litellm_provider": "azure", @@ -4242,6 +4281,7 @@ }, "azure/o4-mini-2025-04-16": { "cache_read_input_token_cost": 2.75e-7, + "deprecation_date": "2026-10-16", "input_cost_per_token": 0.0000011, "litellm_provider": "azure", "max_input_tokens": 200000, @@ -4258,7 +4298,7 @@ "supports_vision": true }, "azure/us/gpt-4.1-2025-04-14": { - "deprecation_date": "2026-11-04", + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 5.5e-7, "input_cost_per_token": 0.0000022, "input_cost_per_token_batches": 0.0000011, @@ -4292,7 +4332,7 @@ "supports_web_search": false }, "azure/us/gpt-4.1-mini-2025-04-14": { - "deprecation_date": "2026-11-04", + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1.1e-7, "input_cost_per_token": 4.4e-7, "input_cost_per_token_batches": 2.2e-7, @@ -4326,7 +4366,7 @@ "supports_web_search": false }, "azure/us/gpt-4.1-nano-2025-04-14": { - "deprecation_date": "2026-11-04", + "deprecation_date": "2026-10-14", "cache_read_input_token_cost": 2.5e-8, "input_cost_per_token": 1.1e-7, "input_cost_per_token_batches": 6e-8, @@ -4359,7 +4399,7 @@ "supports_vision": true }, "azure/us/gpt-4o-2024-08-06": { - "deprecation_date": "2026-02-27", + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 0.000001375, "input_cost_per_token": 0.00000275, "litellm_provider": "azure", @@ -4376,7 +4416,7 @@ "supports_vision": true }, "azure/us/gpt-4o-2024-11-20": { - "deprecation_date": "2026-03-01", + "deprecation_date": "2027-04-14", "cache_creation_input_token_cost": 0.00000138, "input_cost_per_token": 0.00000275, "litellm_provider": "azure", @@ -4393,6 +4433,7 @@ }, "azure/us/gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 8.3e-8, + "deprecation_date": "2027-04-14", "input_cost_per_token": 1.65e-7, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -4409,6 +4450,7 @@ }, "azure/us/gpt-5-2025-08-07": { "cache_read_input_token_cost": 1.375e-7, + "deprecation_date": "2027-02-09", "input_cost_per_token": 0.000001375, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -4441,6 +4483,7 @@ }, "azure/us/gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.75e-8, + "deprecation_date": "2027-02-09", "input_cost_per_token": 2.75e-7, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -4473,6 +4516,7 @@ }, "azure/us/gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5.5e-9, + "deprecation_date": "2027-02-09", "input_cost_per_token": 5.5e-8, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -4539,6 +4583,7 @@ }, "azure/us/gpt-5.1-chat": { "cache_read_input_token_cost": 1.4e-7, + "deprecation_date": "2026-06-29", "input_cost_per_token": 0.00000138, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -4609,6 +4654,7 @@ "azure/us/gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.8e-7, "cache_read_input_token_cost_priority": 5.5e-7, + "deprecation_date": "2027-09-02", "input_cost_per_token": 0.00000275, "input_cost_per_token_priority": 0.0000055, "output_cost_per_token": 0.0000165, @@ -4768,6 +4814,7 @@ "cache_read_input_token_cost": 2.2e-8, "cache_read_input_token_cost_above_272k_tokens": 4.4e-8, "cache_read_input_token_cost_priority": 5.5e-8, + "deprecation_date": "2028-01-11", "input_cost_per_token": 2.2e-7, "input_cost_per_token_above_272k_tokens": 4.4e-7, "input_cost_per_token_priority": 5.5e-7, @@ -4810,6 +4857,7 @@ "cache_read_input_token_cost": 5.5e-7, "cache_read_input_token_cost_above_272k_tokens": 0.0000011, "cache_read_input_token_cost_priority": 0.000001375, + "deprecation_date": "2028-01-11", "input_cost_per_token": 0.0000055, "input_cost_per_token_above_272k_tokens": 0.000011, "input_cost_per_token_priority": 0.00001375, @@ -4852,6 +4900,7 @@ "cache_read_input_token_cost": 2.2e-7, "cache_read_input_token_cost_above_272k_tokens": 4.4e-7, "cache_read_input_token_cost_priority": 5.5e-7, + "deprecation_date": "2028-01-11", "input_cost_per_token": 0.0000022, "input_cost_per_token_above_272k_tokens": 0.0000044, "input_cost_per_token_priority": 0.0000055, @@ -4892,6 +4941,7 @@ }, "azure/us/o1-2024-12-17": { "cache_read_input_token_cost": 0.00000825, + "deprecation_date": "2026-10-21", "input_cost_per_token": 0.0000165, "litellm_provider": "azure", "max_input_tokens": 200000, @@ -4936,7 +4986,7 @@ "supports_vision": false }, "azure/us/o3-2025-04-16": { - "deprecation_date": "2026-04-16", + "deprecation_date": "2026-10-21", "cache_read_input_token_cost": 5.5e-7, "input_cost_per_token": 0.0000022, "litellm_provider": "azure", @@ -4967,6 +5017,7 @@ }, "azure/us/o3-mini-2025-01-31": { "cache_read_input_token_cost": 6.05e-7, + "deprecation_date": "2026-10-01", "input_cost_per_token": 0.00000121, "input_cost_per_token_batches": 6.05e-7, "litellm_provider": "azure", @@ -4983,6 +5034,7 @@ }, "azure/us/o4-mini-2025-04-16": { "cache_read_input_token_cost": 3.1e-7, + "deprecation_date": "2026-10-16", "input_cost_per_token": 0.00000121, "litellm_provider": "azure", "max_input_tokens": 200000, @@ -8474,6 +8526,7 @@ "output_cost_per_token": 5e-7 }, "chatgpt-4o-latest": { + "deprecation_date": "2026-02-17", "input_cost_per_token": 0.000005, "litellm_provider": "openai", "max_input_tokens": 128000, @@ -8581,6 +8634,7 @@ "cache_creation_input_token_cost": 3e-7, "cache_creation_input_token_cost_above_1hr": 0.000006, "cache_read_input_token_cost": 3e-8, + "deprecation_date": "2026-04-20", "input_cost_per_token": 2.5e-7, "litellm_provider": "anthropic", "max_input_tokens": 200000, @@ -8599,7 +8653,7 @@ "cache_creation_input_token_cost": 0.00001875, "cache_creation_input_token_cost_above_1hr": 0.000006, "cache_read_input_token_cost": 0.0000015, - "deprecation_date": "2026-05-01", + "deprecation_date": "2026-01-05", "input_cost_per_token": 0.000015, "litellm_provider": "anthropic", "max_input_tokens": 200000, @@ -8617,6 +8671,7 @@ "claude-4-opus-20250514": { "cache_creation_input_token_cost": 0.00001875, "cache_read_input_token_cost": 0.0000015, + "deprecation_date": "2026-06-15", "input_cost_per_token": 0.000015, "litellm_provider": "anthropic", "max_input_tokens": 200000, @@ -8645,6 +8700,7 @@ "cache_creation_input_token_cost_above_200k_tokens": 0.0000075, "cache_read_input_token_cost": 3e-7, "cache_read_input_token_cost_above_200k_tokens": 6e-7, + "deprecation_date": "2026-06-15", "input_cost_per_token": 0.000003, "input_cost_per_token_above_200k_tokens": 0.000006, "litellm_provider": "anthropic", @@ -8750,6 +8806,72 @@ "supports_vision": true, "prompt_cache_min_tokens": 4096 }, + "claude-mythos-5": { + "cache_creation_input_token_cost": 0.0000125, + "cache_creation_input_token_cost_above_1hr": 0.00002, + "cache_read_input_token_cost": 0.000001, + "input_cost_per_token": 0.00001, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00005, + "prompt_cache_min_tokens": 512, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "source": "https://docs.claude.com/en/docs/about-claude/models/overview", + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "claude-mythos-preview": { + "cache_creation_input_token_cost": 0.0000125, + "cache_creation_input_token_cost_above_1hr": 0.00002, + "cache_read_input_token_cost": 0.000001, + "input_cost_per_token": 0.00001, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00005, + "prompt_cache_min_tokens": 512, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "source": "https://docs.claude.com/en/docs/about-claude/models/overview", + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, "claude-opus-4-1": { "cache_creation_input_token_cost": 0.00001875, "cache_creation_input_token_cost_above_1hr": 0.00003, @@ -8776,7 +8898,8 @@ "supports_native_structured_output": true, "supports_tool_choice": true, "supports_vision": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "deprecation_date": "2026-08-05" }, "claude-opus-4-1-20250805": { "cache_creation_input_token_cost": 0.00001875, @@ -8812,7 +8935,7 @@ "cache_creation_input_token_cost_above_1hr": 0.00003, "cache_read_input_token_cost": 0.0000015, "input_cost_per_token": 0.000015, - "deprecation_date": "2026-05-14", + "deprecation_date": "2026-06-15", "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 32000, @@ -9118,7 +9241,7 @@ "prompt_cache_min_tokens": 512 }, "claude-sonnet-4-20250514": { - "deprecation_date": "2026-05-14", + "deprecation_date": "2026-06-15", "cache_creation_input_token_cost": 0.00000375, "cache_creation_input_token_cost_above_1hr": 0.000006, "cache_read_input_token_cost": 3e-7, @@ -12044,6 +12167,7 @@ "output_cost_per_token": 0.00000185, "supports_function_calling": true, "supports_reasoning": true, + "supports_native_structured_output": true, "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, @@ -15692,6 +15816,7 @@ "supports_tool_choice": true }, "ft:gpt-3.5-turbo": { + "deprecation_date": "2026-10-23", "input_cost_per_token": 0.000003, "input_cost_per_token_batches": 0.0000015, "litellm_provider": "openai", @@ -15705,6 +15830,7 @@ "supports_tool_choice": true }, "ft:gpt-3.5-turbo-0125": { + "deprecation_date": "2026-10-23", "input_cost_per_token": 0.000003, "litellm_provider": "openai", "max_input_tokens": 16385, @@ -15738,6 +15864,7 @@ "supports_tool_choice": true }, "ft:gpt-4-0613": { + "deprecation_date": "2026-10-23", "input_cost_per_token": 0.00003, "litellm_provider": "openai", "max_input_tokens": 8192, @@ -15788,6 +15915,7 @@ }, "ft:gpt-4.1-nano-2025-04-14": { "cache_read_input_token_cost": 5e-8, + "deprecation_date": "2026-10-23", "input_cost_per_token": 2e-7, "input_cost_per_token_batches": 1e-7, "litellm_provider": "openai", @@ -15862,6 +15990,7 @@ }, "ft:o4-mini-2025-04-16": { "cache_read_input_token_cost": 0.000001, + "deprecation_date": "2026-10-23", "input_cost_per_token": 0.000004, "input_cost_per_token_batches": 0.000002, "litellm_provider": "openai", @@ -17661,6 +17790,7 @@ }, "gemini/gemini-2.5-flash-image": { "cache_read_input_token_cost": 3e-8, + "deprecation_date": "2026-10-02", "input_cost_per_audio_token": 0.000001, "input_cost_per_token": 3e-7, "litellm_provider": "gemini", @@ -17806,6 +17936,7 @@ }, "gemini/gemini-2.5-flash-lite-preview-09-2025": { "cache_read_input_token_cost": 1e-8, + "deprecation_date": "2026-03-31", "input_cost_per_audio_token": 3e-7, "input_cost_per_token": 1e-7, "litellm_provider": "gemini", @@ -18160,6 +18291,7 @@ "supports_reasoning": false }, "gemini/gemini-3-pro-image-preview": { + "deprecation_date": "2026-06-25", "input_cost_per_image": 0.0011, "input_cost_per_token": 0.000002, "input_cost_per_token_batches": 0.000001, @@ -18301,6 +18433,7 @@ "web_search_billing_unit": "per_query" }, "gemini/gemini-3.1-flash-image-preview": { + "deprecation_date": "2026-06-25", "input_cost_per_token": 2.5e-7, "input_cost_per_token_batches": 1.25e-7, "litellm_provider": "gemini", @@ -18346,6 +18479,7 @@ "cache_read_input_token_cost": 2.5e-8, "cache_read_input_token_cost_flex": 1.25e-8, "cache_read_input_token_cost_priority": 4.5e-8, + "deprecation_date": "2027-05-07", "input_cost_per_audio_token": 5e-7, "input_cost_per_token": 2.5e-7, "input_cost_per_token_batches": 1.25e-7, @@ -18402,6 +18536,7 @@ }, "gemini/gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-8, + "deprecation_date": "2026-05-25", "input_cost_per_audio_token": 5e-7, "input_cost_per_token": 2.5e-7, "litellm_provider": "gemini", @@ -19013,6 +19148,7 @@ }, "gemini/gemini-robotics-er-1.5-preview": { "cache_read_input_token_cost": 0, + "deprecation_date": "2026-04-30", "input_cost_per_token": 3e-7, "input_cost_per_audio_token": 0.000001, "litellm_provider": "gemini", @@ -19148,6 +19284,37 @@ "supports_web_search": true, "web_search_billing_unit": "per_query" }, + "gemini/gemini-robotics-er-2-streaming-preview": { + "input_cost_per_audio_token": 0.000002, + "input_cost_per_token": 0.000002, + "litellm_provider": "gemini", + "mode": "chat", + "output_cost_per_token": 0.00001, + "search_context_cost_per_query": { + "search_context_size_high": 0.014, + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014 + }, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "web_search_billing_unit": "per_query" + }, "gemini/gemma-3-27b-it": { "input_cost_per_audio_per_second": 0, "input_cost_per_audio_per_second_above_128k_tokens": 0, @@ -20203,6 +20370,7 @@ "supports_vision": true }, "gpt-3.5-turbo": { + "deprecation_date": "2026-10-23", "input_cost_per_token": 5e-7, "litellm_provider": "openai", "max_input_tokens": 16385, @@ -20216,6 +20384,7 @@ "supports_tool_choice": true }, "gpt-3.5-turbo-0125": { + "deprecation_date": "2026-10-23", "input_cost_per_token": 5e-7, "litellm_provider": "openai", "max_input_tokens": 16385, @@ -20257,6 +20426,7 @@ "supports_tool_choice": true }, "gpt-4": { + "deprecation_date": "2026-10-23", "input_cost_per_token": 0.00003, "litellm_provider": "openai", "max_input_tokens": 8192, @@ -20297,7 +20467,7 @@ "supports_tool_choice": true }, "gpt-4-0613": { - "deprecation_date": "2025-06-06", + "deprecation_date": "2026-10-23", "input_cost_per_token": 0.00003, "litellm_provider": "openai", "max_input_tokens": 8192, @@ -20311,7 +20481,7 @@ "supports_tool_choice": true }, "gpt-4-1106-preview": { - "deprecation_date": "2026-03-26", + "deprecation_date": "2026-10-23", "input_cost_per_token": 0.00001, "litellm_provider": "openai", "max_input_tokens": 128000, @@ -20326,6 +20496,7 @@ "supports_tool_choice": true }, "gpt-4-turbo": { + "deprecation_date": "2026-10-23", "input_cost_per_token": 0.00001, "litellm_provider": "openai", "max_input_tokens": 128000, @@ -20342,6 +20513,7 @@ "supports_vision": true }, "gpt-4-turbo-2024-04-09": { + "deprecation_date": "2026-10-23", "input_cost_per_token": 0.00001, "litellm_provider": "openai", "max_input_tokens": 128000, @@ -20411,7 +20583,9 @@ }, "gpt-4.1-2025-04-14": { "cache_read_input_token_cost": 5e-7, + "cache_read_input_token_cost_priority": 8.75e-7, "input_cost_per_token": 0.000002, + "input_cost_per_token_priority": 0.0000035, "input_cost_per_token_batches": 0.000001, "litellm_provider": "openai", "max_input_tokens": 1047576, @@ -20419,6 +20593,7 @@ "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 0.000008, + "output_cost_per_token_priority": 0.000014, "output_cost_per_token_batches": 0.000004, "supported_endpoints": [ "/v1/chat/completions", @@ -20482,7 +20657,9 @@ }, "gpt-4.1-mini-2025-04-14": { "cache_read_input_token_cost": 1e-7, + "cache_read_input_token_cost_priority": 1.75e-7, "input_cost_per_token": 4e-7, + "input_cost_per_token_priority": 7e-7, "input_cost_per_token_batches": 2e-7, "litellm_provider": "openai", "max_input_tokens": 1047576, @@ -20490,6 +20667,7 @@ "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 0.0000016, + "output_cost_per_token_priority": 0.0000028, "output_cost_per_token_batches": 8e-7, "supported_endpoints": [ "/v1/chat/completions", @@ -20517,6 +20695,7 @@ "gpt-4.1-nano": { "cache_read_input_token_cost": 2.5e-8, "cache_read_input_token_cost_priority": 5e-8, + "deprecation_date": "2026-10-23", "input_cost_per_token": 1e-7, "input_cost_per_token_batches": 5e-8, "input_cost_per_token_priority": 2e-7, @@ -20552,7 +20731,10 @@ }, "gpt-4.1-nano-2025-04-14": { "cache_read_input_token_cost": 2.5e-8, + "cache_read_input_token_cost_priority": 5e-8, + "deprecation_date": "2026-10-23", "input_cost_per_token": 1e-7, + "input_cost_per_token_priority": 2e-7, "input_cost_per_token_batches": 5e-8, "litellm_provider": "openai", "max_input_tokens": 1047576, @@ -20560,6 +20742,7 @@ "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-7, + "output_cost_per_token_priority": 8e-7, "output_cost_per_token_batches": 2e-7, "supported_endpoints": [ "/v1/chat/completions", @@ -20607,6 +20790,7 @@ "supports_vision": true }, "gpt-4o-2024-05-13": { + "deprecation_date": "2026-10-23", "input_cost_per_token": 0.000005, "input_cost_per_token_batches": 0.0000025, "input_cost_per_token_priority": 0.00000875, @@ -20628,7 +20812,9 @@ }, "gpt-4o-2024-08-06": { "cache_read_input_token_cost": 0.00000125, + "cache_read_input_token_cost_priority": 0.000002125, "input_cost_per_token": 0.0000025, + "input_cost_per_token_priority": 0.00000425, "input_cost_per_token_batches": 0.00000125, "litellm_provider": "openai", "max_input_tokens": 128000, @@ -20636,6 +20822,7 @@ "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 0.00001, + "output_cost_per_token_priority": 0.000017, "output_cost_per_token_batches": 0.000005, "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -20648,7 +20835,9 @@ }, "gpt-4o-2024-11-20": { "cache_read_input_token_cost": 0.00000125, + "cache_read_input_token_cost_priority": 0.000002125, "input_cost_per_token": 0.0000025, + "input_cost_per_token_priority": 0.00000425, "input_cost_per_token_batches": 0.00000125, "litellm_provider": "openai", "max_input_tokens": 128000, @@ -20656,6 +20845,7 @@ "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 0.00001, + "output_cost_per_token_priority": 0.000017, "output_cost_per_token_batches": 0.000005, "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -20667,6 +20857,7 @@ "supports_vision": true }, "gpt-4o-audio-preview": { + "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 0.00004, "input_cost_per_token": 0.0000025, "litellm_provider": "openai", @@ -20684,6 +20875,7 @@ "supports_tool_choice": true }, "gpt-4o-audio-preview-2024-12-17": { + "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 0.00004, "input_cost_per_token": 0.0000025, "litellm_provider": "openai", @@ -20701,6 +20893,7 @@ "supports_tool_choice": true }, "gpt-4o-audio-preview-2025-06-03": { + "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 0.00004, "input_cost_per_token": 0.0000025, "litellm_provider": "openai", @@ -20742,7 +20935,9 @@ }, "gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 7.5e-8, + "cache_read_input_token_cost_priority": 1.25e-7, "input_cost_per_token": 1.5e-7, + "input_cost_per_token_priority": 2.5e-7, "input_cost_per_token_batches": 7.5e-8, "litellm_provider": "openai", "max_input_tokens": 128000, @@ -20750,6 +20945,7 @@ "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-7, + "output_cost_per_token_priority": 0.000001, "output_cost_per_token_batches": 3e-7, "search_context_cost_per_query": { "search_context_size_high": 0.03, @@ -20766,6 +20962,7 @@ "supports_vision": true }, "gpt-4o-mini-audio-preview": { + "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 0.00001, "input_cost_per_token": 1.5e-7, "litellm_provider": "openai", @@ -20783,6 +20980,7 @@ "supports_tool_choice": true }, "gpt-4o-mini-audio-preview-2024-12-17": { + "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 0.00001, "input_cost_per_token": 1.5e-7, "litellm_provider": "openai", @@ -20827,6 +21025,7 @@ }, "gpt-4o-mini-search-preview-2025-03-11": { "cache_read_input_token_cost": 7.5e-8, + "deprecation_date": "2026-07-23", "input_cost_per_token": 1.5e-7, "input_cost_per_token_batches": 7.5e-8, "litellm_provider": "openai", @@ -20873,6 +21072,7 @@ }, "gpt-4o-search-preview-2025-03-11": { "cache_read_input_token_cost": 0.00000125, + "deprecation_date": "2026-07-23", "input_cost_per_token": 0.0000025, "input_cost_per_token_batches": 0.00000125, "litellm_provider": "openai", @@ -20937,6 +21137,7 @@ "cache_read_input_token_cost": 1.25e-7, "cache_read_input_token_cost_flex": 6.25e-8, "cache_read_input_token_cost_priority": 2.5e-7, + "deprecation_date": "2026-12-11", "input_cost_per_token": 0.00000125, "input_cost_per_token_flex": 6.25e-7, "input_cost_per_token_priority": 0.0000025, @@ -21012,6 +21213,7 @@ }, "gpt-5-chat-latest": { "cache_read_input_token_cost": 1.25e-7, + "deprecation_date": "2026-07-23", "input_cost_per_token": 0.00000125, "litellm_provider": "openai", "max_input_tokens": 128000, @@ -21091,6 +21293,7 @@ "cache_read_input_token_cost": 2.5e-8, "cache_read_input_token_cost_flex": 1.25e-8, "cache_read_input_token_cost_priority": 4.5e-8, + "deprecation_date": "2026-12-11", "input_cost_per_token": 2.5e-7, "input_cost_per_token_flex": 1.25e-7, "input_cost_per_token_priority": 4.5e-7, @@ -21172,7 +21375,9 @@ "gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5e-9, "cache_read_input_token_cost_flex": 2.5e-9, + "deprecation_date": "2026-12-11", "input_cost_per_token": 5e-8, + "input_cost_per_token_priority": 0.0000025, "input_cost_per_token_flex": 2.5e-8, "litellm_provider": "openai", "max_input_tokens": 272000, @@ -21332,6 +21537,7 @@ "gpt-5.1-chat-latest": { "cache_read_input_token_cost": 1.25e-7, "cache_read_input_token_cost_priority": 2.5e-7, + "deprecation_date": "2026-07-23", "input_cost_per_token": 0.00000125, "input_cost_per_token_priority": 0.0000025, "litellm_provider": "openai", @@ -21451,6 +21657,7 @@ "gpt-5.2-chat-latest": { "cache_read_input_token_cost": 1.75e-7, "cache_read_input_token_cost_priority": 3.5e-7, + "deprecation_date": "2026-08-10", "input_cost_per_token": 0.00000175, "input_cost_per_token_priority": 0.0000035, "litellm_provider": "openai", @@ -21489,6 +21696,7 @@ "gpt-5.3-chat-latest": { "cache_read_input_token_cost": 1.75e-7, "cache_read_input_token_cost_priority": 3.5e-7, + "deprecation_date": "2026-08-10", "input_cost_per_token": 0.00000175, "input_cost_per_token_priority": 0.0000035, "litellm_provider": "openai", @@ -22122,6 +22330,7 @@ "supports_xhigh_reasoning_effort": true }, "gpt-audio": { + "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 0.000032, "input_cost_per_token": 0.0000025, "litellm_provider": "openai", @@ -22191,6 +22400,7 @@ "supports_vision": false }, "gpt-audio-2025-08-28": { + "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 0.000032, "input_cost_per_token": 0.0000025, "litellm_provider": "openai", @@ -22227,6 +22437,7 @@ "supports_vision": false }, "gpt-audio-mini": { + "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 0.00001, "input_cost_per_token": 6e-7, "litellm_provider": "openai", @@ -22263,6 +22474,7 @@ "supports_vision": false }, "gpt-audio-mini-2025-10-06": { + "deprecation_date": "2026-07-23", "input_cost_per_audio_token": 0.00001, "input_cost_per_token": 6e-7, "litellm_provider": "openai", @@ -24672,6 +24884,19 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "mistral/labs-leanstral-1-5": { + "input_cost_per_token": 0, + "litellm_provider": "mistral", + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 0, + "source": "https://docs.mistral.ai/models/model-cards/leanstral-1-5", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, "mistral/magistral-medium-1-2-2509": { "input_cost_per_token": 0.000002, "litellm_provider": "mistral", @@ -25064,6 +25289,21 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "mistral/mistral-small-2603": { + "input_cost_per_token": 1.5e-7, + "litellm_provider": "mistral", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 6e-7, + "source": "https://docs.mistral.ai/models/model-cards/mistral-small-4-0-26-03", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, "mistral/mistral-small-3-2-2506": { "input_cost_per_token": 6e-8, "litellm_provider": "mistral", @@ -27156,6 +27396,7 @@ }, "o1": { "cache_read_input_token_cost": 0.0000075, + "deprecation_date": "2026-10-23", "input_cost_per_token": 0.000015, "litellm_provider": "openai", "max_input_tokens": 200000, @@ -27175,6 +27416,7 @@ }, "o1-2024-12-17": { "cache_read_input_token_cost": 0.0000075, + "deprecation_date": "2026-10-23", "input_cost_per_token": 0.000015, "litellm_provider": "openai", "max_input_tokens": 200000, @@ -27232,13 +27474,20 @@ }, "o3-2025-04-16": { "cache_read_input_token_cost": 5e-7, + "cache_read_input_token_cost_flex": 2.5e-7, + "cache_read_input_token_cost_priority": 8.75e-7, + "deprecation_date": "2026-12-11", "input_cost_per_token": 0.000002, + "input_cost_per_token_flex": 0.000001, + "input_cost_per_token_priority": 0.0000035, "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 0.000008, + "output_cost_per_token_flex": 0.000004, + "output_cost_per_token_priority": 0.000014, "supported_endpoints": [ "/v1/responses", "/v1/chat/completions", @@ -27264,6 +27513,7 @@ }, "o3-deep-research": { "cache_read_input_token_cost": 0.0000025, + "deprecation_date": "2026-07-23", "input_cost_per_token": 0.00001, "input_cost_per_token_batches": 0.000005, "litellm_provider": "openai", @@ -27298,6 +27548,7 @@ }, "o3-deep-research-2025-06-26": { "cache_read_input_token_cost": 0.0000025, + "deprecation_date": "2026-07-23", "input_cost_per_token": 0.00001, "input_cost_per_token_batches": 0.000005, "litellm_provider": "openai", @@ -27332,6 +27583,7 @@ }, "o3-mini": { "cache_read_input_token_cost": 5.5e-7, + "deprecation_date": "2026-10-23", "input_cost_per_token": 0.0000011, "litellm_provider": "openai", "max_input_tokens": 200000, @@ -27349,6 +27601,7 @@ }, "o3-mini-2025-01-31": { "cache_read_input_token_cost": 5.5e-7, + "deprecation_date": "2026-10-23", "input_cost_per_token": 0.0000011, "litellm_provider": "openai", "max_input_tokens": 200000, @@ -27368,6 +27621,7 @@ "cache_read_input_token_cost": 2.75e-7, "cache_read_input_token_cost_flex": 1.375e-7, "cache_read_input_token_cost_priority": 5e-7, + "deprecation_date": "2026-10-23", "input_cost_per_token": 0.0000011, "input_cost_per_token_flex": 5.5e-7, "input_cost_per_token_priority": 0.000002, @@ -27391,13 +27645,20 @@ }, "o4-mini-2025-04-16": { "cache_read_input_token_cost": 2.75e-7, + "cache_read_input_token_cost_flex": 1.375e-7, + "cache_read_input_token_cost_priority": 5e-7, + "deprecation_date": "2026-10-23", "input_cost_per_token": 0.0000011, + "input_cost_per_token_flex": 5.5e-7, + "input_cost_per_token_priority": 0.000002, "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 0.0000044, + "output_cost_per_token_flex": 0.0000022, + "output_cost_per_token_priority": 0.000008, "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_pdf_input": true, @@ -27410,6 +27671,7 @@ }, "o4-mini-deep-research": { "cache_read_input_token_cost": 5e-7, + "deprecation_date": "2026-07-23", "input_cost_per_token": 0.000002, "input_cost_per_token_batches": 0.000001, "litellm_provider": "openai", @@ -27444,6 +27706,7 @@ }, "o4-mini-deep-research-2025-06-26": { "cache_read_input_token_cost": 5e-7, + "deprecation_date": "2026-07-23", "input_cost_per_token": 0.000002, "input_cost_per_token_batches": 0.000001, "litellm_provider": "openai", @@ -30829,6 +31092,14 @@ "supports_function_calling": true, "supports_system_messages": true }, + "replicate/openai/gpt-oss-20b": { + "input_cost_per_token": 9e-8, + "output_cost_per_token": 3.6e-7, + "litellm_provider": "replicate", + "mode": "chat", + "supports_function_calling": true, + "supports_system_messages": true + }, "replicate/openai/o1": { "input_cost_per_token": 0.000015, "output_cost_per_token": 0.00006, @@ -30872,14 +31143,6 @@ "supports_function_calling": true, "supports_system_messages": true }, - "replicateopenai/gpt-oss-20b": { - "input_cost_per_token": 9e-8, - "output_cost_per_token": 3.6e-7, - "litellm_provider": "replicate", - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true - }, "sagemaker/meta-textgeneration-llama-2-13b-f": { "input_cost_per_token": 0, "litellm_provider": "sagemaker", @@ -37643,71 +37906,129 @@ "supports_tool_choice": true, "supports_web_search": true }, + "xai/grok-4.20-0309-non-reasoning": { + "cache_read_input_token_cost": 2e-7, + "input_cost_per_token": 0.00000125, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 0.0000025, + "source": "https://docs.x.ai/docs/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 0.0000025, + "output_cost_per_token_above_200k_tokens": 0.000005, + "cache_read_input_token_cost_above_200k_tokens": 4e-7, + "supports_response_schema": true + }, "xai/grok-4.20-0309-reasoning": { "cache_read_input_token_cost": 2e-7, - "input_cost_per_token": 0.000002, + "input_cost_per_token": 0.00000125, "litellm_provider": "xai", - "max_input_tokens": 2000000, - "max_output_tokens": 2000000, - "max_tokens": 2000000, + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, "mode": "chat", - "output_cost_per_token": 0.000006, + "output_cost_per_token": 0.0000025, "source": "https://docs.x.ai/docs/models", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 0.0000025, + "output_cost_per_token_above_200k_tokens": 0.000005, + "cache_read_input_token_cost_above_200k_tokens": 4e-7, + "supports_prompt_caching": true, + "supports_response_schema": true }, "xai/grok-4.20-beta-0309-non-reasoning": { "cache_read_input_token_cost": 2e-7, - "input_cost_per_token": 0.000002, + "input_cost_per_token": 0.00000125, "litellm_provider": "xai", - "max_input_tokens": 2000000, - "max_output_tokens": 2000000, - "max_tokens": 2000000, + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, "mode": "chat", - "output_cost_per_token": 0.000006, + "output_cost_per_token": 0.0000025, "source": "https://docs.x.ai/docs/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 0.0000025, + "output_cost_per_token_above_200k_tokens": 0.000005, + "cache_read_input_token_cost_above_200k_tokens": 4e-7, + "supports_response_schema": true }, "xai/grok-4.20-beta-0309-reasoning": { "cache_read_input_token_cost": 2e-7, - "input_cost_per_token": 0.000002, + "input_cost_per_token": 0.00000125, "litellm_provider": "xai", - "max_input_tokens": 2000000, - "max_output_tokens": 2000000, - "max_tokens": 2000000, + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, "mode": "chat", - "output_cost_per_token": 0.000006, + "output_cost_per_token": 0.0000025, "source": "https://docs.x.ai/docs/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 0.0000025, + "output_cost_per_token_above_200k_tokens": 0.000005, + "cache_read_input_token_cost_above_200k_tokens": 4e-7, + "supports_response_schema": true + }, + "xai/grok-4.20-multi-agent-0309": { + "cache_read_input_token_cost": 2e-7, + "input_cost_per_token": 0.00000125, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 0.0000025, + "source": "https://docs.x.ai/docs/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 0.0000025, + "output_cost_per_token_above_200k_tokens": 0.000005, + "cache_read_input_token_cost_above_200k_tokens": 4e-7, + "supports_response_schema": true }, "xai/grok-4.20-multi-agent-beta-0309": { "cache_read_input_token_cost": 2e-7, - "input_cost_per_token": 0.000002, + "input_cost_per_token": 0.00000125, "litellm_provider": "xai", - "max_input_tokens": 2000000, - "max_output_tokens": 2000000, - "max_tokens": 2000000, + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, "mode": "chat", - "output_cost_per_token": 0.000006, + "output_cost_per_token": 0.0000025, "source": "https://docs.x.ai/docs/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 0.0000025, + "output_cost_per_token_above_200k_tokens": 0.000005, + "cache_read_input_token_cost_above_200k_tokens": 4e-7, + "supports_response_schema": true }, "xai/grok-4.3": { "cache_read_input_token_cost": 2e-7, @@ -37752,8 +38073,8 @@ "supports_web_search": true }, "xai/grok-4.5": { - "cache_read_input_token_cost": 5e-7, - "cache_read_input_token_cost_above_200k_tokens": 0.000001, + "cache_read_input_token_cost": 3e-7, + "cache_read_input_token_cost_above_200k_tokens": 6e-7, "input_cost_per_token": 0.000002, "input_cost_per_token_above_200k_tokens": 0.000004, "litellm_provider": "xai", @@ -37773,8 +38094,8 @@ "supports_web_search": true }, "xai/grok-4.5-latest": { - "cache_read_input_token_cost": 5e-7, - "cache_read_input_token_cost_above_200k_tokens": 0.000001, + "cache_read_input_token_cost": 3e-7, + "cache_read_input_token_cost_above_200k_tokens": 6e-7, "input_cost_per_token": 0.000002, "input_cost_per_token_above_200k_tokens": 0.000004, "litellm_provider": "xai", @@ -37806,52 +38127,85 @@ "supports_vision": true, "supports_web_search": true }, + "xai/grok-build-0.1": { + "cache_read_input_token_cost": 2e-7, + "input_cost_per_token": 0.000001, + "litellm_provider": "xai", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 0.000002, + "source": "https://docs.x.ai/docs/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "input_cost_per_token_above_200k_tokens": 0.000002, + "output_cost_per_token_above_200k_tokens": 0.000004, + "cache_read_input_token_cost_above_200k_tokens": 4e-7, + "supports_response_schema": true, + "supports_vision": true + }, "xai/grok-code-fast": { - "cache_read_input_token_cost": 2e-8, - "input_cost_per_token": 2e-7, + "cache_read_input_token_cost": 2e-7, + "input_cost_per_token": 0.000001, "litellm_provider": "xai", "max_input_tokens": 256000, "max_output_tokens": 256000, "max_tokens": 256000, "mode": "chat", - "output_cost_per_token": 0.0000015, + "output_cost_per_token": 0.000002, "source": "https://docs.x.ai/docs/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "input_cost_per_token_above_200k_tokens": 0.000002, + "output_cost_per_token_above_200k_tokens": 0.000004, + "cache_read_input_token_cost_above_200k_tokens": 4e-7, + "supports_response_schema": true, + "supports_vision": true }, "xai/grok-code-fast-1": { - "cache_read_input_token_cost": 2e-8, - "input_cost_per_token": 2e-7, + "cache_read_input_token_cost": 2e-7, + "input_cost_per_token": 0.000001, "litellm_provider": "xai", "max_input_tokens": 256000, "max_output_tokens": 256000, "max_tokens": 256000, "mode": "chat", - "output_cost_per_token": 0.0000015, + "output_cost_per_token": 0.000002, "source": "https://docs.x.ai/docs/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "deprecation_date": "2026-05-15" + "input_cost_per_token_above_200k_tokens": 0.000002, + "output_cost_per_token_above_200k_tokens": 0.000004, + "cache_read_input_token_cost_above_200k_tokens": 4e-7, + "supports_response_schema": true, + "supports_vision": true }, "xai/grok-code-fast-1-0825": { - "cache_read_input_token_cost": 2e-8, - "input_cost_per_token": 2e-7, + "cache_read_input_token_cost": 2e-7, + "input_cost_per_token": 0.000001, "litellm_provider": "xai", "max_input_tokens": 256000, "max_output_tokens": 256000, "max_tokens": 256000, "mode": "chat", - "output_cost_per_token": 0.0000015, + "output_cost_per_token": 0.000002, "source": "https://docs.x.ai/docs/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "deprecation_date": "2026-05-15" + "input_cost_per_token_above_200k_tokens": 0.000002, + "output_cost_per_token_above_200k_tokens": 0.000004, + "cache_read_input_token_cost_above_200k_tokens": 4e-7, + "supports_response_schema": true, + "supports_vision": true }, "xai/grok-vision-beta": { "input_cost_per_image": 0.000005, @@ -37905,6 +38259,7 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, + "supports_native_structured_output": true, "supports_system_messages": true, "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" From 5e83283b3296336cd84ad226085566d671a8f630 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 11 Aug 2026 01:53:32 -0400 Subject: [PATCH 20/30] Add remaining oepnai compatible providers --- cecli/resources/providers.json | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/cecli/resources/providers.json b/cecli/resources/providers.json index 7faca9d876c..aa489216d21 100644 --- a/cecli/resources/providers.json +++ b/cecli/resources/providers.json @@ -145,6 +145,13 @@ ], "display_name": "gmi" }, + "gradient_ai": { + "api_base": "https://inference.do-ai.run/v1", + "api_key_env": [ + "GRADIENT_AI_API_KEY" + ], + "display_name": "gradient_ai" + }, "groq": { "api_base": "https://api.groq.com/openai/v1", "api_key_env": [ @@ -214,6 +221,16 @@ ], "display_name": "meta_llama" }, + "minimax": { + "api_base": "https://api.minimax.io/v1", + "api_key_env": [ + "MINIMAX_API_KEY" + ], + "base_url_env": [ + "MINIMAX_API_BASE" + ], + "display_name": "minimax" + }, "mistral": { "api_base": "https://api.mistral.ai/v1", "api_key_env": [ From afb0a1b8580cfa20efe73ae131ddae18c88e270d Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 11 Aug 2026 07:29:31 -0400 Subject: [PATCH 21/30] Add Azure and Bedrock providers based off of LiteLLM's implementation --- cecli/helpers/llms/aws_sigv4.py | 220 +++++++++++++ cecli/helpers/llms/config.py | 8 + cecli/helpers/llms/domains/__init__.py | 7 +- cecli/helpers/llms/domains/bedrock.py | 307 ++++++++++++++++++ cecli/helpers/llms/domains/chat.py | 34 +- cecli/helpers/llms/pipeline.py | 16 + cecli/helpers/llms/providers/azure.py | 47 +++ cecli/helpers/llms/providers/bedrock.py | 49 +++ .../helpers/llms/providers/bedrock_mantle.py | 125 +++++++ cecli/resources/providers.json | 31 ++ tests/helpers/test_llms_aws_sigv4.py | 153 +++++++++ tests/helpers/test_llms_hyperscalers.py | 306 +++++++++++++++++ tests/helpers/test_llms_providers_json.py | 36 +- 13 files changed, 1316 insertions(+), 23 deletions(-) create mode 100644 cecli/helpers/llms/aws_sigv4.py create mode 100644 cecli/helpers/llms/domains/bedrock.py create mode 100644 cecli/helpers/llms/providers/azure.py create mode 100644 cecli/helpers/llms/providers/bedrock.py create mode 100644 cecli/helpers/llms/providers/bedrock_mantle.py create mode 100644 tests/helpers/test_llms_aws_sigv4.py create mode 100644 tests/helpers/test_llms_hyperscalers.py diff --git a/cecli/helpers/llms/aws_sigv4.py b/cecli/helpers/llms/aws_sigv4.py new file mode 100644 index 00000000000..f398897ca92 --- /dev/null +++ b/cecli/helpers/llms/aws_sigv4.py @@ -0,0 +1,220 @@ +"""Self-contained AWS Signature Version 4 request signing (stdlib only). + +Reimplements the subset of botocore's ``SigV4Auth`` that cecli needs for the +Bedrock / Bedrock Mantle providers, so ``boto3`` is not a runtime dependency. +The signing-header filter mirrors litellm's +``BaseAWSLLM._filter_headers_for_aws_signature``: only ``host``, +``content-type``, ``date``, ``x-amz-*`` and ``x-amzn-*`` headers participate in +canonicalization (forwarded client headers stay unsigned). + +Verified against the AWS SigV4 test vector from the official docs (IAM +``ListUsers`` example) in ``tests/helpers/test_llms_aws_sigv4.py``. +""" + +from __future__ import annotations + +import hashlib +import hmac +import urllib.parse +from datetime import datetime, timezone +from typing import Dict, Optional + +#: Headers AWS SigV4 includes in the canonical request / signed headers. +_SIGNABLE_HEADERS = { + "host", + "content-type", + "date", + "x-amz-date", + "x-amz-security-token", + "x-amz-content-sha256", + "x-amz-algorithm", + "x-amz-credential", + "x-amz-signedheaders", + "x-amz-signature", +} + + +class AWSCredentials: + """Minimal AWS credential holder (access key / secret / optional session token).""" + + def __init__( + self, + access_key: str, + secret_key: str, + session_token: Optional[str] = None, + expiry: Optional[datetime] = None, + ) -> None: + self.access_key = access_key + self.secret_key = secret_key + self.session_token = session_token + self.expiry = expiry + + @classmethod + def from_env(cls) -> Optional["AWSCredentials"]: + """Build credentials from the standard AWS environment variables.""" + import os + + access_key = os.environ.get("AWS_ACCESS_KEY_ID") + secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY") + if not access_key or not secret_key: + return None + return cls( + access_key=access_key, + secret_key=secret_key, + session_token=os.environ.get("AWS_SESSION_TOKEN"), + ) + + +def _filter_headers_for_signature(headers: Dict[str, str]) -> Dict[str, str]: + """Return only the headers AWS SigV4 includes in the signature.""" + out: Dict[str, str] = {} + for name, value in headers.items(): + if value is None: + continue + lower = name.lower() + if lower in _SIGNABLE_HEADERS or lower.startswith("x-amz-") or lower.startswith("x-amzn-"): + out[name] = value + return out + + +def _sha256_hex(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _hmac(key: bytes, msg: bytes) -> bytes: + return hmac.new(key, msg, hashlib.sha256).digest() + + +def _canonical_uri(path: str) -> str: + """URI-encode each path segment, never re-encoding the ``/`` separator.""" + if not path or path == "/": + return "/" + segments = [urllib.parse.quote(seg, safe="/-_.~") for seg in path.split("/")] + return "/".join(segments) + + +def _canonical_query(query: str) -> str: + """Sort query params by (encoded key, encoded value) and join with '&'.""" + params = urllib.parse.parse_qsl(query, keep_blank_values=True) + encoded = [ + (urllib.parse.quote(k, safe="-_.~"), urllib.parse.quote(v, safe="-_.~")) for k, v in params + ] + encoded.sort() + return "&".join(f"{k}={v}" for k, v in encoded) + + +def _canonical_headers(headers: Dict[str, str]) -> tuple[str, str]: + """Return (canonical_headers, signed_headers) sorted by lower-cased name.""" + items = sorted( + ((name.lower(), value.strip()) for name, value in headers.items() if value is not None), + key=lambda pair: pair[0], + ) + canonical = "".join(f"{name}:{value}\n" for name, value in items) + signed = ";".join(name for name, _ in items) + return canonical, signed + + +def sign_request( + method: str, + url: str, + payload: bytes, + credentials: AWSCredentials, + region: str, + service: str, + headers: Optional[Dict[str, str]] = None, + now: Optional[datetime] = None, +) -> Dict[str, str]: + """Sign a request with AWS SigV4 and return the headers to send. + + Args: + method: HTTP method (``POST`` for the Bedrock APIs). + url: Full request URL. + payload: Encoded request body. + credentials: AWS credentials (access key / secret / optional session token). + region: AWS region (e.g. ``us-east-1``). + service: AWS service name (``bedrock`` for Bedrock / Mantle). + headers: Extra headers to sign (e.g. ``content-type``); ``host`` and + ``x-amz-date`` are added automatically. + now: Fixed timestamp (for deterministic tests). + + Returns: + The complete headers dict including ``Authorization``, ``X-Amz-Date`` + and ``X-Amz-Security-Token`` (when a session token is set). The payload + hash is part of the canonical request but not sent as a header, matching + the AWS reference examples. + """ + now = now or datetime.now(timezone.utc) + amz_date = now.strftime("%Y%m%dT%H%M%SZ") + date_stamp = now.strftime("%Y%m%d") + + parsed = urllib.parse.urlsplit(url) + host = parsed.netloc + path = parsed.path or "/" + query = parsed.query + + payload_hash = _sha256_hex(payload) + + # Normalize header names to lowercase so a caller-supplied ``Content-Type`` + # and our ``setdefault`` can never collide into a duplicate signed header. + sig_headers = { + name.lower(): value for name, value in (headers or {}).items() if value is not None + } + sig_headers.setdefault("host", host) + sig_headers.setdefault("x-amz-date", amz_date) + sig_headers.setdefault("content-type", "application/json") + if credentials.session_token: + sig_headers.setdefault("x-amz-security-token", credentials.session_token) + + # Only AWS-relevant headers participate in canonicalization (mirrors litellm). + sig_headers = _filter_headers_for_signature(sig_headers) + + canonical_headers, signed_headers = _canonical_headers(sig_headers) + canonical_request = "\n".join( + [ + method.upper(), + _canonical_uri(path), + _canonical_query(query), + canonical_headers, + signed_headers, + payload_hash, + ] + ) + + scope = f"{date_stamp}/{region}/{service}/aws4_request" + string_to_sign = "\n".join( + [ + "AWS4-HMAC-SHA256", + amz_date, + scope, + _sha256_hex(canonical_request.encode("utf-8")), + ] + ) + + k_date = _hmac(("AWS4" + credentials.secret_key).encode("utf-8"), date_stamp.encode("utf-8")) + k_region = _hmac(k_date, region.encode("utf-8")) + k_service = _hmac(k_region, service.encode("utf-8")) + k_signing = _hmac(k_service, b"aws4_request") + signature = hmac.new(k_signing, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest() + + authorization = ( + f"AWS4-HMAC-SHA256 Credential={credentials.access_key}/{scope}, " + f"SignedHeaders={signed_headers}, Signature={signature}" + ) + + out = dict(sig_headers) + out["Authorization"] = authorization + return out + + +def resolve_aws_region() -> Optional[str]: + """Resolve the AWS region from the environment (litellm-compatible order).""" + import os + + return os.environ.get("AWS_REGION_NAME") or os.environ.get("AWS_REGION") or None + + +__all__ = [ + "AWSCredentials", + "resolve_aws_region", + "sign_request", +] diff --git a/cecli/helpers/llms/config.py b/cecli/helpers/llms/config.py index 98201733ba1..74f16b7bd8f 100644 --- a/cecli/helpers/llms/config.py +++ b/cecli/helpers/llms/config.py @@ -81,6 +81,13 @@ def resolve_model_config(model: str) -> Dict[str, Any]: family = "chat" else: family = "responses" + elif provider in ("bedrock", "bedrock_converse"): + # AWS Bedrock Converse wire (SigV4-signed; see domains/bedrock.py). + family = "bedrock" + elif provider == "bedrock_mantle": + # Mantle is an OpenAI-compatible chat wire (SigV4-signed via the chat + # family's signer hook); see providers/bedrock_mantle.py. + family = "chat" elif mode == "responses" or "/v1/responses" in endpoints: family = "responses" elif provider == "anthropic": @@ -102,6 +109,7 @@ def resolve_model_config(model: str) -> Dict[str, Any]: "api_key_env": key_env, "extra_headers": extra_headers, "extra_body": extra_body, + "extra_query": dict(pcfg.get("extra_query") or {}), "api_block": api_block, "llm_block": llm_block, } diff --git a/cecli/helpers/llms/domains/__init__.py b/cecli/helpers/llms/domains/__init__.py index 398103b81cf..3bd0bfcb63e 100644 --- a/cecli/helpers/llms/domains/__init__.py +++ b/cecli/helpers/llms/domains/__init__.py @@ -2,12 +2,13 @@ One module per API family: chat (OpenAI /v1/chat/completions), responses (OpenAI /v1/responses), messages (Anthropic /v1/messages), gemini -(generateContent). Each exports ``*_complete`` / ``*_stream`` entry points -plus payload builders and response normalizers. +(generateContent), bedrock (AWS Bedrock Converse). Each exports ``*_complete`` +/ ``*_stream`` entry points plus payload builders and response normalizers. """ from __future__ import annotations +from .bedrock import bedrock_complete, bedrock_stream from .chat import chat_complete, chat_stream from .gemini import gemini_complete, gemini_stream from .messages import anthropic_complete, anthropic_stream @@ -22,4 +23,6 @@ "anthropic_stream", "gemini_complete", "gemini_stream", + "bedrock_complete", + "bedrock_stream", ] diff --git a/cecli/helpers/llms/domains/bedrock.py b/cecli/helpers/llms/domains/bedrock.py new file mode 100644 index 00000000000..68681504bab --- /dev/null +++ b/cecli/helpers/llms/domains/bedrock.py @@ -0,0 +1,307 @@ +"""AWS Bedrock Converse API adapter (non-streaming). + +Converse is Bedrock's provider-neutral chat wire: OpenAI-style messages are +transformed to the Converse ``messages``/``system``/``inferenceConfig``/ +``toolConfig`` shape, and the request is authenticated with AWS Signature V4 +(see :mod:`cecli.helpers.llms.aws_sigv4`). The response is normalized back to +:class:`~cecli.helpers.llms.types.CompletionResponse`. + +Streaming (``/model/{id}/converse-stream``) uses AWS's binary event-stream +encoding, which is not implemented yet; the provider entry sets +``supports_stream: false`` and :func:`bedrock_stream` raises a clear error. +""" + +from __future__ import annotations + +import json +from typing import Any, AsyncIterator, Dict, List, Optional + +from ..aws_sigv4 import AWSCredentials, resolve_aws_region, sign_request +from ..runtime import VERIFY_SSL, make_client +from ..types import ( + Choice, + CompletionChunk, + CompletionResponse, + PartsMessage, + TextPart, + ToolCallPart, + Usage, + parts_message_to_message, +) + +DEFAULT_TIMEOUT = 120.0 + +#: Converse ``stopReason`` -> OpenAI finish_reason mapping. +_STOP_REASON_MAP = { + "end_turn": "stop", + "tool_use": "tool_calls", + "max_tokens": "length", + "guardrail_intervened": "content_filter", +} + + +def bedrock_payload( + resolved: Dict[str, Any], + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]], + kwargs: Dict[str, Any], +) -> Dict[str, Any]: + """Transform OpenAI-style messages into a Bedrock Converse request body.""" + payload: Dict[str, Any] = {} + + system_blocks: List[Dict[str, Any]] = [] + converse_messages: List[Dict[str, Any]] = [] + + for msg in messages: + role = msg.get("role") + + if role == "system": + text = msg.get("content") + if text: + system_blocks.append({"text": text}) + continue + + if role == "tool": + converse_messages.append( + { + "role": "user", + "content": [ + { + "toolResult": { + "toolUseId": msg.get("tool_call_id", ""), + "content": [{"text": msg.get("content") or ""}], + "status": "success", + } + } + ], + } + ) + continue + + if role == "assistant": + blocks: List[Dict[str, Any]] = [] + content = msg.get("content") or "" + if content: + blocks.append({"text": content}) + + for tc in msg.get("tool_calls") or []: + fn = tc.get("function") or {} + args_raw = fn.get("arguments") or "{}" + + try: + args = json.loads(args_raw) if isinstance(args_raw, str) else args_raw + except json.JSONDecodeError: + args = {} + + blocks.append( + { + "toolUse": { + "toolUseId": tc.get("id", ""), + "name": fn.get("name", ""), + "input": args, + } + } + ) + + converse_messages.append({"role": "assistant", "content": blocks}) + continue + + # user + blocks = _user_content_blocks(msg) + converse_messages.append({"role": "user", "content": blocks}) + + if system_blocks: + payload["system"] = system_blocks + + payload["messages"] = converse_messages + + inference: Dict[str, Any] = {} + + max_tokens = kwargs.get("max_tokens") + if max_tokens: + inference["maxTokens"] = max_tokens + + temperature = kwargs.get("temperature") + if temperature is not None: + inference["temperature"] = temperature + + top_p = kwargs.get("top_p") + if top_p is not None: + inference["topP"] = top_p + + stop = kwargs.get("stop") or kwargs.get("stop_sequences") + if stop: + inference["stopSequences"] = stop if isinstance(stop, list) else [stop] + + if inference: + payload["inferenceConfig"] = inference + + if tools: + specs = [] + for tool in tools: + fn = tool.get("function") or {} + spec: Dict[str, Any] = {"name": fn.get("name", "")} + if fn.get("description"): + spec["description"] = fn["description"] + if fn.get("parameters"): + spec["inputSchema"] = fn["parameters"] + specs.append({"toolSpec": spec}) + + tool_config: Dict[str, Any] = {"tools": specs} + + tool_choice = kwargs.get("tool_choice") + if tool_choice in ("none", "auto", "any"): + tool_config["toolChoice"] = {"type": tool_choice} + + payload["toolConfig"] = tool_config + + extra_body = dict(resolved.get("extra_body") or {}) + extra_body.update(kwargs.get("extra_body") or {}) + payload.update(extra_body) + return payload + + +def _user_content_blocks(msg: Dict[str, Any]) -> List[Dict[str, Any]]: + """Convert a user message's content (str or OpenAI part list) to Converse text blocks.""" + content = msg.get("content") + blocks: List[Dict[str, Any]] = [] + + if isinstance(content, str): + if content: + blocks.append({"text": content}) + + elif isinstance(content, list): + for block in content: + if isinstance(block, dict) and block.get("type") == "text" and block.get("text"): + blocks.append({"text": block["text"]}) + + return blocks + + +def bedrock_region(resolved: Dict[str, Any]) -> Optional[str]: + """Resolve the AWS region for the request (explicit override, then env).""" + return resolved.get("aws_region") or resolve_aws_region() + + +def bedrock_endpoint(resolved: Dict[str, Any]) -> str: + """Return the Converse endpoint URL for the resolved model.""" + region = bedrock_region(resolved) + + if not region: + raise ValueError( + "Bedrock requires an AWS region: set AWS_REGION_NAME or AWS_REGION " + "(or pass aws_region in the provider config)." + ) + + route = resolved["route"] + return f"https://bedrock-runtime.{region}.amazonaws.com/model/{route}/converse" + + +def _signed_headers( + resolved: Dict[str, Any], url: str, body: bytes, headers: Dict[str, str] +) -> Dict[str, str]: + """Sign the Converse request with AWS SigV4.""" + creds = AWSCredentials.from_env() + + if creds is None: + raise ValueError( + "Bedrock requires AWS credentials: set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY " + "(AWS_SESSION_TOKEN when using temporary credentials)." + ) + + region = bedrock_region(resolved) + hdrs = {"Content-Type": "application/json", **headers} + return sign_request("POST", url, body, creds, region, "bedrock", headers=hdrs) + + +async def bedrock_complete( + resolved: Dict[str, Any], + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]], + key: Optional[str], + headers: Dict[str, str], + kwargs: Dict[str, Any], +) -> CompletionResponse: + """Send a non-streaming Bedrock Converse completion.""" + url = bedrock_endpoint(resolved) + payload = bedrock_payload(resolved, messages, tools, kwargs) + body = json.dumps(payload) + + hdrs = _signed_headers(resolved, url, body.encode("utf-8"), headers) + + async with make_client(timeout=DEFAULT_TIMEOUT, verify=VERIFY_SSL) as client: + resp = await client.post(url, content=body.encode("utf-8"), headers=hdrs) + resp.raise_for_status() + data = resp.json() + + return normalize_bedrock_response(data, resolved["model"]) + + +async def bedrock_stream( + resolved: Dict[str, Any], + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]], + key: Optional[str], + headers: Dict[str, str], + kwargs: Dict[str, Any], +) -> AsyncIterator[CompletionChunk]: + """Streaming is not implemented for Bedrock Converse yet.""" + raise NotImplementedError( + "Bedrock Converse streaming requires AWS event-stream decoding and is not supported yet. " + "Use a non-streaming request (the bedrock provider sets supports_stream: false)." + ) + + +def normalize_bedrock_response(data: Dict[str, Any], model: str) -> CompletionResponse: + """Normalize a Converse response body to a CompletionResponse.""" + output = data.get("output") or {} + message = output.get("message") or {} + content = message.get("content") or [] + + parts = [] + + for block in content: + if block.get("text"): + parts.append(TextPart(text=block["text"])) + elif block.get("toolUse"): + tool_use = block["toolUse"] + parts.append( + ToolCallPart( + name=tool_use.get("name", ""), + arguments=tool_use.get("input") or {}, + tool_call_id=tool_use.get("toolUseId"), + ) + ) + + stop_reason = data.get("stopReason") + finish_reason = _STOP_REASON_MAP.get(stop_reason) + + usage_raw = data.get("usage") or {} + usage = Usage( + prompt_tokens=usage_raw.get("inputTokens"), + completion_tokens=usage_raw.get("outputTokens"), + total_tokens=usage_raw.get("totalTokens"), + ) + + pm = PartsMessage(role="assistant", parts=parts) + choices = [Choice(index=0, message=parts_message_to_message(pm), finish_reason=finish_reason)] + + provider_fields: Dict[str, Any] = {} + if stop_reason: + provider_fields["stop_reason"] = stop_reason + + return CompletionResponse( + id=None, + model=model, + choices=choices, + usage=usage or None, + provider_specific_fields=provider_fields, + ) + + +__all__ = [ + "bedrock_complete", + "bedrock_payload", + "bedrock_stream", + "normalize_bedrock_response", +] diff --git a/cecli/helpers/llms/domains/chat.py b/cecli/helpers/llms/domains/chat.py index 5bf4d658e03..8b7499a7c03 100644 --- a/cecli/helpers/llms/domains/chat.py +++ b/cecli/helpers/llms/domains/chat.py @@ -96,10 +96,23 @@ async def chat_complete( ) -> CompletionResponse: url = f"{resolved['api_base']}/chat/completions" payload = chat_payload(resolved, messages, tools, False, kwargs) - hdrs = {"Authorization": f"Bearer {key}", "Content-Type": "application/json", **headers} + hdrs = {"Content-Type": "application/json", **headers} + body: Optional[bytes] = None + signer = resolved.get("_signer") + + if signer: + url, hdrs, body = signer(url, payload, hdrs, key) + + params = dict(resolved.get("extra_query") or {}) or None + post_kwargs: Dict[str, Any] = {} + + if body is not None: + post_kwargs["content"] = body + else: + post_kwargs["json"] = payload async with make_client(timeout=DEFAULT_TIMEOUT, verify=VERIFY_SSL) as client: - resp = await client.post(url, json=payload, headers=hdrs) + resp = await client.post(url, headers=hdrs, params=params, **post_kwargs) resp.raise_for_status() data = resp.json() @@ -116,10 +129,23 @@ async def chat_stream( ) -> AsyncIterator[CompletionChunk]: url = f"{resolved['api_base']}/chat/completions" payload = chat_payload(resolved, messages, tools, True, kwargs) - hdrs = {"Authorization": f"Bearer {key}", "Content-Type": "application/json", **headers} + hdrs = {"Content-Type": "application/json", **headers} + body: Optional[bytes] = None + signer = resolved.get("_signer") + + if signer: + url, hdrs, body = signer(url, payload, hdrs, key) + + params = dict(resolved.get("extra_query") or {}) or None + stream_kwargs: Dict[str, Any] = {} + + if body is not None: + stream_kwargs["content"] = body + else: + stream_kwargs["json"] = payload async with make_client(timeout=DEFAULT_TIMEOUT, verify=VERIFY_SSL) as client: - async with client.stream("POST", url, json=payload, headers=hdrs) as resp: + async with client.stream("POST", url, headers=hdrs, params=params, **stream_kwargs) as resp: resp.raise_for_status() last_finish_reason = None diff --git a/cecli/helpers/llms/pipeline.py b/cecli/helpers/llms/pipeline.py index 31d95c2da7d..116a0a5de2b 100644 --- a/cecli/helpers/llms/pipeline.py +++ b/cecli/helpers/llms/pipeline.py @@ -16,6 +16,8 @@ from .domains import ( anthropic_complete, anthropic_stream, + bedrock_complete, + bedrock_stream, chat_complete, chat_stream, gemini_complete, @@ -49,6 +51,11 @@ async def acompletion( family = resolved["family"] + # Providers that need to sign the final request (URL + body) expose a + # ``sign_request`` hook; the family adapter invokes it after building the + # payload (e.g. Bedrock Mantle's SigV4 path). + resolved["_signer"] = getattr(provider, "sign_request", None) + headers = dict(resolved.get("extra_headers") or {}) headers.update(extra_headers or {}) headers = provider.build_headers(resolved, key, family, headers) @@ -81,6 +88,9 @@ async def _complete_family( if family == "gemini": return await gemini_complete(resolved, messages, tools, key, headers, kwargs) + if family == "bedrock": + return await bedrock_complete(resolved, messages, tools, key, headers, kwargs) + return await chat_complete(resolved, messages, tools, key, headers, kwargs) @@ -111,6 +121,12 @@ async def _stream_family( return + if family == "bedrock": + async for chunk in bedrock_stream(resolved, messages, tools, key, headers, kwargs): + yield chunk + + return + async for chunk in chat_stream(resolved, messages, tools, key, headers, kwargs): yield chunk diff --git a/cecli/helpers/llms/providers/azure.py b/cecli/helpers/llms/providers/azure.py new file mode 100644 index 00000000000..462c4f43057 --- /dev/null +++ b/cecli/helpers/llms/providers/azure.py @@ -0,0 +1,47 @@ +"""Azure OpenAI provider adapter. + +Azure OpenAI speaks the OpenAI-compatible wire but authenticates with an +``api-key`` header (not ``Authorization: Bearer``) and requires an +``api-version`` query param on every request. The api-version rides in the +provider config's ``extra_query`` (surfaced by ``resolve_model_config``) and is +appended by the chat domain; the api-key header is applied here. +""" + +from __future__ import annotations + +import os +from typing import Any, Dict, Optional + +from .base import ProviderAdapter + + +class AzureProvider(ProviderAdapter): + """Azure OpenAI: api-key header + api-version query param.""" + + provider: str = "azure" + + def resolve_api_key(self, resolved: Dict[str, Any], api_key: Optional[str]) -> Optional[str]: + """Return the Azure API key (AZURE_API_KEY first, then AZURE_OPENAI_API_KEY).""" + if api_key: + return api_key + + return os.environ.get("AZURE_API_KEY") or os.environ.get("AZURE_OPENAI_API_KEY") + + def build_headers( + self, + resolved: Dict[str, Any], + key: Optional[str], + family: str, + headers: Dict[str, str], + ) -> Dict[str, str]: + """Authenticate with the ``api-key`` header instead of Bearer.""" + merged = dict(headers) + + if key: + merged["api-key"] = key + + merged.setdefault("Content-Type", "application/json") + return merged + + +__all__ = ["AzureProvider"] diff --git a/cecli/helpers/llms/providers/bedrock.py b/cecli/helpers/llms/providers/bedrock.py new file mode 100644 index 00000000000..005f6c4c23b --- /dev/null +++ b/cecli/helpers/llms/providers/bedrock.py @@ -0,0 +1,49 @@ +"""AWS Bedrock provider adapter. + +Bedrock authenticates every request with AWS Signature V4 (service +``bedrock``) using ambient AWS credentials — there is no API key. This adapter +resolves the region (``{region}`` placeholder in the api_base template is +substituted from the environment) and leaves auth to the bedrock domain, which +signs the final URL + body via :mod:`cecli.helpers.llms.aws_sigv4`. +""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +from ..aws_sigv4 import resolve_aws_region +from .base import ProviderAdapter + + +class BedrockProvider(ProviderAdapter): + """Bedrock: SigV4-signed Converse requests, no API key.""" + + provider: str = "bedrock" + + def resolve_api_base(self, resolved: Dict[str, Any]) -> str: + """Substitute the ``{region}`` placeholder from the environment.""" + base = resolved["api_base"] + + if not resolved.get("aws_region"): + region = resolve_aws_region() + + if region: + resolved["aws_region"] = region + elif "{region}" in base: + raise ValueError( + "Bedrock requires an AWS region: set AWS_REGION_NAME or AWS_REGION." + ) + + region = resolved.get("aws_region") + + if region and "{region}" in base: + return base.replace("{region}", region) + + return base + + def resolve_api_key(self, resolved: Dict[str, Any], api_key: Optional[str]) -> Optional[str]: + """Bedrock uses SigV4 (AWS env credentials); no API key is needed.""" + return None + + +__all__ = ["BedrockProvider"] diff --git a/cecli/helpers/llms/providers/bedrock_mantle.py b/cecli/helpers/llms/providers/bedrock_mantle.py new file mode 100644 index 00000000000..99c2b014984 --- /dev/null +++ b/cecli/helpers/llms/providers/bedrock_mantle.py @@ -0,0 +1,125 @@ +"""AWS Bedrock Mantle provider adapter. + +Bedrock Mantle exposes an OpenAI-compatible chat wire (``/v1/chat/completions``) +at ``https://bedrock-mantle.{region}.api.aws/v1``. Auth is either a bearer token +(``BEDROCK_MANTLE_API_KEY`` / ``AWS_BEARER_TOKEN_BEDROCK``) or AWS Signature V4 +when no token is set. When a token is present the chat family sends ``Bearer``; +otherwise :meth:`sign_request` (invoked by the chat domain) signs the request +with SigV4, mirroring litellm's ``BedrockMantleAuthMixin``. +""" + +from __future__ import annotations + +import json +import os +import re +from typing import Any, Dict, Optional, Tuple + +from ..aws_sigv4 import AWSCredentials, resolve_aws_region, sign_request +from .base import ProviderAdapter + +#: Host pattern ``https://bedrock-mantle.{region}.api.aws/...``. +_MANTLE_HOST_RE = re.compile(r"^https?://bedrock-mantle\.([^./]+)\.api\.aws", re.IGNORECASE) +_DEFAULT_REGION = "us-east-1" + + +class BedrockMantleProvider(ProviderAdapter): + """Bedrock Mantle: OpenAI-compatible chat with Bearer or SigV4 auth.""" + + provider: str = "bedrock_mantle" + + def _region_from_base(self, api_base: str) -> Optional[str]: + match = _MANTLE_HOST_RE.match(api_base.rstrip("/")) + + if match and match.group(1) != "{region}": + # Skip the literal ``{region}`` placeholder from an unsubstituted + # template so it never becomes a "region". + return match.group(1) + + return None + match = _MANTLE_HOST_RE.match(api_base.rstrip("/")) + + if match: + return match.group(1) + + return None + + def resolve_region(self, resolved: Dict[str, Any]) -> str: + """Resolve the signing region (explicit > host > env > default).""" + if resolved.get("aws_region"): + return resolved["aws_region"] + + base = resolved.get("api_base") or "" + host_region = self._region_from_base(base) + + if host_region: + return host_region + + return os.environ.get("BEDROCK_MANTLE_REGION") or resolve_aws_region() or _DEFAULT_REGION + + def resolve_api_base(self, resolved: Dict[str, Any]) -> str: + """Substitute the ``{region}`` placeholder in the api_base template.""" + base = resolved["api_base"] + region = self.resolve_region(resolved) + resolved["aws_region"] = region + + if "{region}" in base: + return base.replace("{region}", region) + + return base + + def resolve_api_key(self, resolved: Dict[str, Any], api_key: Optional[str]) -> Optional[str]: + """Return a bearer token when one is configured (else SigV4 is used).""" + if api_key: + return api_key + + return os.environ.get("BEDROCK_MANTLE_API_KEY") or os.environ.get( + "AWS_BEARER_TOKEN_BEDROCK" + ) + + def build_headers( + self, + resolved: Dict[str, Any], + key: Optional[str], + family: str, + headers: Dict[str, str], + ) -> Dict[str, str]: + """Add the Bearer token when present (SigV4 path adds its own headers).""" + merged = dict(headers) + + if key: + merged["Authorization"] = f"Bearer {key}" + + merged.setdefault("Content-Type", "application/json") + return merged + + def sign_request( + self, + url: str, + payload: Dict[str, Any], + headers: Dict[str, str], + key: Optional[str], + ) -> Tuple[str, Dict[str, str], Optional[bytes]]: + """Sign a Mantle request with SigV4 (no-op when a bearer token is set).""" + if key: + return url, headers, None + + creds = AWSCredentials.from_env() + + if creds is None: + raise ValueError( + "Bedrock Mantle requires either a bearer token (BEDROCK_MANTLE_API_KEY or " + "AWS_BEARER_TOKEN_BEDROCK) or AWS credentials (AWS_ACCESS_KEY_ID / " + "AWS_SECRET_ACCESS_KEY)." + ) + + region = self.resolve_region( + {**{k: v for k, v in [("api_base", url)]}, "aws_region": None} or {} + ) + region = region or _DEFAULT_REGION + body = json.dumps(payload).encode("utf-8") + signed = sign_request("POST", url, body, creds, region, "bedrock", headers=headers) + return url, signed, body + + +__all__ = ["BedrockMantleProvider"] diff --git a/cecli/resources/providers.json b/cecli/resources/providers.json index aa489216d21..9edc89bb551 100644 --- a/cecli/resources/providers.json +++ b/cecli/resources/providers.json @@ -20,6 +20,19 @@ ], "display_name": "apertis" }, + "azure": { + "api_base": "https://{resource}.openai.azure.com/openai/deployments/{deployment}", + "api_key_env": [ + "AZURE_API_KEY" + ], + "base_url_env": [ + "AZURE_API_BASE" + ], + "display_name": "azure", + "extra_query": { + "api-version": "2024-10-21" + } + }, "azure_ai": { "api_base": "https://{resource}.services.ai.azure.com", "api_key_env": [ @@ -34,6 +47,24 @@ ], "display_name": "baseten" }, + "bedrock": { + "api_base": "https://bedrock-runtime.{region}.amazonaws.com", + "api_key_env": [ + "AWS_ACCESS_KEY_ID" + ], + "display_name": "bedrock", + "supports_stream": false + }, + "bedrock_mantle": { + "api_base": "https://bedrock-mantle.{region}.api.aws/v1", + "api_key_env": [ + "BEDROCK_MANTLE_API_KEY" + ], + "base_url_env": [ + "BEDROCK_MANTLE_API_BASE" + ], + "display_name": "bedrock_mantle" + }, "cerebras": { "api_base": "https://api.cerebras.ai/v1", "api_key_env": [ diff --git a/tests/helpers/test_llms_aws_sigv4.py b/tests/helpers/test_llms_aws_sigv4.py new file mode 100644 index 00000000000..acb70ca4ab8 --- /dev/null +++ b/tests/helpers/test_llms_aws_sigv4.py @@ -0,0 +1,153 @@ +"""AWS SigV4 signing tests for :mod:`cecli.helpers.llms.aws_sigv4`. + +The implementation is verified for parity against botocore's own ``SigV4Auth`` +(Amazon's reference SDK implementation) with a pinned clock, so the tests are +meaningful without any live AWS credentials. +""" + +from __future__ import annotations + +from datetime import datetime + +import botocore.auth as botauth +import pytest +from botocore.auth import SigV4Auth +from botocore.awsrequest import AWSRequest +from botocore.credentials import Credentials + +from cecli.helpers.llms.aws_sigv4 import ( + AWSCredentials, + resolve_aws_region, + sign_request, +) + +ACCESS_KEY = "AKIAIOSFODNN7EXAMPLE" +SECRET_KEY = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" +FIXED_NOW = datetime(2015, 8, 30, 12, 36, 0) + + +@pytest.fixture(autouse=True) +def _pin_botocore_clock(monkeypatch): + """Pin botocore's clock so the reference signature is deterministic.""" + monkeypatch.setattr(botauth, "get_current_datetime", lambda: FIXED_NOW) + + +def _botocore_authorization(method, url, payload, headers, region, service, session=None): + creds = Credentials(ACCESS_KEY, SECRET_KEY, session) + request = AWSRequest(method=method, url=url, data=payload, headers=headers) + SigV4Auth(creds, service, region).add_auth(request) + return dict(request.headers)["Authorization"] + + +def _mine_authorization(method, url, payload, headers, region, service, session=None): + creds = AWSCredentials(ACCESS_KEY, SECRET_KEY, session_token=session) + signed = sign_request( + method, + url, + payload, + creds, + region, + service, + headers=headers, + now=FIXED_NOW, + ) + return signed["Authorization"] + + +def test_matches_botocore_get_with_query(): + url = "https://iam.amazonaws.com/?Action=ListUsers&Version=2010-05-08" + headers = {"content-type": "application/x-www-form-urlencoded; charset=utf-8"} + assert _mine_authorization( + "GET", url, b"", headers, "us-east-1", "iam" + ) == _botocore_authorization("GET", url, b"", headers, "us-east-1", "iam") + + +def test_matches_botocore_bedrock_converse_post(): + url = "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-3-5-sonnet-20240620-v1/converse" + payload = b'{"modelId": "x", "messages": [{"role": "user", "content": [{"text": "hi"}]}]}' + headers = {"Content-Type": "application/json", "X-Amz-Target": "BedrockRuntime.Converse"} + assert _mine_authorization( + "POST", url, payload, headers, "us-east-1", "bedrock" + ) == _botocore_authorization("POST", url, payload, headers, "us-east-1", "bedrock") + + +def test_matches_botocore_with_session_token(): + """botocore adds X-Amz-Security-Token automatically; ours must sign it too.""" + url = "https://bedrock-runtime.us-east-1.amazonaws.com/model/m/converse" + payload = b'{"messages": []}' + session = "FwoGZXIvYXdzEBEaD..." + headers = {"content-type": "application/json"} + assert _mine_authorization( + "POST", url, payload, headers, "us-east-1", "bedrock", session=session + ) == ( + _botocore_authorization( + "POST", url, payload, headers, "us-east-1", "bedrock", session=session + ) + ) + + +def test_authorization_header_shape(): + url = "https://bedrock-runtime.us-east-1.amazonaws.com/model/m/converse" + headers = sign_request( + "POST", + url, + b'{"x": 1}', + AWSCredentials(ACCESS_KEY, SECRET_KEY), + "us-east-1", + "bedrock", + now=FIXED_NOW, + ) + auth = headers["Authorization"] + assert auth.startswith( + f"AWS4-HMAC-SHA256 Credential={ACCESS_KEY}/20150830/us-east-1/bedrock/aws4_request, " + ) + assert "SignedHeaders=content-type;host;x-amz-date" in auth + assert "Signature=" in auth + assert headers["host"] == "bedrock-runtime.us-east-1.amazonaws.com" + assert headers["x-amz-date"] == "20150830T123600Z" + + +def test_caller_headers_case_insensitive_no_duplicate(): + """A caller-supplied capitalized Content-Type must not double-sign content-type.""" + url = "https://bedrock-runtime.us-east-1.amazonaws.com/model/m/converse" + headers = sign_request( + "POST", + url, + b"{}", + AWSCredentials(ACCESS_KEY, SECRET_KEY), + "us-east-1", + "bedrock", + headers={"Content-Type": "application/json"}, + now=FIXED_NOW, + ) + signed = headers["Authorization"].split("SignedHeaders=")[1].split(",")[0] + assert signed.count("content-type") == 1 + + +def test_resolve_aws_region_prefers_region_name(monkeypatch): + monkeypatch.setenv("AWS_REGION_NAME", "eu-west-1") + monkeypatch.setenv("AWS_REGION", "us-east-1") + assert resolve_aws_region() == "eu-west-1" + + +def test_resolve_aws_region_falls_back(monkeypatch): + monkeypatch.delenv("AWS_REGION_NAME", raising=False) + monkeypatch.setenv("AWS_REGION", "ap-southeast-2") + assert resolve_aws_region() == "ap-southeast-2" + + +def test_credentials_from_env(monkeypatch): + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKID") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "SECRET") + monkeypatch.setenv("AWS_SESSION_TOKEN", "TOKEN") + creds = AWSCredentials.from_env() + assert creds is not None + assert creds.access_key == "AKID" + assert creds.secret_key == "SECRET" + assert creds.session_token == "TOKEN" + + +def test_credentials_from_env_missing(monkeypatch): + monkeypatch.delenv("AWS_ACCESS_KEY_ID", raising=False) + monkeypatch.delenv("AWS_SECRET_ACCESS_KEY", raising=False) + assert AWSCredentials.from_env() is None diff --git a/tests/helpers/test_llms_hyperscalers.py b/tests/helpers/test_llms_hyperscalers.py new file mode 100644 index 00000000000..a7514ce6994 --- /dev/null +++ b/tests/helpers/test_llms_hyperscalers.py @@ -0,0 +1,306 @@ +"""Offline tests for the Azure / Bedrock / Bedrock Mantle providers. + +No live cloud credentials: request construction (URL, headers, query params, +SigV4-signed payloads) is asserted against fixtures, and the response +normalization is checked against representative provider payloads. +""" + +from __future__ import annotations + +import asyncio + +import cecli.helpers.llms.domains.chat as chat_domain +from cecli.helpers.llms.config import resolve_model_config +from cecli.helpers.llms.domains.bedrock import ( + bedrock_endpoint, + bedrock_payload, + normalize_bedrock_response, +) +from cecli.helpers.llms.providers import get_provider_adapter +from cecli.helpers.llms.providers.azure import AzureProvider +from cecli.helpers.llms.providers.bedrock import BedrockProvider +from cecli.helpers.llms.providers.bedrock_mantle import BedrockMantleProvider + +# --------------------------------------------------------------------------- +# Adapter registration + auth helpers +# --------------------------------------------------------------------------- + + +def test_adapters_auto_registered(): + assert get_provider_adapter("azure").provider == "azure" + assert get_provider_adapter("bedrock").provider == "bedrock" + assert get_provider_adapter("bedrock_mantle").provider == "bedrock_mantle" + + +def test_azure_build_headers_uses_api_key_header(): + adapter = AzureProvider() + headers = adapter.build_headers({}, "sk-azure", "chat", {}) + assert headers["api-key"] == "sk-azure" + assert "Authorization" not in headers + + +def test_azure_resolve_api_key_env(monkeypatch): + monkeypatch.setenv("AZURE_API_KEY", "key1") + assert AzureProvider().resolve_api_key({}, None) == "key1" + + monkeypatch.delenv("AZURE_API_KEY") + monkeypatch.setenv("AZURE_OPENAI_API_KEY", "key2") + assert AzureProvider().resolve_api_key({}, None) == "key2" + + +def test_bedrock_provider_region_substitution(monkeypatch): + monkeypatch.setenv("AWS_REGION_NAME", "eu-west-1") + resolved = {"api_base": "https://bedrock-runtime.{region}.amazonaws.com"} + base = BedrockProvider().resolve_api_base(resolved) + assert base == "https://bedrock-runtime.eu-west-1.amazonaws.com" + assert resolved["aws_region"] == "eu-west-1" + + +def test_bedrock_mantle_region_substitution(): + resolved = {"api_base": "https://bedrock-mantle.{region}.api.aws/v1"} + base = BedrockMantleProvider().resolve_api_base(resolved) + assert base == "https://bedrock-mantle.us-east-1.api.aws/v1" + assert resolved["aws_region"] == "us-east-1" + + +# --------------------------------------------------------------------------- +# Bedrock Converse wire +# --------------------------------------------------------------------------- + + +def test_bedrock_payload_messages_and_tools(): + messages = [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "What is 2+2?"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "call_1", "function": {"name": "add", "arguments": '{"a": 2, "b": 2}'}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "4"}, + {"role": "user", "content": "Thanks!"}, + ] + tools = [ + { + "function": { + "name": "add", + "description": "Add two numbers", + "parameters": {"type": "object"}, + } + } + ] + payload = bedrock_payload( + {"extra_body": {}}, messages, tools, {"max_tokens": 64, "temperature": 0.5} + ) + + assert payload["system"] == [{"text": "You are helpful."}] + assert payload["messages"][0] == {"role": "user", "content": [{"text": "What is 2+2?"}]} + assert payload["messages"][1]["content"][0]["toolUse"]["toolUseId"] == "call_1" + assert payload["messages"][1]["content"][0]["toolUse"]["input"] == {"a": 2, "b": 2} + assert payload["messages"][2]["content"][0]["toolResult"]["toolUseId"] == "call_1" + assert payload["messages"][2]["content"][0]["toolResult"]["content"] == [{"text": "4"}] + assert payload["inferenceConfig"] == {"maxTokens": 64, "temperature": 0.5} + assert payload["toolConfig"]["tools"][0]["toolSpec"]["name"] == "add" + + +def test_bedrock_payload_user_content_list_blocks(): + messages = [{"role": "user", "content": [{"type": "text", "text": "hello"}]}] + payload = bedrock_payload({"extra_body": {}}, messages, None, {}) + assert payload["messages"] == [{"role": "user", "content": [{"text": "hello"}]}] + + +def test_normalize_bedrock_response_text_and_tool_use(): + data = { + "output": { + "message": { + "role": "assistant", + "content": [ + {"text": "Let me compute."}, + {"toolUse": {"toolUseId": "call_1", "name": "add", "input": {"a": 2, "b": 2}}}, + ], + } + }, + "stopReason": "tool_use", + "usage": {"inputTokens": 10, "outputTokens": 5, "totalTokens": 15}, + } + resp = normalize_bedrock_response(data, "bedrock/m") + assert resp.choices[0].finish_reason == "tool_calls" + assert resp.choices[0].message.content == "Let me compute." + assert len(resp.choices[0].message.tool_calls) == 1 + assert resp.choices[0].message.tool_calls[0].name == "add" + assert resp.choices[0].message.tool_calls[0].arguments == {"a": 2, "b": 2} + assert resp.usage.prompt_tokens == 10 + assert resp.usage.completion_tokens == 5 + + +def test_normalize_bedrock_response_stop_reason_mapping(): + resp = normalize_bedrock_response( + {"output": {"message": {"content": [{"text": "done"}]}}, "stopReason": "end_turn"}, + "bedrock/m", + ) + assert resp.choices[0].finish_reason == "stop" + assert resp.choices[0].message.content == "done" + + +def test_bedrock_endpoint_requires_region(): + try: + bedrock_endpoint({"route": "m"}) + except ValueError as exc: + assert "region" in str(exc) + else: + raise AssertionError("expected ValueError without a region") + + +# --------------------------------------------------------------------------- +# Chat-family wiring: signer hook + extra_query +# --------------------------------------------------------------------------- + + +class _FakeResponse: + def raise_for_status(self): + pass + + def json(self): + return { + "id": "x", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "hi"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + + +class _FakeClient: + def __init__(self): + self.calls = [] + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + return None + + async def post(self, url, **kwargs): + self.calls.append({"url": url, **kwargs}) + return _FakeResponse() + + def stream(self, *args, **kwargs): + raise NotImplementedError + + +def _run(coro): + return asyncio.new_event_loop().run_until_complete(coro) + + +def test_chat_domain_passes_extra_query_params(monkeypatch): + client = _FakeClient() + monkeypatch.setattr(chat_domain, "make_client", lambda *a, **k: client) + resolved = { + "api_base": "https://res.openai.azure.com/openai/deployments/gpt-4o", + "model": "azure/m", + "route": "m", + "extra_query": {"api-version": "2024-10-21"}, + } + _run( + chat_domain.chat_complete( + resolved, [{"role": "user", "content": "hi"}], None, "key", {}, {} + ) + ) + call = client.calls[0] + assert call["url"] == "https://res.openai.azure.com/openai/deployments/gpt-4o/chat/completions" + assert call["params"] == {"api-version": "2024-10-21"} + + +def test_chat_domain_invokes_signer(monkeypatch): + client = _FakeClient() + monkeypatch.setattr(chat_domain, "make_client", lambda *a, **k: client) + + def signer(url, payload, headers, key): + return url, {"Authorization": "AWS4-HMAC-SHA256 ..."}, b"{}" + + resolved = { + "api_base": "https://bedrock-mantle.us-east-1.api.aws/v1", + "model": "bedrock_mantle/m", + "route": "m", + "_signer": signer, + } + _run( + chat_domain.chat_complete(resolved, [{"role": "user", "content": "hi"}], None, None, {}, {}) + ) + call = client.calls[0] + assert call["content"] == b"{}" + assert call["headers"]["Authorization"].startswith("AWS4-HMAC-SHA256") + + +def test_pipeline_azure_wires_api_key_and_version(monkeypatch): + import cecli.helpers.llms.pipeline as pipeline + + client = _FakeClient() + monkeypatch.setattr(chat_domain, "make_client", lambda *a, **k: client) + monkeypatch.setenv("AZURE_API_KEY", "sk-azure") + + _run( + pipeline.acompletion( + "azure/deployment-model", + [{"role": "user", "content": "hi"}], + api_base="https://myres.openai.azure.com/openai/deployments/gpt-4o", + ) + ) + call = client.calls[0] + assert ( + call["url"] == "https://myres.openai.azure.com/openai/deployments/gpt-4o/chat/completions" + ) + assert call["headers"]["api-key"] == "sk-azure" + assert "Authorization" not in call["headers"] + assert call["params"] == {"api-version": "2024-10-21"} + + +def test_pipeline_bedrock_mantle_sigv4(monkeypatch): + import cecli.helpers.llms.pipeline as pipeline + + client = _FakeClient() + monkeypatch.setattr(chat_domain, "make_client", lambda *a, **k: client) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKID") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "SECRET") + + _run( + pipeline.acompletion( + "bedrock_mantle/claude-sonnet-4", + [{"role": "user", "content": "hi"}], + api_base="https://bedrock-mantle.us-east-1.api.aws/v1", + ) + ) + call = client.calls[0] + assert call["url"] == "https://bedrock-mantle.us-east-1.api.aws/v1/chat/completions" + assert call["headers"]["Authorization"].startswith("AWS4-HMAC-SHA256") + assert call["content"] is not None # signed body bytes + assert call["headers"]["host"] == "bedrock-mantle.us-east-1.api.aws" + + +def test_pipeline_bedrock_mantle_bearer_when_token_set(monkeypatch): + import cecli.helpers.llms.pipeline as pipeline + + client = _FakeClient() + monkeypatch.setattr(chat_domain, "make_client", lambda *a, **k: client) + monkeypatch.setenv("BEDROCK_MANTLE_API_KEY", "bearer-token") + + _run( + pipeline.acompletion( + "bedrock_mantle/claude-sonnet-4", + [{"role": "user", "content": "hi"}], + api_base="https://bedrock-mantle.us-east-1.api.aws/v1", + ) + ) + call = client.calls[0] + assert call["headers"]["Authorization"] == "Bearer bearer-token" + + +def test_resolve_model_config_surfaces_extra_query(): + resolved = resolve_model_config("azure/sample-model") + assert resolved["extra_query"] == {"api-version": "2024-10-21"} diff --git a/tests/helpers/test_llms_providers_json.py b/tests/helpers/test_llms_providers_json.py index 81b690aad37..b59068bad77 100644 --- a/tests/helpers/test_llms_providers_json.py +++ b/tests/helpers/test_llms_providers_json.py @@ -4,15 +4,12 @@ ships with (base URL + API-key env var). This locks in that each one: - is loaded by ``ModelProviderManager`` (supports_provider == True) -- resolves through ``resolve_model_config`` to the ``chat`` family (the - OpenAI-compatible /chat/completions wire, which is what these providers - advertise); ``chatgpt`` is the exception and uses OpenAI's newer - ``responses`` family - OpenAI-compatible /chat/completions wire, which is what these providers - advertise) +- resolves through ``resolve_model_config`` to the expected API family (``chat`` + for the OpenAI /chat/completions wire; ``responses`` for chatgpt; ``bedrock`` + for AWS Bedrock Converse) - keeps its configured base URL and key env var -- routes through the base OpenAI-style provider adapter (Bearer auth), since - none of them register a custom adapter +- routes through the base OpenAI-style provider adapter (Bearer auth) unless it + registers a dedicated adapter (azure, bedrock, bedrock_mantle) No network: resolution and adapter dispatch are offline. """ @@ -26,6 +23,12 @@ RESOURCE_FILE = "providers.json" +#: Providers whose model-settings default resolves to a non-``chat`` family. +EXPECTED_FAMILIES = { + "chatgpt": "responses", # ChatGPT subscription routes via /v1/responses + "bedrock": "bedrock", # AWS Bedrock Converse wire +} + def _providers_json() -> dict: resource = importlib_resources.files("cecli.resources").joinpath(RESOURCE_FILE) @@ -44,26 +47,25 @@ def test_every_providers_json_entry_is_supported(): assert pcfg.get("api_key_env") == cfg["api_key_env"] -def test_every_provider_resolves_to_chat_family_with_base_and_key(): +def test_every_provider_resolves_to_expected_family_with_base_and_key(): providers = _providers_json() for name, cfg in providers.items(): resolved = resolve_model_config(f"{name}/sample-model") - if name == "chatgpt": # ChatGPT subscription routes via /v1/responses - assert resolved["family"] == "responses", f"{name} should use responses" - else: - assert resolved["family"] == "chat", f"{name} should use chat completions" + expected_family = EXPECTED_FAMILIES.get(name, "chat") + assert resolved["family"] == expected_family, f"{name} should use {expected_family}" assert resolved["provider"] == name assert resolved["api_base"] == cfg["api_base"].rstrip("/") assert resolved["api_key_env"] in cfg["api_key_env"] -def test_unregistered_providers_use_openai_style_base_adapter(): +def test_providers_use_registered_or_openai_style_base_adapter(): + """Every entry routes through its registered adapter, or the base adapter.""" registry_names = set(_providers_json()) - # None of the providers.json entries register a dedicated adapter, so - # dispatch falls back to the base OpenAI-style adapter (Bearer auth). for name in registry_names: adapter = get_provider_adapter(name) assert isinstance(adapter, ProviderAdapter) - assert adapter.provider == "openai" + # Registered adapters advertise their own slug; the base adapter is + # shared by all unregistered OpenAI-compatible providers. + assert adapter.provider in ("openai", name), f"{name} adapter mismatch" From bcdbc9ff0763126d9f6328699902f2a9e4d24e82 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 11 Aug 2026 08:58:40 -0400 Subject: [PATCH 22/30] Make the system httpx version agnostic to match mcp library to have xactly one server stack imported at one time and also use a token estimator instead of tiktoken --- cecli/coders/base_coder.py | 2 +- cecli/helpers/llms/litellm_compat.py | 57 +++++++++--- .../helpers/llms/providers/github_copilot.py | 2 +- cecli/helpers/llms/runtime.py | 2 +- cecli/helpers/llms/utils.py | 2 +- cecli/http.py | 52 +++++++++++ cecli/mcp/server.py | 4 +- cecli/scrape.py | 2 +- requirements.txt | 88 +++---------------- requirements/common-constraints.txt | 41 ++------- requirements/requirements-help.txt | 2 +- requirements/requirements.in | 6 +- 12 files changed, 129 insertions(+), 131 deletions(-) create mode 100644 cecli/http.py diff --git a/cecli/coders/base_coder.py b/cecli/coders/base_coder.py index 25b1bd4c792..9236c4aa733 100755 --- a/cecli/coders/base_coder.py +++ b/cecli/coders/base_coder.py @@ -3060,7 +3060,7 @@ async def _execute_local_tools(self, tool_calls): async def _execute_mcp_tools(self, server, tool_calls): """Execute MCP tools via LiteLLM.""" - import httpx + from cecli.http import httpx tool_responses = [] try: diff --git a/cecli/helpers/llms/litellm_compat.py b/cecli/helpers/llms/litellm_compat.py index b07af409a4f..fba671364ab 100644 --- a/cecli/helpers/llms/litellm_compat.py +++ b/cecli/helpers/llms/litellm_compat.py @@ -20,14 +20,14 @@ import asyncio import json import os +import re import warnings from dataclasses import dataclass, field from types import SimpleNamespace from typing import Any, Dict, List, Optional -import httpx - from cecli.dump import dump # noqa: F401 +from cecli.http import httpx warnings.filterwarnings("ignore", category=UserWarning, module="pydantic") @@ -982,16 +982,27 @@ def model_cost_items(self) -> List[Any]: def encode( self, model: Optional[str] = None, text: Optional[str] = None, **kwargs: Any - ) -> List[int]: - """Tokenize ``text`` with tiktoken (fallback: cl100k_base).""" - import tiktoken + ) -> range: + """Estimate tokens for ``text`` without loading a BPE tokenizer. + + tiktoken's cl100k_base/o200k_base encodings cost ~30-95MB of RSS the + first time they are loaded, so we approximate instead: roughly one token + per four ASCII characters (the classic rule of thumb), plus one token per + non-ASCII character (CJK/emoji tokenize at about one token per char), and + never fewer tokens than there are whitespace-delimited word/punctuation + runs (dense code with short tokens is otherwise undercounted). + + The estimate lands within ~±30% of cl100k_base on code and prose, which + is plenty for context-window checks, cost display, and file-size + warnings. + + Returns a ``range`` whose length is the estimated token count; the + individual values are dummy ids (callers only use ``len()``/iteration). + """ - try: - enc = tiktoken.encoding_for_model(model) - except Exception: - enc = tiktoken.get_encoding("cl100k_base") + n = _estimate_token_count(text or "") - return enc.encode(text or "") + return range(n) def token_counter( self, model: Optional[str] = None, messages: Optional[Any] = None, **kwargs: Any @@ -1117,3 +1128,29 @@ def _load_litellm(self) -> None: litellm = LazyLiteLLM() __all__ = ["litellm"] + + +# --------------------------------------------------------------------------- +# Token-count estimator (tiktoken-free) +# --------------------------------------------------------------------------- + +_WORD_OR_PUNCT_RE = re.compile(r"\w+|[^\w\s]") +_NON_ASCII_RE = re.compile(r"[^\x00-\x7f]") + + +def _estimate_token_count(text: str) -> int: + """Estimate the number of tokens in ``text`` without a BPE tokenizer. + + Uses the classic ~4 chars/token rule of thumb, lifted for non-ASCII text + (CJK/emoji tokenize at roughly one token per character) and floored by the + number of word/punctuation runs so dense code with many short tokens isn't + undercounted. + """ + + if not text: + return 0 + + units = len(_WORD_OR_PUNCT_RE.findall(text)) + non_ascii = len(_NON_ASCII_RE.findall(text)) + + return max(1, len(text) // 4, units) + non_ascii diff --git a/cecli/helpers/llms/providers/github_copilot.py b/cecli/helpers/llms/providers/github_copilot.py index 68bed22cbf0..a5175116dcd 100644 --- a/cecli/helpers/llms/providers/github_copilot.py +++ b/cecli/helpers/llms/providers/github_copilot.py @@ -17,7 +17,7 @@ from typing import Any, Dict, Optional from uuid import uuid4 -import httpx +from cecli.http import httpx from .base import ProviderAdapter diff --git a/cecli/helpers/llms/runtime.py b/cecli/helpers/llms/runtime.py index 81ddac31438..1f6039d324a 100644 --- a/cecli/helpers/llms/runtime.py +++ b/cecli/helpers/llms/runtime.py @@ -10,7 +10,7 @@ import ssl from typing import Any -import httpx +from cecli.http import httpx #: Global TLS verification flag; set False for ``--no-verify-ssl``. VERIFY_SSL = True diff --git a/cecli/helpers/llms/utils.py b/cecli/helpers/llms/utils.py index 7d2b1dee4e0..b69933c32d6 100644 --- a/cecli/helpers/llms/utils.py +++ b/cecli/helpers/llms/utils.py @@ -10,7 +10,7 @@ import re from typing import Any, AsyncIterator, Dict, List, Optional, Tuple -import httpx +from cecli.http import httpx async def sse_json_lines(resp: httpx.Response) -> AsyncIterator[Dict[str, Any]]: diff --git a/cecli/http.py b/cecli/http.py new file mode 100644 index 00000000000..f46029c7fb1 --- /dev/null +++ b/cecli/http.py @@ -0,0 +1,52 @@ +"""Shared HTTP client module selection for cecli. + +mcp SDK 2.x migrated from ``httpx`` to ``httpx2`` (a drop-in fork exposing the +same public API). To keep a single HTTP stack loaded and to keep cecli's +``except httpx.*`` handlers compatible with the exceptions raised by the +installed mcp SDK, :data:`httpx` here is the module the mcp SDK itself uses: + +- mcp SDK >= 2 -> ``httpx2`` (imported and aliased as ``httpx``) +- mcp SDK < 2 -> ``httpx`` + +If the mcp version cannot be determined (or ``httpx2`` is not installed), plain +``httpx`` is used. +""" + +from __future__ import annotations + +import importlib.metadata + + +def _mcp_major_version() -> int: + """Return the installed mcp SDK major version (1, 2, ...).""" + + try: + return int(importlib.metadata.version("mcp").split(".")[0]) + + except Exception: + return 1 + + +def _load_http_client(): + """Select the HTTP client module matching the installed mcp SDK.""" + + if _mcp_major_version() >= 2: + try: + import httpx2 + + return httpx2 + + except ImportError: + pass + + import httpx + + return httpx + + +#: The HTTP client module used by the installed mcp SDK (``httpx2`` on mcp SDK +#: 2.x, ``httpx`` otherwise). Import it as ``httpx`` so cecli code stays +#: provider-agnostic: ``from cecli.http import httpx``. +httpx = _load_http_client() + +__all__ = ["httpx", "_mcp_major_version"] diff --git a/cecli/mcp/server.py b/cecli/mcp/server.py index 79c329fef3a..386db276111 100644 --- a/cecli/mcp/server.py +++ b/cecli/mcp/server.py @@ -7,7 +7,6 @@ from enum import Enum, auto from urllib.parse import urlparse -import httpx from mcp import ClientSession, StdioServerParameters from mcp.client.auth import OAuthClientProvider from mcp.client.sse import sse_client @@ -16,6 +15,7 @@ from mcp.shared.auth import OAuthClientMetadata from cecli.decoding import safe_open +from cecli.http import httpx from .oauth import ( FileBasedTokenStorage, @@ -164,7 +164,7 @@ def is_session_expired_error(exc): Returns: bool: True if the error indicates a 404 session expiry """ - import httpx + from cecli.http import httpx if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code == 404: return True diff --git a/cecli/scrape.py b/cecli/scrape.py index 390e3e9b295..e048f20bc04 100755 --- a/cecli/scrape.py +++ b/cecli/scrape.py @@ -203,7 +203,7 @@ async def scrape_with_playwright(self, url): return content, mime_type def scrape_with_httpx(self, url): - import httpx + from cecli.http import httpx headers = {"User-Agent": f"Mozilla./5.0 ({coder_user_agent})"} try: diff --git a/requirements.txt b/requirements.txt index d1073e4095a..6fc58bd5299 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,6 @@ anyio==4.11.0 # -c requirements/common-constraints.txt # httpx # mcp - # openai # sse-starlette # starlette # watchfiles @@ -46,16 +45,7 @@ charset-normalizer==3.4.9 click==8.3.1 # via # -c requirements/common-constraints.txt - # click-default-group - # llm - # sqlite-utils # uvicorn -click-default-group==1.2.4 - # via - # llm - # sqlite-utils -condense-json==1.1 - # via llm configargparse==1.7.1 # via # -c requirements/common-constraints.txt @@ -73,10 +63,6 @@ diskcache==5.6.3 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in -distro==1.9.0 - # via - # -c requirements/common-constraints.txt - # openai gitdb==4.0.12 # via # -c requirements/common-constraints.txt @@ -98,7 +84,6 @@ httpx==0.28.1 # via # -c requirements/common-constraints.txt # mcp - # openai httpx-sse==0.4.3 # via # -c requirements/common-constraints.txt @@ -117,10 +102,6 @@ importlib-resources==6.5.2 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in -jiter==0.12.0 - # via - # -c requirements/common-constraints.txt - # openai json-repair==0.60.1 # via # -c requirements/common-constraints.txt @@ -138,8 +119,6 @@ linkify-it-py==2.0.3 # via # -c requirements/common-constraints.txt # markdown-it-py -llm==0.32 - # via -r requirements/requirements.in marisa-trie==1.4.1 # via # -c requirements/common-constraints.txt @@ -175,10 +154,6 @@ numpy==2.3.5 # -c requirements/common-constraints.txt # rustworkx # soundfile -openai==2.53.0 - # via - # -c requirements/common-constraints.txt - # llm orjson==3.11.9 # via # -c requirements/common-constraints.txt @@ -203,20 +178,10 @@ pillow==12.0.0 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in -pip==25.3 - # via - # -c requirements/common-constraints.txt - # llm - # sqlite-utils platformdirs==4.5.0 # via # -c requirements/common-constraints.txt # textual -pluggy==1.6.0 - # via - # -c requirements/common-constraints.txt - # llm - # sqlite-utils prompt-toolkit==3.0.52 # via # -c requirements/common-constraints.txt @@ -229,8 +194,6 @@ ptyprocess==0.7.0 # via # -c requirements/common-constraints.txt # pexpect -puremagic==2.2.0 - # via llm py-cymbal==0.2.1 # via # -c requirements/common-constraints.txt @@ -242,9 +205,7 @@ pycparser==2.23 pydantic==2.12.4 # via # -c requirements/common-constraints.txt - # llm # mcp - # openai # pydantic-settings pydantic-core==2.41.5 # via @@ -275,10 +236,6 @@ pyperclip==1.11.0 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in -python-dateutil==2.9.0.post0 - # via - # -c requirements/common-constraints.txt - # sqlite-utils python-dotenv==1.2.2 # via # -c requirements/common-constraints.txt @@ -287,13 +244,10 @@ python-multipart==0.0.20 # via # -c requirements/common-constraints.txt # mcp -python-ulid==4.0.1 - # via llm pyyaml==6.0.3 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in - # llm rapidfuzz==3.14.5 # via # -c requirements/common-constraints.txt @@ -303,14 +257,10 @@ referencing==0.37.0 # -c requirements/common-constraints.txt # jsonschema # jsonschema-specifications -regex==2025.11.3 - # via - # -c requirements/common-constraints.txt - # tiktoken requests==2.32.5 # via # -c requirements/common-constraints.txt - # tiktoken + # -r requirements/requirements.in rich==14.2.0 # via # -c requirements/common-constraints.txt @@ -325,18 +275,10 @@ rustworkx==0.17.1 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in -setuptools==80.9.0 - # via - # -c requirements/common-constraints.txt - # llm shtab==1.8.0 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in -six==1.17.0 - # via - # -c requirements/common-constraints.txt - # python-dateutil smmap==5.0.2 # via # -c requirements/common-constraints.txt @@ -345,7 +287,6 @@ sniffio==1.3.1 # via # -c requirements/common-constraints.txt # anyio - # openai socksio==1.0.0 # via # -c requirements/common-constraints.txt @@ -362,10 +303,6 @@ soupsieve==2.8 # via # -c requirements/common-constraints.txt # beautifulsoup4 -sqlite-fts4==1.0.3 - # via sqlite-utils -sqlite-utils==4.1.1 - # via llm sse-starlette==3.0.3 # via # -c requirements/common-constraints.txt @@ -374,29 +311,19 @@ starlette==0.50.0 # via # -c requirements/common-constraints.txt # mcp -tabulate==0.10.0 - # via sqlite-utils textual==8.2.8 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in -tiktoken==0.13.0 - # via - # -c requirements/common-constraints.txt - # -r requirements/requirements.in tomlkit==0.14.0 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in -tqdm==4.67.1 - # via - # -c requirements/common-constraints.txt - # openai -tree-sitter==0.25.2 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in # tree-sitter-language-pack + # tree-sitter-languages tree-sitter-c-sharp==0.23.5 # via # -c requirements/common-constraints.txt @@ -409,6 +336,10 @@ tree-sitter-language-pack==0.13.0 # via # -c requirements/common-constraints.txt # -r requirements/requirements.in +tree-sitter-languages==1.10.2 + # via + # -c requirements/common-constraints.txt + # -r requirements/requirements.in tree-sitter-yaml==0.7.2 # via # -c requirements/common-constraints.txt @@ -420,11 +351,13 @@ truststore==0.10.4 typing-extensions==4.15.0 # via # -c requirements/common-constraints.txt + # anyio # beautifulsoup4 # mcp - # openai # pydantic # pydantic-core + # referencing + # starlette # textual # typing-inspection typing-inspection==0.4.2 @@ -465,3 +398,6 @@ zipp==3.23.0 # via # -c requirements/common-constraints.txt # importlib-metadata + +tree-sitter==0.23.2; python_version < "3.10" +tree-sitter>=0.25.1; python_version >= "3.10" diff --git a/requirements/common-constraints.txt b/requirements/common-constraints.txt index bad8d696db4..fbcbe7aff34 100644 --- a/requirements/common-constraints.txt +++ b/requirements/common-constraints.txt @@ -5,7 +5,6 @@ aiohappyeyeballs==2.6.1 aiohttp==3.13.2 # via # huggingface-hub - # litellm # llama-index-core aiosignal==1.4.0 # via aiohttp @@ -17,7 +16,6 @@ anyio==4.11.0 # via # httpx # mcp - # openai # sse-starlette # starlette # watchfiles @@ -52,7 +50,6 @@ charset-normalizer==3.4.9 # requests click==8.3.1 # via - # litellm # nltk # pip-tools # typer @@ -88,10 +85,6 @@ diskcache==5.6.3 # via -r requirements/requirements.in distlib==0.4.0 # via virtualenv -distro==1.9.0 - # via openai -fastuuid==0.14.0 - # via litellm filelock==3.20.0 # via # huggingface-hub @@ -135,10 +128,8 @@ httpcore==1.0.9 # via httpx httpx==0.28.1 # via - # litellm # llama-index-core # mcp - # openai httpx-sse==0.4.3 # via mcp huggingface-hub[inference]==0.36.0 @@ -158,9 +149,7 @@ idna==3.11 imgcat==0.6.0 # via -r requirements/requirements-dev.in importlib-metadata==8.7.0 - # via - # -r requirements/requirements.in - # litellm + # via -r requirements/requirements.in importlib-resources==6.5.2 # via -r requirements/requirements.in iniconfig==2.3.0 @@ -168,11 +157,8 @@ iniconfig==2.3.0 jinja2==3.1.6 # via # banks - # litellm # memray # torch -jiter==0.12.0 - # via openai joblib==1.5.2 # via # nltk @@ -182,7 +168,6 @@ json-repair==0.60.1 jsonschema==4.25.1 # via # -r requirements/requirements.in - # litellm # mcp jsonschema-specifications==2025.9.1 # via jsonschema @@ -190,8 +175,6 @@ kiwisolver==1.4.9 # via matplotlib linkify-it-py==2.0.3 # via markdown-it-py -litellm==1.81.11 - # via -r requirements/requirements.in llama-index-core==0.14.8 # via llama-index-embeddings-huggingface llama-index-embeddings-huggingface==0.6.1 @@ -300,8 +283,6 @@ nvidia-nvtx-cu12==12.8.90 # via torch objgraph==3.6.2 # via -r requirements/requirements-dev.in -openai>=2.32.0 - # via litellm orjson==3.11.9 # via -r requirements/requirements.in oslex==0.1.3 @@ -364,12 +345,10 @@ pycparser==2.23 pydantic==2.12.4 # via # banks - # litellm # llama-index-core # llama-index-instrumentation # llama-index-workflows # mcp - # openai # pydantic-settings pydantic-core==2.41.5 # via pydantic @@ -418,7 +397,6 @@ python-dateutil==2.9.0.post0 # pandas python-dotenv==1.2.2 # via - # litellm # pydantic-settings # pytest-env python-multipart==0.0.20 @@ -445,6 +423,7 @@ regex==2025.11.3 # transformers requests==2.32.5 # via + # -r requirements/requirements.in # huggingface-hub # llama-index-core # tiktoken @@ -487,9 +466,7 @@ six==1.17.0 smmap==5.0.2 # via gitdb sniffio==1.3.1 - # via - # anyio - # openai + # via anyio socksio==1.0.0 # via -r requirements/requirements.in sounddevice==0.5.3 @@ -514,14 +491,10 @@ textual==8.2.8 # memray threadpoolctl==3.6.0 # via scikit-learn -tiktoken>=0.13.0 - # via - # litellm - # llama-index-core +tiktoken==0.13.0 + # via llama-index-core tokenizers==0.22.1 - # via - # litellm - # transformers + # via transformers tomlkit==0.14.0 # via -r requirements/requirements.in torch==2.9.1 @@ -531,7 +504,6 @@ tqdm==4.67.1 # huggingface-hub # llama-index-core # nltk - # openai # sentence-transformers # transformers transformers==4.57.2 @@ -567,7 +539,6 @@ typing-extensions==4.15.0 # llama-index-core # llama-index-workflows # mcp - # openai # pydantic # pydantic-core # pyee diff --git a/requirements/requirements-help.txt b/requirements/requirements-help.txt index 9ecb75a35f5..16b5f2c6373 100644 --- a/requirements/requirements-help.txt +++ b/requirements/requirements-help.txt @@ -353,7 +353,7 @@ threadpoolctl==3.6.0 # via # -c requirements/common-constraints.txt # scikit-learn -tiktoken==0.12.0 +tiktoken==0.13.0 # via # -c requirements/common-constraints.txt # llama-index-core diff --git a/requirements/requirements.in b/requirements/requirements.in index ac3f3a49dfa..ec7b55f0a2e 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -13,8 +13,10 @@ GitPython>=3.1.45 pathspec>=0.12.1 # communication -llm>=0.32 -tiktoken>=0.13.0 +# token counts are estimated (no BPE tokenizer); tiktoken was ~30-95MB RSS +# NOTE: requests is imported directly by cecli (models/onboarding/versioncheck); +# it used to come in transitively via tiktoken, so declare it explicitly now. +requests>=2.32.0 mcp>=1.24.0 socksio>=1.0.0 truststore From 040092f96ff6c480baaf847ccc01a8e35d2ed2a0 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 11 Aug 2026 08:59:21 -0400 Subject: [PATCH 23/30] Bump Version --- cecli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cecli/__init__.py b/cecli/__init__.py index 29a2d797be6..3536c41b24f 100644 --- a/cecli/__init__.py +++ b/cecli/__init__.py @@ -1,6 +1,6 @@ from packaging import version -__version__ = "1.0.5.dev" +__version__ = "1.2.0.dev" safe_version = __version__ try: From 8f70dfee84ec87a5bba5250eacff6c27fca11f10 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 11 Aug 2026 15:39:23 -0400 Subject: [PATCH 24/30] Fix sig v4 tests --- tests/helpers/test_llms_aws_sigv4.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/helpers/test_llms_aws_sigv4.py b/tests/helpers/test_llms_aws_sigv4.py index acb70ca4ab8..b51ddedb0c4 100644 --- a/tests/helpers/test_llms_aws_sigv4.py +++ b/tests/helpers/test_llms_aws_sigv4.py @@ -2,18 +2,15 @@ The implementation is verified for parity against botocore's own ``SigV4Auth`` (Amazon's reference SDK implementation) with a pinned clock, so the tests are -meaningful without any live AWS credentials. +meaningful without any live AWS credentials. botocore is only used as a test +oracle; the parity tests are skipped when it is not installed. """ from __future__ import annotations from datetime import datetime -import botocore.auth as botauth import pytest -from botocore.auth import SigV4Auth -from botocore.awsrequest import AWSRequest -from botocore.credentials import Credentials from cecli.helpers.llms.aws_sigv4 import ( AWSCredentials, @@ -29,10 +26,20 @@ @pytest.fixture(autouse=True) def _pin_botocore_clock(monkeypatch): """Pin botocore's clock so the reference signature is deterministic.""" + try: + import botocore.auth as botauth + except ImportError: + return + monkeypatch.setattr(botauth, "get_current_datetime", lambda: FIXED_NOW) def _botocore_authorization(method, url, payload, headers, region, service, session=None): + pytest.importorskip("botocore") + from botocore.auth import SigV4Auth + from botocore.awsrequest import AWSRequest + from botocore.credentials import Credentials + creds = Credentials(ACCESS_KEY, SECRET_KEY, session) request = AWSRequest(method=method, url=url, data=payload, headers=headers) SigV4Auth(creds, service, region).add_auth(request) From b4c0aba99acad0472ab6b9d5a4783ed8364846f3 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 11 Aug 2026 20:24:32 -0400 Subject: [PATCH 25/30] Actuaaly import default httpx as fallback --- cecli/mcp/server.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cecli/mcp/server.py b/cecli/mcp/server.py index 386db276111..f96fed7baef 100644 --- a/cecli/mcp/server.py +++ b/cecli/mcp/server.py @@ -541,12 +541,18 @@ def _get_http_client_module(): """Return the HTTP client module used by the installed mcp SDK. mcp SDK 2.x migrated from httpx to httpx2; earlier versions use httpx. + + Note: ``cecli.http.httpx`` aliases ``httpx2`` when mcp SDK 2.x is + installed, so import the real module here instead of relying on the + module-level ``httpx`` name (which may be the httpx2 alias). """ if _get_mcp_major_version() >= 2: import httpx2 return httpx2 + import httpx + return httpx From a2512b110d8d20a984370faa0b22049cda934d16 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 11 Aug 2026 20:40:52 -0400 Subject: [PATCH 26/30] Add dotenv as dependency explicitly --- requirements.txt | 1 + requirements/common-constraints.txt | 1 + requirements/requirements.in | 1 + 3 files changed, 3 insertions(+) diff --git a/requirements.txt b/requirements.txt index 6fc58bd5299..6cca5ff1bdd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -239,6 +239,7 @@ pyperclip==1.11.0 python-dotenv==1.2.2 # via # -c requirements/common-constraints.txt + # -r requirements/requirements.in # pydantic-settings python-multipart==0.0.20 # via diff --git a/requirements/common-constraints.txt b/requirements/common-constraints.txt index fbcbe7aff34..62c222fe470 100644 --- a/requirements/common-constraints.txt +++ b/requirements/common-constraints.txt @@ -397,6 +397,7 @@ python-dateutil==2.9.0.post0 # pandas python-dotenv==1.2.2 # via + # -r requirements/requirements.in # pydantic-settings # pytest-env python-multipart==0.0.20 diff --git a/requirements/requirements.in b/requirements/requirements.in index ec7b55f0a2e..284f3914e7d 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -1,6 +1,7 @@ # configuration configargparse>=1.7.1 shtab>=1.7.2 +python-dotenv>=1.0.1 # operating system integration oslex>=0.1.3 From 0ec401bb5da4c187303eb79ac81920c316681b9c Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 11 Aug 2026 20:54:11 -0400 Subject: [PATCH 27/30] Add guard for empty grep parameters --- cecli/helpers/hashline.py | 2 ++ cecli/helpers/hashpos/hashpos.py | 2 ++ cecli/tools/grep.py | 7 ++++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/cecli/helpers/hashline.py b/cecli/helpers/hashline.py index f344332b40c..5b9918f6ba9 100644 --- a/cecli/helpers/hashline.py +++ b/cecli/helpers/hashline.py @@ -73,6 +73,8 @@ def strip_hashline(text: str) -> str: """ Remove HashPos prefixes from the start of every line. """ + if text is None: + return "" return HashPos.strip_prefix(text) diff --git a/cecli/helpers/hashpos/hashpos.py b/cecli/helpers/hashpos/hashpos.py index 7d62e9a3992..6c0d2cb1664 100644 --- a/cecli/helpers/hashpos/hashpos.py +++ b/cecli/helpers/hashpos/hashpos.py @@ -206,6 +206,8 @@ def get_wrapped_id(public_id: str) -> str: @staticmethod def strip_prefix(text: str) -> str: + if text is None: + return "" lines = text.splitlines(keepends=True) result_lines = [] for line in lines: diff --git a/cecli/tools/grep.py b/cecli/tools/grep.py index 4a3c1d66ebe..7c3e26059db 100644 --- a/cecli/tools/grep.py +++ b/cecli/tools/grep.py @@ -352,7 +352,7 @@ def execute( all_operation_results = [] for search_op in searches: - pattern = strip_hashline(search_op.get("pattern")) + pattern = strip_hashline(search_op.get("pattern", "")) file_pattern = search_op.get("file_glob", "*") directory = search_op.get("directory", search_op.get("path", ".")) use_regex = search_op.get("use_regex", False) @@ -377,6 +377,11 @@ def execute( "files": [], } + if not pattern: + op_result["error"] = "Search operation requires a non-empty 'pattern'." + all_operation_results.append(op_result) + continue + try: search_dir_path = Path(repo.root) / directory From 043583ba7c546232b2df372487889e23a8066075 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 11 Aug 2026 21:00:12 -0400 Subject: [PATCH 28/30] Prevent CI test hang --- tests/unit/test_ws_server.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/tests/unit/test_ws_server.py b/tests/unit/test_ws_server.py index d17638a110f..7da38172cc9 100644 --- a/tests/unit/test_ws_server.py +++ b/tests/unit/test_ws_server.py @@ -170,19 +170,23 @@ async def test_multiple_clients_receive_broadcast(self): uri = f"ws://{bridge.host}:{bridge.port}" async def connect_and_get(): - ws = await websockets.connect(uri) - await asyncio.sleep(0.1) # Let connection register + # Use an explicit, generous open timeout: slow CI runners can + # occasionally exceed websockets' 10s default handshake timeout. + ws = await websockets.connect(uri, open_timeout=15.0) return ws ws1 = await connect_and_get() ws2 = await connect_and_get() - await asyncio.sleep(0.1) + # Wait until the server has registered both connections so the + # broadcast below cannot race connection registration. + await wait_for_registered_clients(bridge, count=2, timeout=5.0) await bridge._broadcast("tool_output", text="broadcast", coder_uuid="coder-1") - await asyncio.sleep(0.1) - msg1 = json.loads(await ws1.recv()) - msg2 = json.loads(await ws2.recv()) + # Bound the receives: a missed broadcast should fail fast with a + # clear TimeoutError instead of hanging the whole CI job. + msg1 = json.loads(await asyncio.wait_for(ws1.recv(), timeout=5.0)) + msg2 = json.loads(await asyncio.wait_for(ws2.recv(), timeout=5.0)) await ws1.close() await ws2.close() @@ -374,3 +378,15 @@ async def connect_and_collect(bridge, broadcast_coro): await asyncio.sleep(0.1) msg = await ws.recv() return json.loads(msg) + + +async def wait_for_registered_clients(bridge, count: int, timeout: float) -> None: + """Wait until the bridge has registered at least ``count`` connected clients.""" + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + while len(bridge._connections) < count: + if loop.time() >= deadline: + raise TimeoutError( + f"bridge registered {len(bridge._connections)}/{count} clients within {timeout}s" + ) + await asyncio.sleep(0.01) From 2d0584309ad95ff463fb70873447fbda3ea0131a Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 11 Aug 2026 21:32:07 -0400 Subject: [PATCH 29/30] fix: deep-merge CLI yaml-to-json args with config-file values Fold the agent-config deep-merge from PR #636 into convert_yaml_to_json_string(value, config_file_value=None) so --agent-config isn't parsed differently from the other yaml-to-json args. All such args (agent_config, tui_config, mcp_servers, custom, security_config, retries, hooks, workspaces, model_providers, server_config) now deep-merge with the value for their hyphenated config-file key (CLI wins per-key, file-only keys preserved; underscore-key variants unsupported). Parsing now uses json.loads first so JSON booleans/null parse correctly before falling back to ast.literal_eval for Python-literal forms. --- cecli/main.py | 125 +++++++++++++------------ tests/basic/test_agent_config_merge.py | 61 +++++++++--- 2 files changed, 117 insertions(+), 69 deletions(-) diff --git a/cecli/main.py b/cecli/main.py index e2861aab959..3556016c9a3 100644 --- a/cecli/main.py +++ b/cecli/main.py @@ -73,61 +73,83 @@ def new_event_loop(self): from .dump import dump # noqa -def convert_yaml_to_json_string(value): +def convert_yaml_to_json_string(value, config_file_value=None): """ Convert YAML dict/list values to JSON strings for compatibility. configargparse.YAMLConfigFileParser converts YAML to Python objects, but some arguments expect JSON strings. This function handles: - Direct dict/list objects - - String representations of dicts/lists (Python literals) + - String representations of dicts/lists (JSON or Python literals) - Already JSON strings (passed through unchanged) + When config_file_value is provided (the value for the same option from the + merged config files), the CLI value is deep-merged on top of it so CLI keys + win per-key while keys provided only by the config files (e.g. + skills_paths, skills_init) are preserved instead of being discarded + wholesale. configargparse discards config-file values for options that are + also given on the command line, so without this merge a CLI --agent-config + silently drops every agent-config key that lives only in .cecli.conf.yml. + Args: value: The value to convert + config_file_value: Optional config-file value to deep-merge underneath + the CLI value Returns: str: JSON string if value is a dict/list, otherwise the original value """ if value is None: return None - if isinstance(value, (dict, list)): - return json.dumps(value) + + parsed = value + if isinstance(value, str): try: - import ast + parsed = json.loads(value) + except (json.JSONDecodeError, TypeError): + try: + import ast + + parsed = ast.literal_eval(value) + except (SyntaxError, ValueError): + return value + + if isinstance(parsed, (dict, list)): + if isinstance(parsed, dict) and config_file_value is not None: + from cecli.helpers.config_utils import deep_merge + + try: + file_value = config_file_value + + if isinstance(file_value, str): + file_value = json.loads(file_value) + + if isinstance(file_value, dict) and file_value: + parsed = deep_merge(file_value, parsed, deep_merge_arrays=False) + except Exception: + pass + + return json.dumps(parsed) - parsed = ast.literal_eval(value) - if isinstance(parsed, (dict, list)): - return json.dumps(parsed) - except (SyntaxError, ValueError): - pass return value -def merge_agent_config(cli_agent_config: str, file_agent_config) -> str: - """ - Deep-merge the config-file agent-config into the CLI agent-config so CLI - values override individual keys while keys provided only by the config - files (e.g. skills_paths, skills_init) are preserved instead of being - discarded wholesale when --agent-config is passed on the CLI. - - configargparse discards config-file values for options that are also given - on the command line, so without this merge a CLI --agent-config silently - drops every agent-config key that lives only in .cecli.conf.yml. - """ - try: - from cecli.helpers.config_utils import deep_merge - - file_ac = file_agent_config - if isinstance(file_ac, str): - file_ac = json.loads(file_ac) - cli_ac = json.loads(cli_agent_config) - if isinstance(file_ac, dict) and file_ac and isinstance(cli_ac, dict): - return json.dumps(deep_merge(file_ac, cli_ac, deep_merge_arrays=False)) - except Exception: - pass - return cli_agent_config +# yaml-to-json args: argparse dest -> config-file key (hyphenated). +# CLI values for these args deep-merge with the merged config-file value for +# the same option, so CLI keys win per-key while file-only keys are preserved. +YAML_TO_JSON_ARG_KEYS = { + "agent_config": "agent-config", + "tui_config": "tui-config", + "mcp_servers": "mcp-servers", + "custom": "custom", + "security_config": "security-config", + "retries": "retries", + "hooks": "hooks", + "workspaces": "workspaces", + "model_providers": "model-providers", + "server_config": "server-config", +} def check_config_files_for_yes(config_files): @@ -754,31 +776,18 @@ async def main_async( if len(unknown): print("Unknown Args: ", unknown) - if hasattr(args, "agent_config") and args.agent_config is not None: - args.agent_config = convert_yaml_to_json_string(args.agent_config) - # CLI --agent-config should deep-merge with (not replace) the - # agent-config from the merged config files so file-only keys - # (e.g. skills_paths, skills_init) are preserved. - file_agent_config = merged_config.get("agent-config") or merged_config.get("agent_config") - args.agent_config = merge_agent_config(args.agent_config, file_agent_config) - if hasattr(args, "tui_config") and args.tui_config is not None: - args.tui_config = convert_yaml_to_json_string(args.tui_config) - if hasattr(args, "mcp_servers") and args.mcp_servers is not None: - args.mcp_servers = convert_yaml_to_json_string(args.mcp_servers) - if hasattr(args, "custom") and args.custom is not None: - args.custom = convert_yaml_to_json_string(args.custom) - if hasattr(args, "security_config") and args.security_config is not None: - args.security_config = convert_yaml_to_json_string(args.security_config) - if hasattr(args, "retries") and args.retries is not None: - args.retries = convert_yaml_to_json_string(args.retries) - if hasattr(args, "hooks") and args.hooks is not None: - args.hooks = convert_yaml_to_json_string(args.hooks) - if hasattr(args, "workspaces") and args.workspaces is not None: - args.workspaces = convert_yaml_to_json_string(args.workspaces) - if hasattr(args, "model_providers") and args.model_providers is not None: - args.model_providers = convert_yaml_to_json_string(args.model_providers) - if hasattr(args, "server_config") and args.server_config is not None: - args.server_config = convert_yaml_to_json_string(args.server_config) + # ── Convert yaml-to-json arguments, deep-merging CLI values with ────── + # the merged config-file values so CLI keys win per-key while file-only + # keys are preserved. + for arg_name, config_key in YAML_TO_JSON_ARG_KEYS.items(): + if hasattr(args, arg_name) and getattr(args, arg_name) is not None: + config_file_value = merged_config.get(config_key) + + setattr( + args, + arg_name, + convert_yaml_to_json_string(getattr(args, arg_name), config_file_value), + ) # Interpolate environment variables in all string arguments for key, value in vars(args).items(): diff --git a/tests/basic/test_agent_config_merge.py b/tests/basic/test_agent_config_merge.py index 8b9e3d9b58d..1bfcd6cacf5 100644 --- a/tests/basic/test_agent_config_merge.py +++ b/tests/basic/test_agent_config_merge.py @@ -6,7 +6,7 @@ from cecli.args import get_parser from cecli.helpers import config_utils -from cecli.main import convert_yaml_to_json_string, merge_agent_config +from cecli.main import YAML_TO_JSON_ARG_KEYS, convert_yaml_to_json_string def test_cli_agent_config_merges_with_config_file(): @@ -19,7 +19,7 @@ def test_cli_agent_config_merges_with_config_file(): "skills_paths": ["/tmp/skills"], } cli = json.dumps({"command_timeout": 120}) - merged = json.loads(merge_agent_config(cli, file_ac)) + merged = json.loads(convert_yaml_to_json_string(cli, file_ac)) assert merged["command_timeout"] == 120 # CLI wins per-key assert merged["skip_cli_confirmations"] is True # file-only key preserved assert merged["tools_paths"] == ["/tmp/mytools"] # file-only key preserved @@ -30,7 +30,7 @@ def test_file_agent_config_given_as_json_string(): """Merged config from read_and_merge_all_configs holds agent-config as a JSON string (YAML block scalar) - the helper must handle that form.""" file_ac = '{"command_timeout": 0, "skills_paths": ["/tmp/skills"]}' - merged = json.loads(merge_agent_config('{"skip_cli_confirmations": true}', file_ac)) + merged = json.loads(convert_yaml_to_json_string('{"skip_cli_confirmations": true}', file_ac)) assert merged["skip_cli_confirmations"] is True assert merged["command_timeout"] == 0 assert merged["skills_paths"] == ["/tmp/skills"] @@ -39,22 +39,24 @@ def test_file_agent_config_given_as_json_string(): def test_no_file_agent_config_returns_cli_unchanged(): """Without a config-file agent-config the merge must be a no-op.""" cli = '{"command_timeout": 120}' - assert merge_agent_config(cli, None) == cli - assert merge_agent_config(cli, {}) == cli - assert merge_agent_config(cli, "not json") == cli + assert convert_yaml_to_json_string(cli, None) == cli + assert convert_yaml_to_json_string(cli, {}) == cli + assert convert_yaml_to_json_string(cli, "not json") == cli def test_nested_dict_keys_deep_merged(): """Nested dicts under agent-config are merged recursively, CLI wins.""" file_ac = {"nested": {"a": 1, "b": 2}} - merged = json.loads(merge_agent_config('{"nested": {"b": 9, "c": 3}}', file_ac)) + merged = json.loads(convert_yaml_to_json_string('{"nested": {"b": 9, "c": 3}}', file_ac)) assert merged["nested"] == {"a": 1, "b": 9, "c": 3} def test_cli_array_replaces_file_array(): """Arrays are not deep-merged: CLI list values win wholesale.""" file_ac = {"skills_paths": ["/tmp/file-skills"]} - merged = json.loads(merge_agent_config('{"skills_paths": ["/tmp/cli-skills"]}', file_ac)) + merged = json.loads( + convert_yaml_to_json_string('{"skills_paths": ["/tmp/cli-skills"]}', file_ac) + ) assert merged["skills_paths"] == ["/tmp/cli-skills"] @@ -84,11 +86,48 @@ def test_main_async_pipeline_preserves_file_keys(tmp_path): finally: os.unlink(tmp) # main_async deletes the temp file before the merge point - args.agent_config = convert_yaml_to_json_string(args.agent_config) - file_ac = merged_config.get("agent-config") - args.agent_config = merge_agent_config(args.agent_config, file_ac) + config_key = YAML_TO_JSON_ARG_KEYS["agent_config"] + file_ac = merged_config.get(config_key) + args.agent_config = convert_yaml_to_json_string(args.agent_config, file_ac) merged = json.loads(args.agent_config) assert merged["command_timeout"] == 0 assert merged["skills_paths"] == ["./.cecli/skills"] assert merged["skills_init"] == ["android-cli"] + + +def test_all_yaml_to_json_args_deep_merge_with_config_file(): + """Every yaml-to-json arg converted in main_async deep-merges with the + value for its hyphenated config-file key (CLI wins per-key, file-only + keys preserved, no underscore-key variants).""" + assert set(YAML_TO_JSON_ARG_KEYS) == { + "agent_config", + "tui_config", + "mcp_servers", + "custom", + "security_config", + "retries", + "hooks", + "workspaces", + "model_providers", + "server_config", + } + assert all("_" not in key for key in YAML_TO_JSON_ARG_KEYS.values()) + + file_value = { + "command_timeout": 0, + "skills_paths": ["/tmp/skills"], + "nested": {"a": 1, "b": 2}, + } + cli_value = {"command_timeout": 120, "nested": {"b": 9}} + + for arg_name, config_key in YAML_TO_JSON_ARG_KEYS.items(): + merged_config = {config_key: file_value} + merged_arg = convert_yaml_to_json_string( + json.dumps(cli_value), merged_config.get(config_key) + ) + + merged = json.loads(merged_arg) + assert merged["command_timeout"] == 120 # CLI wins per-key + assert merged["skills_paths"] == ["/tmp/skills"] # file-only key preserved + assert merged["nested"] == {"a": 1, "b": 9} # nested deep-merged From c5b7f128816323f57a3125e90789d5606e499570 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 11 Aug 2026 22:20:00 -0400 Subject: [PATCH 30/30] Move prompt queue advancement into run_one() --- cecli/coders/base_coder.py | 73 +++++++++----------------------------- 1 file changed, 16 insertions(+), 57 deletions(-) diff --git a/cecli/coders/base_coder.py b/cecli/coders/base_coder.py index 6062ffdff59..5a4c6c6cce2 100755 --- a/cecli/coders/base_coder.py +++ b/cecli/coders/base_coder.py @@ -35,10 +35,10 @@ import cecli.prompts.utils.system as prompts from cecli import __version__, models, urls, utils -from cecli.commands import Commands, ReloadProgramSignal, SwitchCoderSignal +from cecli.commands import Commands, SwitchCoderSignal from cecli.decoding import safe_open from cecli.exceptions import LiteLLMExceptions -from cecli.helpers import command_parser, coroutines, nested, responses +from cecli.helpers import command_parser, command_queue, coroutines, nested, responses from cecli.helpers.conversation import ConversationService, MessageTag from cecli.helpers.file_system import FileSystemService from cecli.helpers.io_proxy import IOProxy @@ -1527,16 +1527,6 @@ async def _run_linear(self, with_message=None, preproc=True): await self.commands.cmd_running_event.wait() continue - # Process any queued prompts (CLI-33) before waiting for input - if self.prompt_queue and not self._processing_queue: - self._processing_queue = True - try: - processed = await self._process_next_queued_prompt(preproc) - finally: - self._processing_queue = False - if processed: - continue - if not self.suppress_announcements_for_next_prompt: self.show_announcements() self.suppress_announcements_for_next_prompt = True @@ -1726,22 +1716,6 @@ async def output_task(self, preproc): await self.commands.cmd_running_event.wait() continue - # Process any queued prompts (CLI-33) while idle - if ( - self.prompt_queue - and not self._processing_queue - and not self.user_message - and not coroutines.is_active(self.io.output_task) - ): - self._processing_queue = True - try: - processed = await self._process_next_queued_prompt(preproc) - finally: - self._processing_queue = False - if processed: - await self.auto_save_session() - continue - # Check if we have a user message to process if self.user_message and not self.io.get_confirmation_acknowledgement(): user_message = self.user_message @@ -1839,35 +1813,6 @@ async def generate(self, user_message, preproc): # Trim memory in the background so it doesn't stall the event loop coroutines.fire_and_forget(asyncio.to_thread(trim_memory)) - async def _process_next_queued_prompt(self, preproc: bool) -> bool: - """Pop and run the next queued prompt (CLI-33). - - Dequeues the next prompt from this coder's ``prompt_queue`` (FIFO) and - runs it through the normal generation path. Returns True when a queued - prompt was processed so the caller's loop can continue draining the - queue. - """ - from cecli.helpers import command_queue - - item = command_queue.dequeue_prompt(self) - if item is None: - return False - - self.io.tool_output(f"Processing queued prompt (id: {item['id']})...") - try: - self.io.output_task = asyncio.create_task(self.generate(item["text"], preproc)) - await self.io.output_task - except SwitchCoderSignal: - raise - except ReloadProgramSignal: - raise - except asyncio.CancelledError: - raise - except Exception as e: - self.io.tool_error(f"Error processing queued prompt (id: {item['id']}): {e}") - - return True - def copy_context(self): if self.auto_copy_context: self.commands.execute("copy-context", "") @@ -2015,6 +1960,20 @@ async def run_one(self, user_message, preproc): await self.auto_save_session(force=True) + # Move to the next queued prompt (CLI-33) only after the current message + # has fully completed, so the queue drains within run_one() instead of + # being watched by the generation loops. + if self.prompt_queue and not self._processing_queue: + self._processing_queue = True + try: + item = command_queue.dequeue_prompt(self) + finally: + self._processing_queue = False + + if item is not None: + self.io.tool_output(f"Processing queued prompt (id: {item['id']})...") + await self.run_one(item["text"], preproc) + if not await HookIntegration.call_end_hooks(self): self.io.tool_warning("Execution stopped by end hook") return