feat: add LiteLLM as AI gateway provider

pyproject.toml

@@ -66,6 +66,7 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
+litellm = ["litellm>=1.80.0,<2.0"]
 multimodal = ["everalgo-parser[svg]>=0.1.0"]  # [svg] bundles cairosvg → SVG works by default
 
 [build-system]

src/everos/component/embedding/__init__.py

@@ -19,6 +19,7 @@
 from .accessor import EmbeddingNotConfiguredError as EmbeddingNotConfiguredError
 from .accessor import get_embedder as get_embedder
 from .factory import build_embedding_provider as build_embedding_provider
+from .litellm_provider import LiteLLMEmbeddingProvider as LiteLLMEmbeddingProvider
 from .openai_provider import OpenAIEmbeddingProvider as OpenAIEmbeddingProvider
 from .protocol import EmbeddingError as EmbeddingError
 from .protocol import EmbeddingProvider as EmbeddingProvider
@@ -27,6 +28,7 @@
     "EmbeddingError",
     "EmbeddingNotConfiguredError",
     "EmbeddingProvider",
+    "LiteLLMEmbeddingProvider",
     "OpenAIEmbeddingProvider",
     "build_embedding_provider",
     "get_embedder",

src/everos/component/embedding/factory.py

@@ -1,4 +1,11 @@
-"""Factory for building an embedding provider from :class:`EmbeddingSettings`."""
+"""Factory for building an embedding provider from :class:`EmbeddingSettings`.
+
+Dispatches on ``settings.provider``:
+
+- ``"openai"`` — :class:`OpenAIEmbeddingProvider` (requires ``base_url``).
+- ``"litellm"`` — :class:`LiteLLMEmbeddingProvider` (routes via
+  model-id prefix; ``base_url`` optional).
+"""
 
 from __future__ import annotations
 
@@ -7,7 +14,6 @@
 from .openai_provider import OpenAIEmbeddingProvider
 from .protocol import EmbeddingProvider
 
-# Vector dim for the LanceDB index column — see ``17_lancedb_tables_design.md``.
 _DEFAULT_DIM = 1024
 
 
@@ -16,7 +22,7 @@ def build_embedding_provider(
     *,
     dim: int = _DEFAULT_DIM,
 ) -> EmbeddingProvider:
-    """Build an OpenAI-compatible embedding provider from settings.
+    """Build an embedding provider from settings.
 
     Args:
         settings: The :class:`EmbeddingSettings` slice from
@@ -29,8 +35,14 @@ def build_embedding_provider(
         ``embed_batch``.
 
     Raises:
-        ValueError: If ``model``, ``api_key`` or ``base_url`` is unset.
+        ValueError: If required fields are unset for the chosen provider.
     """
+    if settings.provider == "litellm":
+        return _build_litellm(settings, dim=dim)
+    return _build_openai(settings, dim=dim)
+
+
+def _build_openai(settings: EmbeddingSettings, *, dim: int) -> EmbeddingProvider:
     if not settings.model:
         raise ValueError(
             "Embedding model is not configured "
@@ -54,3 +66,26 @@ def build_embedding_provider(
         batch_size=settings.batch_size,
         max_concurrent=settings.max_concurrent,
     )
+
+
+def _build_litellm(settings: EmbeddingSettings, *, dim: int) -> EmbeddingProvider:
+    from .litellm_provider import LiteLLMEmbeddingProvider
+
+    if not settings.model:
+        raise ValueError(
+            "Embedding model is not configured "
+            "(set EVEROS_EMBEDDING__MODEL or [embedding] model in user toml)"
+        )
+    return LiteLLMEmbeddingProvider(
+        model=settings.model,
+        api_key=(
+            settings.api_key.get_secret_value()
+            if settings.api_key is not None
+            else None
+        ),
+        base_url=settings.base_url,
+        dim=dim,
+        timeout=settings.timeout_seconds,
+        batch_size=settings.batch_size,
+        max_concurrent=settings.max_concurrent,
+    )

src/everos/component/embedding/litellm_provider.py

@@ -0,0 +1,99 @@
+"""LiteLLM AI gateway embedding provider.
+
+Wraps ``litellm.aembedding`` so any provider litellm supports (OpenAI,
+Cohere, Bedrock, HuggingFace, …) can serve embeddings without
+per-provider forks.  Mirrors :class:`OpenAIEmbeddingProvider`'s
+batching + concurrency model: inputs are chunked by ``batch_size``
+and an :class:`asyncio.Semaphore` bounds in-flight requests.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Sequence
+from typing import Any
+
+from .protocol import EmbeddingError
+
+
+class LiteLLMEmbeddingProvider:
+    """LiteLLM-backed embedding provider with batching + concurrency.
+
+    Args:
+        model: LiteLLM embedding model id
+            (e.g. ``"openai/text-embedding-3-small"``).
+        api_key: Bearer credential forwarded to litellm. When *not* set
+            litellm falls back to provider-specific env vars.
+        base_url: Optional proxy / gateway URL.
+        dim: Target vector dimension. Vectors longer than this are
+            truncated client-side.
+        timeout: Per-request timeout, seconds.
+        batch_size: How many inputs per ``/embeddings`` call.
+        max_concurrent: Cap on in-flight chunked requests.
+    """
+
+    def __init__(
+        self,
+        *,
+        model: str,
+        api_key: str | None = None,
+        base_url: str | None = None,
+        dim: int = 1024,
+        timeout: float = 30.0,
+        batch_size: int = 10,
+        max_concurrent: int = 5,
+    ) -> None:
+        self.dim = dim
+        self._model = model
+        self._api_key = api_key
+        self._base_url = base_url
+        self._timeout = timeout
+        self._batch_size = batch_size
+        self._semaphore = asyncio.Semaphore(max_concurrent)
+
+    async def embed(self, text: str) -> list[float]:
+        """Embed a single string."""
+        vectors = await self._embed_chunk([text])
+        return vectors[0]
+
+    async def embed_batch(self, texts: Sequence[str]) -> list[list[float]]:
+        """Embed many strings, preserving input order."""
+        if not texts:
+            return []
+        chunks = [
+            list(texts[i : i + self._batch_size])
+            for i in range(0, len(texts), self._batch_size)
+        ]
+        results = await asyncio.gather(*(self._embed_chunk(c) for c in chunks))
+        return [vec for chunk in results for vec in chunk]
+
+    async def _embed_chunk(self, chunk: list[str]) -> list[list[float]]:
+        """One embedding call, semaphore-guarded."""
+        try:
+            import litellm
+        except ImportError as exc:
+            raise EmbeddingError(
+                "litellm is not installed. Install with: pip install 'everos[litellm]'"
+            ) from exc
+
+        kwargs: dict[str, Any] = {
+            "model": self._model,
+            "input": chunk,
+            "drop_params": True,
+            "timeout": self._timeout,
+        }
+        if self._api_key is not None:
+            kwargs["api_key"] = self._api_key
+        if self._base_url:
+            kwargs["api_base"] = self._base_url
+
+        async with self._semaphore:
+            try:
+                response = await litellm.aembedding(**kwargs)
+            except Exception as exc:
+                qualname = f"{type(exc).__module__}.{type(exc).__name__}"
+                if qualname.startswith("litellm.") or qualname.startswith("openai."):
+                    raise EmbeddingError(str(exc)) from exc
+                raise
+
+        return [list(item.embedding[: self.dim]) for item in response.data]

src/everos/component/llm/__init__.py

@@ -24,6 +24,7 @@
 from .client import get_llm_client as get_llm_client
 from .client import get_multimodal_llm_client as get_multimodal_llm_client
 from .factory import build_llm_provider as build_llm_provider
+from .litellm_provider import LiteLLMProvider as LiteLLMProvider
 from .openai_provider import OpenAIProvider as OpenAIProvider
 from .protocol import ChatMessage as ChatMessage
 from .protocol import ChatResponse as ChatResponse
@@ -37,6 +38,7 @@
     "LLMClient",
     "LLMError",
     "LLMNotConfiguredError",
+    "LiteLLMProvider",
     "OpenAIProvider",
     "Usage",
     "build_llm_provider",

src/everos/component/llm/client.py

@@ -1,10 +1,14 @@
 """Process-wide LLM client accessor.
 
-Lazy singleton — first call reads settings and builds the algo LLM
+Lazy singleton -- first call reads settings and builds the algo LLM
 client; subsequent calls return the cached instance. Raises
 :class:`LLMNotConfiguredError` when no credentials are present so
 misconfiguration surfaces at app startup (via the LLM lifespan
 provider) instead of silently failing per-request downstream.
+
+When ``settings.llm.provider`` is ``"litellm"``, the factory-built
+:class:`LiteLLMProvider` is used instead of the everalgo default
+client, giving users access to 100+ LLM providers.
 """
 
 from __future__ import annotations
@@ -39,6 +43,17 @@ def get_llm_client() -> LLMClient:
         return _llm_client
 
     llm_cfg = load_settings().llm
+
+    if llm_cfg.provider == "litellm":
+        from .factory import build_llm_provider
+
+        try:
+            _llm_client = build_llm_provider(llm_cfg)
+        except ValueError as exc:
+            raise LLMNotConfiguredError(str(exc)) from exc
+        logger.info("llm_client_built", model=llm_cfg.model, provider="litellm")
+        return _llm_client
+
     api_key = (
         llm_cfg.api_key.get_secret_value() if llm_cfg.api_key is not None else None
     )
@@ -60,7 +75,7 @@ def get_llm_client() -> LLMClient:
 def get_multimodal_llm_client() -> LLMClient:
     """Return the singleton multimodal LLM client (for everalgo.parser).
 
-    Reads the flat ``[multimodal]`` config — kept separate from the main
+    Reads the flat ``[multimodal]`` config -- kept separate from the main
     ``[llm]`` so parsing can target a vision/audio-capable endpoint.
 
     Raises:

src/everos/component/llm/factory.py

@@ -1,4 +1,11 @@
-"""Factory for building an LLM provider from :class:`LLMSettings`."""
+"""Factory for building an LLM provider from :class:`LLMSettings`.
+
+Dispatches on ``settings.provider``:
+
+- ``"openai"`` — :class:`OpenAIProvider` (requires ``base_url``).
+- ``"litellm"`` — :class:`LiteLLMProvider` (routes via model-id
+  prefix; ``base_url`` optional).
+"""
 
 from __future__ import annotations
 
@@ -9,12 +16,10 @@
 
 
 def build_llm_provider(settings: LLMSettings) -> LLMClient:
-    """Build an OpenAI-compatible LLM provider from settings.
+    """Build an LLM provider from settings.
 
     Unwraps :class:`pydantic.SecretStr` here so downstream callers never
-    touch the raw key directly. Fails fast if either ``api_key`` or
-    ``base_url`` is missing — caller is expected to set them via
-    ``.env`` / user toml / programmatic init before calling.
+    touch the raw key directly.
 
     Args:
         settings: The :class:`LLMSettings` slice from
@@ -26,8 +31,15 @@ def build_llm_provider(settings: LLMSettings) -> LLMClient:
         operators via ``llm=``.
 
     Raises:
-        ValueError: If ``api_key`` or ``base_url`` is unset.
+        ValueError: If required credentials are unset for the chosen
+            provider.
     """
+    if settings.provider == "litellm":
+        return _build_litellm(settings)
+    return _build_openai(settings)
+
+
+def _build_openai(settings: LLMSettings) -> LLMClient:
     if settings.api_key is None:
         raise ValueError(
             "LLM api_key is not configured "
@@ -43,3 +55,17 @@ def build_llm_provider(settings: LLMSettings) -> LLMClient:
         api_key=settings.api_key.get_secret_value(),
         base_url=settings.base_url,
     )
+
+
+def _build_litellm(settings: LLMSettings) -> LLMClient:
+    from .litellm_provider import LiteLLMProvider
+
+    return LiteLLMProvider(
+        model=settings.model,
+        api_key=(
+            settings.api_key.get_secret_value()
+            if settings.api_key is not None
+            else None
+        ),
+        base_url=settings.base_url,
+    )