diff --git a/docs/architecture.md b/docs/architecture.md index 72caa63..9dd3187 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -45,13 +45,54 @@ for latency (smaller batch, faster per-token). ### Proxy (`xpyd/proxy.py`) -The central routing and scheduling component. It: +`Proxy` owns scheduling, tokenizer state, backend forwarding, and instance +membership. `ProxyServer` constructs these runtime components and starts uvicorn. +The existing `xpyd.proxy:main` CLI entry point remains supported. + +### CLI and Application Assembly + +- `xpyd/cli.py` parses arguments, loads and overrides configuration, and runs + configuration-only commands. `load_config()` returns a `ProxyConfig` when a + server should start, or `None` when a configuration command completes without + starting one. It does not import or construct the proxy runtime. +- `xpyd/app.py` exposes `create_app(config, proxy, health_monitor=None)`. It + assembles FastAPI around an existing proxy, mounts its router, and owns CORS, + readiness checks, and the discovery/health-monitor/ZMQ-listener lifecycle. + Constructing the app does not start a network listener or background services. + +Backends may be offline at startup. Discovery starts in the background, inference +returns 503 until ready, and health/status/metrics/admin endpoints remain +accessible. + +### Endpoint Modules (`xpyd/routes/`) + +Each logical endpoint has a module with `register(router, server)`. +`routes/__init__.py` registers them all. URL aliases share their logical module: +`score.py` handles `/score` and `/v1/score`, `rerank.py` handles all three rerank +paths, and `health.py` handles `/health` and `/ping`. `status.py` serves both +`/status` and `/status/instances`; `instances.py` handles addition and removal. +Existing OPTIONS handlers stay beside their endpoints. + +`completions.py` and `chat_completions.py` are separate HTTP entry points. Both +delegate to `xpyd/completion_handler.py`, which contains shared validation, +request preparation, aggregated/P-D execution, response conversion, first-token +generators, metrics, and reservation cleanup. Passthrough endpoints reuse +`Proxy.post_to_instance()`. + +Runtime imports are one-way: + +```text +proxy -> cli -> config +proxy -> app -> discovery / health_monitor +proxy -> routes -> completion_handler +proxy -> completion_handler +``` -- Accepts OpenAI-compatible API requests from clients -- Routes requests through the prefill → decode pipeline -- Manages prefill and decode instance pools -- Supports dynamic instance addition/removal via admin API -- Provides health checking and status endpoints +App assembly, routes, and completion helpers receive the proxy object rather +than importing its implementation at runtime. References to `Proxy` for type +annotations are guarded by `TYPE_CHECKING`. Completion helpers do not import +routes or app assembly. Resource ownership during streaming is unchanged: +request completion and reservation release remain tied to response consumption. ### Scheduling Policies diff --git a/tests/unit/routes/test_admin_routes.py b/tests/unit/routes/test_admin_routes.py index 914081b..ba454ad 100644 --- a/tests/unit/routes/test_admin_routes.py +++ b/tests/unit/routes/test_admin_routes.py @@ -10,10 +10,16 @@ from fastapi.testclient import TestClient # First Party -from xpyd.routes.admin import register +from xpyd.routes import instances, status from ._helpers import build_app + +def register(router, server): + instances.register(router, server) + status.register(router, server) + + API_KEY = "s3cr3t" diff --git a/tests/unit/routes/test_completion_routes.py b/tests/unit/routes/test_completion_routes.py new file mode 100644 index 0000000..c6d763e --- /dev/null +++ b/tests/unit/routes/test_completion_routes.py @@ -0,0 +1,81 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Completion entry points share execution without coupling route modules.""" + +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest +from fastapi import Request +from fastapi.responses import JSONResponse +from fastapi.testclient import TestClient + +from xpyd.routes import chat_completions, completions + +from ._helpers import build_app + + +@pytest.fixture( + params=[ + (completions, "/v1/completions", False, "custom_create_completion"), + ( + chat_completions, + "/v1/chat/completions", + True, + "custom_create_chat_completion", + ), + ], + ids=["completions", "chat-completions"], +) +def endpoint(request): + return request.param + + +def test_default_entry_delegates_to_shared_handler(endpoint): + module, path, is_chat, _ = endpoint + server = SimpleNamespace( + custom_create_completion=None, + custom_create_chat_completion=None, + ) + handler = AsyncMock(return_value=JSONResponse({"shared": True}, status_code=202)) + with patch.object(module, "handle_completion", handler): + client = TestClient(build_app(module.register, server)) + response = client.post(path, json={"model": "example"}) + + assert response.status_code == 202 + assert response.json() == {"shared": True} + handler.assert_awaited_once() + assert handler.call_args.args[0] == path + assert isinstance(handler.call_args.args[1], Request) + assert handler.call_args.args[2] is server + assert handler.call_args.kwargs == {"is_chat": is_chat} + assert client.options(path).status_code == 200 + other_path = "/v1/completions" if is_chat else "/v1/chat/completions" + assert client.post(other_path, json={}).status_code == 404 + + +def test_custom_handler_keeps_content_type_validation(endpoint): + module, path, _, custom_attribute = endpoint + calls = [] + + async def custom(request: Request): + calls.append(await request.json()) + return JSONResponse({"custom": True}) + + server = SimpleNamespace( + custom_create_completion=None, + custom_create_chat_completion=None, + ) + setattr(server, custom_attribute, custom) + client = TestClient(build_app(module.register, server)) + + assert ( + client.post( + path, content="{}", headers={"content-type": "text/plain"} + ).status_code + == 415 + ) + assert calls == [] + response = client.post(path, json={"model": "example"}) + assert response.status_code == 200 + assert response.json() == {"custom": True} + assert calls == [{"model": "example"}] diff --git a/tests/unit/routes/test_forward_routes.py b/tests/unit/routes/test_forward_routes.py index e135ae5..96a2fba 100644 --- a/tests/unit/routes/test_forward_routes.py +++ b/tests/unit/routes/test_forward_routes.py @@ -10,10 +10,32 @@ from fastapi.testclient import TestClient # First Party -from xpyd.routes.forward import register +from xpyd.routes import ( + detokenize, + embeddings, + invocations, + pooling, + rerank, + score, + tokenize, +) from ._helpers import build_app + +def register(router, server): + for module in ( + tokenize, + detokenize, + embeddings, + pooling, + score, + rerank, + invocations, + ): + module.register(router, server) + + # (route, forwarded path, expected json template, a valid request body) ENDPOINTS = [ ( diff --git a/tests/unit/routes/test_health_routes.py b/tests/unit/routes/test_health_routes.py index 4074c18..58d4e9e 100644 --- a/tests/unit/routes/test_health_routes.py +++ b/tests/unit/routes/test_health_routes.py @@ -16,11 +16,16 @@ # First Party from xpyd.proxy import Proxy -from xpyd.routes.health import register +from xpyd.routes import health, metrics, models, version from ._helpers import FakeResponse, build_app, fake_aiohttp, make_proxy +def register(router, server): + for module in (health, models, version, metrics): + module.register(router, server) + + class _HealthServer: """Records how the routes call into the proxy fetch helper.""" diff --git a/tests/unit/test_aggregated_completion.py b/tests/unit/test_aggregated_completion.py index 02bee28..cddd14d 100644 --- a/tests/unit/test_aggregated_completion.py +++ b/tests/unit/test_aggregated_completion.py @@ -6,9 +6,9 @@ import pytest +from xpyd.completion_handler import handle_completion from xpyd.proxy import Proxy from xpyd.registry import InstanceRegistry -from xpyd.routes.completions import handle_completion from xpyd.scheduler import SchedulingContext, default_registry MODEL = "qwen-2" @@ -174,7 +174,7 @@ async def forward(*args, **kwargs): yield b"" # pragma: no cover proxy.forward_request = forward - with patch("xpyd.routes.completions.track_request_end") as ended: + with patch("xpyd.completion_handler.track_request_end") as ended: with pytest.raises(asyncio.CancelledError): await handle_completion("/v1/completions", request, proxy, False) ended.assert_called_once() diff --git a/tests/unit/test_app.py b/tests/unit/test_app.py new file mode 100644 index 0000000..56e20f9 --- /dev/null +++ b/tests/unit/test_app.py @@ -0,0 +1,275 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Application assembly, dependency boundaries and proxy-first lifecycle.""" + +import json +import subprocess +import sys +from unittest.mock import AsyncMock, Mock, patch +from urllib.parse import urlsplit + +import pytest +from fastapi.testclient import TestClient + +from xpyd.app import create_app +from xpyd.config import ProxyConfig +from xpyd.proxy import ProxyServer + +from .routes._helpers import FakeResponse, fake_aiohttp + + +def test_support_modules_do_not_import_proxy_runtime(): + result = subprocess.run( + [ + sys.executable, + "-c", + "\n".join( + [ + "import importlib, sys", + "for name in ('xpyd.cli', 'xpyd.app', 'xpyd.routes',", + " 'xpyd.completion_handler'):", + " importlib.import_module(name)", + " assert 'xpyd.proxy' not in sys.modules, name", + ] + ), + ], + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize("topology", ["disaggregated", "aggregated", "mixed"]) +def test_proxy_first_discovery_loss_and_reconnection(topology): + roles = [] + if topology != "aggregated": + roles.extend(["prefill", "decode", "decode"]) + if topology != "disaggregated": + roles.extend(["aggregated", "aggregated"]) + addresses = [f"127.0.0.1:{18100 + index}" for index in range(len(roles))] + model_by_address = { + address: "agg-model" if role == "aggregated" else "pd-model" + for role, address in zip(roles, addresses) + } + config = ProxyConfig( + instances=[ + { + "role": role, + "address": address, + "model": model_by_address[address] if topology == "mixed" else "", + } + for role, address in zip(roles, addresses) + ], + scheduling="roundrobin", + probe_interval_seconds=3600, + heartbeat_interval_seconds=0.001, + health_check={"enabled": True, "interval_seconds": 3600}, + ) + server = ProxyServer(config) + proxy = server.proxy_instance + proxy.ensure_tokenizer = Mock(return_value=True) + forwarded = [] + + async def forward(url, data, **kwargs): + address = urlsplit(url).netloc + assert address in online + assert data["model"] == model_by_address[address] + forwarded.append(address) + yield json.dumps({"choices": [{"text": "ok"}]}).encode() + + proxy.forward_request = forward + app = create_app(config, proxy, server.health_monitor) + discovery = proxy.discovery + monitor = server.health_monitor + assert discovery._task is None + assert monitor._task is None + online = set() + + def backend(_method, url, _payload): + target = urlsplit(url) + if target.netloc not in online: + raise ConnectionRefusedError(target.netloc) + if target.path == "/health": + return FakeResponse(200, {"status": "ok"}) + assert target.path == "/v1/models" + return FakeResponse(200, {"data": [{"id": model_by_address[target.netloc]}]}) + + with fake_aiohttp(backend) as session, TestClient(app) as client: + assert not discovery._task.done() + assert not monitor._task.done() + + async def probe(): + await discovery._probe_all(session) + await monitor.check_once() + + def check_status(): + response = client.get("/status/instances") + assert response.status_code == 200 + body = response.json() + assert set(body) == { + "prefill_instances", + "decode_instances", + "aggregated_instances", + } + for role in ("prefill", "decode", "aggregated"): + assert len(body[f"{role}_instances"]) == roles.count(role) + entries = [entry for instances in body.values() for entry in instances] + assert {entry["address"] for entry in entries} == set(addresses) + assert { + entry["address"] for entry in entries if entry["status"] == "healthy" + } == online + assert all(entry["active_requests"] == 0 for entry in entries) + + def complete(expected_status): + for model in set(model_by_address.values()): + for path, prompt in ( + ("/v1/completions", {"prompt": "hello"}), + ( + "/v1/chat/completions", + {"messages": [{"role": "user", "content": "hello"}]}, + ), + ): + response = client.post(path, json={"model": model, **prompt}) + assert response.status_code == expected_status, response.text + + client.portal.call(probe) + complete(503) + assert forwarded == [] + check_status() + assert client.get("/status").status_code == 200 + assert client.get("/metrics").status_code == 200 + assert set(client.get("/health").json()) == set(addresses) + assert set(client.get("/ping").json()) == set(addresses) + assert client.post("/instances/add", json={}).status_code == 422 + assert client.post("/instances/remove", json={}).status_code == 422 + + online.update(addresses) + client.portal.call(probe) + assert discovery.is_ready + assert {call.args[0] for call in proxy.ensure_tokenizer.call_args_list} == set( + model_by_address.values() + ) + assert { + entry["id"] for entry in client.get("/v1/models").json()["data"] + } == set(model_by_address.values()) + complete(200) + check_status() + preflight = client.options( + "/v1/completions", + headers={ + "Origin": "https://example.com", + "Access-Control-Request-Method": "POST", + }, + ) + assert preflight.status_code == 200 + assert preflight.headers["access-control-allow-origin"] == "*" + + for role in ("decode", "aggregated"): + candidates = [ + address + for address, configured_role in zip(addresses, roles) + if configured_role == role + ] + if candidates: + online.remove(candidates[0]) + client.portal.call(probe) + assert discovery.is_ready + complete(200) + check_status() + + with patch("xpyd.discovery.logger.info") as heartbeat: + discovery._last_heartbeat = None + client.portal.call(discovery._log_heartbeat_if_due) + fields = [] + if topology != "aggregated": + fields.extend(["P=1/1 online", "D=1/2 online"]) + if topology != "disaggregated": + fields.append("aggregated=1/2 online") + assert heartbeat.call_args.args == ( + "Node heartbeat | mode=%s | %s", + topology, + " | ".join(fields), + ) + + online.clear() + client.portal.call(probe) + assert not discovery.is_ready + complete(503) + check_status() + + online.update(addresses) + client.portal.call(probe) + assert discovery.is_ready + complete(200) + check_status() + + assert discovery._task.done() + assert monitor._task.done() + + +def test_zmq_lifecycle_order(): + config = ProxyConfig( + model="example", + prefill=["127.0.0.1:18100"], + decode=["127.0.0.1:18200"], + scheduling="roundrobin", + disaggregated_mode="zmq", + zmq={ + "receivers": { + "127.0.0.1:18200": { + "host": "127.0.0.1", + "init_ports": [7300], + "alloc_ports": [7400], + } + } + }, + health_check={"enabled": True}, + ) + server = ProxyServer(config) + proxy = server.proxy_instance + app = create_app(config, proxy, server.health_monitor) + events = [] + listener = Mock( + start=AsyncMock(side_effect=lambda: events.append("listener:start")), + stop=AsyncMock(side_effect=lambda: events.append("listener:stop")), + ) + with ( + patch( + "xpyd.zmq_notifications.ZmqNotificationListener", return_value=listener + ) as factory, + patch.object( + proxy.discovery, + "start", + AsyncMock(side_effect=lambda: events.append("discovery:start")), + ), + patch.object( + proxy.discovery, + "stop", + AsyncMock(side_effect=lambda: events.append("discovery:stop")), + ), + patch.object( + server.health_monitor, + "start", + AsyncMock(side_effect=lambda: events.append("health:start")), + ), + patch.object( + server.health_monitor, + "stop", + AsyncMock(side_effect=lambda: events.append("health:stop")), + ), + ): + assert events == [] + with TestClient(app): + assert proxy.zmq_notifications is listener + assert events == ["listener:start", "discovery:start", "health:start"] + factory.assert_called_once_with( + config.zmq.host, config.zmq.port, config.zmq.notification_timeout_seconds + ) + assert events == [ + "listener:start", + "discovery:start", + "health:start", + "discovery:stop", + "listener:stop", + "health:stop", + ] diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 44a674e..cbbc28b 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -8,17 +8,15 @@ import pytest -from xpyd.config import ProxyConfig -from xpyd.proxy import ( - Proxy, - ProxyServer, +from xpyd.cli import ( _apply_cli_overrides, _build_parser, _normalize_cli_args, _print_config_summary, _resolve_config_path, - main, ) +from xpyd.config import ProxyConfig +from xpyd.proxy import Proxy, ProxyServer, main class TestSubcommandParser: diff --git a/tests/unit/test_completion_handler.py b/tests/unit/test_completion_handler.py index dd128b8..b118aa6 100644 --- a/tests/unit/test_completion_handler.py +++ b/tests/unit/test_completion_handler.py @@ -8,8 +8,7 @@ from fastapi import HTTPException from starlette.responses import JSONResponse -from xpyd.metrics import REGISTRY -from xpyd.routes.completions import ( +from xpyd.completion_handler import ( _chat_completion_nonstream, _chat_completion_stream, _RequestReservation, @@ -23,6 +22,7 @@ tokenize_zmq_prompt, validate_completion_request, ) +from xpyd.metrics import REGISTRY from xpyd.scheduler import RoundRobinSchedulingPolicy, Scheduler, SchedulingContext @@ -547,8 +547,8 @@ async def test_invalid_json(self, server): raw_request.json = AsyncMock(side_effect=ValueError("bad json")) with ( - patch("xpyd.routes.completions.track_request_start", return_value=0), - patch("xpyd.routes.completions.track_request_end") as track_end, + patch("xpyd.completion_handler.track_request_start", return_value=0), + patch("xpyd.completion_handler.track_request_end") as track_end, ): result = await handle_completion( "/v1/completions", raw_request, server, is_chat=False @@ -593,8 +593,8 @@ async def test_missing_required_field(self, server): raw_request.json = AsyncMock(return_value={}) with ( - patch("xpyd.routes.completions.track_request_start", return_value=0), - patch("xpyd.routes.completions.track_request_end") as track_end, + patch("xpyd.completion_handler.track_request_start", return_value=0), + patch("xpyd.completion_handler.track_request_end") as track_end, ): result = await handle_completion( "/v1/completions", raw_request, server, is_chat=False @@ -620,9 +620,9 @@ async def test_no_available_instance(self, server): before = REGISTRY.get_sample_value("proxy_instance_errors_total", labels) or 0 with ( - patch("xpyd.routes.completions.track_request_start", return_value=0), - patch("xpyd.routes.completions.track_request_end") as track_end, - patch("xpyd.routes.completions.logger"), + patch("xpyd.completion_handler.track_request_start", return_value=0), + patch("xpyd.completion_handler.track_request_end") as track_end, + patch("xpyd.completion_handler.logger"), ): result = await handle_completion( "/v1/completions", raw_request, server, is_chat=False @@ -694,8 +694,8 @@ async def test_unknown_aggregated_model_ends_metrics(self, server): server.registry.get_registered_models.return_value = ["known-model"] with ( - patch("xpyd.routes.completions.track_request_start", return_value=0), - patch("xpyd.routes.completions.track_request_end") as track_end, + patch("xpyd.completion_handler.track_request_start", return_value=0), + patch("xpyd.completion_handler.track_request_end") as track_end, ): result = await handle_completion( "/v1/completions", raw_request, server, is_chat=False @@ -795,9 +795,9 @@ async def forward(url, data, **kwargs): server.forward_request = forward with ( - patch("xpyd.routes.completions.track_request_start", return_value=0), - patch("xpyd.routes.completions.track_request_end"), - patch("xpyd.routes.completions.record_disaggregated_metrics"), + patch("xpyd.completion_handler.track_request_start", return_value=0), + patch("xpyd.completion_handler.track_request_end"), + patch("xpyd.completion_handler.record_disaggregated_metrics"), ): response = await handle_completion( "/v1/chat/completions", raw_request, server, is_chat=True diff --git a/xpyd/app.py b/xpyd/app.py new file mode 100644 index 0000000..fc1e2f7 --- /dev/null +++ b/xpyd/app.py @@ -0,0 +1,93 @@ +# SPDX-License-Identifier: Apache-2.0 +"""FastAPI application assembly and background service lifecycle.""" + +import asyncio + +from fastapi import FastAPI, Request +from fastapi.middleware.cors import CORSMiddleware + +from xpyd.config import ProxyConfig +from xpyd.discovery import NodeDiscovery +from xpyd.errors import PROXY_ERROR, error_response +from xpyd.health_monitor import HealthMonitor + + +def create_app( + config: ProxyConfig, + proxy: "Proxy", # noqa: F821 + health_monitor: HealthMonitor | None = None, +) -> FastAPI: + """Assemble an app around an existing proxy without starting a listener.""" + discovery = NodeDiscovery( + prefill_instances=proxy.prefill_instances, + decode_instances=proxy.decode_instances, + probe_interval=config.probe_interval_seconds, + wait_timeout=config.wait_timeout_seconds, + heartbeat_interval=config.heartbeat_interval_seconds, + registry=proxy.registry, + aggregated_instances=[ + address + for instances in proxy.aggregated_instances.values() + for address in instances + ], + on_model_discovered=lambda model: asyncio.to_thread( + proxy.ensure_tokenizer, model + ), + ) + proxy.discovery = discovery + proxy.health_monitor = health_monitor + + app = FastAPI() + app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=False, + allow_methods=["*"], + allow_headers=["*"], + ) + + @app.middleware("http") + async def _check_readiness(request: Request, call_next): + # Operational endpoints must remain accessible before backends are ready. + path = request.url.path + if path in ( + "/health", + "/ping", + "/status", + "/metrics", + "/instances/add", + "/instances/remove", + ) or path.startswith("/status/"): + return await call_next(request) + if not discovery.is_ready: + return error_response("Waiting for backend nodes", PROXY_ERROR, 503) + return await call_next(request) + + @app.on_event("startup") + async def _start_discovery(): + if config.disaggregated_mode == "zmq": + from xpyd.zmq_notifications import ZmqNotificationListener + + zmq_config = config.zmq + assert zmq_config is not None + listener = ZmqNotificationListener( + zmq_config.host, + zmq_config.port, + zmq_config.notification_timeout_seconds, + ) + await listener.start() + proxy.zmq_notifications = listener + await discovery.start() + if health_monitor: + await health_monitor.start() + + @app.on_event("shutdown") + async def _stop_discovery(): + await discovery.stop() + if proxy.zmq_notifications is not None: + await proxy.zmq_notifications.stop() + if health_monitor: + await health_monitor.stop() + + app.include_router(proxy.router) + return app diff --git a/xpyd/cli.py b/xpyd/cli.py new file mode 100644 index 0000000..6146881 --- /dev/null +++ b/xpyd/cli.py @@ -0,0 +1,271 @@ +# SPDX-License-Identifier: Apache-2.0 +"""CLI parsing and configuration commands, independent of the proxy runtime.""" + +import argparse +import os +import sys +from pathlib import Path + +from xpyd.config import ProxyConfig + +_VERSION = "1.8.0" + + +def _valid_port(value: str) -> int: + """Parse a TCP port for argparse with a concise user-facing error.""" + try: + port = int(value) + except ValueError as exc: + raise argparse.ArgumentTypeError("port must be an integer") from exc + if not 1 <= port <= 65535: + raise argparse.ArgumentTypeError("port must be between 1 and 65535") + return port + + +def _build_parser(): + """Build the subcommand argument parser for the proxy CLI.""" + parser = argparse.ArgumentParser( + prog="xpyd", + description="xPyD — lightweight disaggregated proxy server", + ) + parser.add_argument( + "--version", + "-V", + action="version", + version=f"%(prog)s {_VERSION}", + ) + + subparsers = parser.add_subparsers(dest="command") + + proxy_parser = subparsers.add_parser( + "proxy", + prog="xpyd [proxy]", + help="Start the proxy server (default command)", + ) + proxy_parser.add_argument( + "--config", + "-c", + type=str, + default=None, + help="Path to YAML configuration file", + ) + proxy_parser.add_argument( + "--validate-config", + type=str, + default=None, + metavar="FILE", + help="Validate YAML config and exit (no server start)", + ) + proxy_parser.add_argument( + "--init-config", + nargs="?", + const="./xpyd.yaml", + default=None, + metavar="PATH", + help="Generate xpyd.yaml, optionally through an interactive wizard " + "(default path: ./xpyd.yaml)", + ) + proxy_parser.add_argument( + "--force", + action="store_true", + help="Overwrite an existing --init-config destination", + ) + proxy_parser.add_argument( + "--port", + type=_valid_port, + default=None, + help="Override the port from config", + ) + proxy_parser.add_argument( + "--log-level", + choices=("debug", "info", "warning", "error"), + default=None, + dest="log_level", + help="Override log level: debug|info|warning|error", + ) + proxy_parser.add_argument( + "--disaggregated-mode", + choices=("direct", "nixl", "zmq"), + default=None, + help="disaggregated transfer mode (default: config value or direct)", + ) + proxy_parser.add_argument( + "--first-token-source", + choices=("prefill", "decode"), + default=None, + help="Backend that provides the first client-visible token", + ) + + # fix-config subcommand + fix_parser = subparsers.add_parser( + "fix-config", + help="Auto-fix common config mistakes", + ) + fix_parser.add_argument( + "config_path", + type=str, + help="Path to YAML configuration file to fix", + ) + fix_parser.add_argument( + "--write", + action="store_true", + default=False, + help="Write fixes back to the file (creates timestamped .bak backup). " + "Note: does not preserve YAML comments or formatting.", + ) + fix_parser.add_argument( + "--interactive", + action="store_true", + default=False, + help="Interactively acknowledge ambiguous suggestions one by one", + ) + + return parser + + +def _normalize_cli_args(argv: list[str]) -> list[str]: + """Treat omitted subcommands as the default ``proxy`` command.""" + if argv and argv[0] == "proxy": + if len(argv) > 1 and argv[1] in {"fix-config", "--version", "-V"}: + return argv[1:] + return argv + if argv and argv[0] in {"fix-config", "--version", "-V"}: + return argv + return ["proxy", *argv] + + +def _resolve_config_path(args): + """Resolve the config file path: --config > XPYD_CONFIG env > ./xpyd.yaml. + + Returns the path string, or ``None`` when no config can be found. + """ + if args.config: + return args.config + env_config = os.environ.get("XPYD_CONFIG") + if env_config: + return env_config + default_path = "./xpyd.yaml" + if os.path.exists(default_path): + print( + "No config specified; found ./xpyd.yaml and using it.", + flush=True, + ) + return default_path + return None + + +def _print_config_summary(config: ProxyConfig) -> None: + """Print topology-aware details for a validated configuration.""" + if config.instances is None: + topology = "disaggregated" if config.prefill else "decode-only" + print(f" topology: {topology}") + if config.prefill: + print(f" transfer: {config.disaggregated_mode}") + print(f" model: {config.model}") + print(f" prefill: {len(config.prefill)} instances") + print(f" decode: {len(config.decode)} instances") + else: + roles = {instance.role for instance in config.instances} + topology = "aggregated" if roles == {"aggregated"} else "disaggregated" + print(f" topology: {topology}") + if topology == "disaggregated": + print(f" transfer: {config.disaggregated_mode}") + models = sorted( + {instance.model for instance in config.instances if instance.model} + ) + print(f" models: {', '.join(models) if models else 'auto-detected'}") + for role in ("aggregated", "prefill", "decode"): + count = sum(instance.role == role for instance in config.instances) + if count: + print(f" {role}: {count} instances") + print(f" port: {config.port}") + print(f" log_level: {config.log_level}") + + +def _apply_cli_overrides(config: ProxyConfig, args) -> ProxyConfig: + """Apply explicitly supplied CLI values and revalidate the result.""" + overrides = { + field: value + for field in ( + "port", + "log_level", + "disaggregated_mode", + "first_token_source", + ) + if (value := getattr(args, field, None)) is not None + } + if not overrides: + return config + + model_schedulers = config._model_schedulers.copy() + updated = ProxyConfig.model_validate({**config.model_dump(), **overrides}) + updated._model_schedulers = model_schedulers + return updated + + +def load_config(argv: list[str] | None = None) -> ProxyConfig | None: + """Run configuration commands or return the configuration to start a proxy.""" + from xpyd.init_config import generate_config + + parser = _build_parser() + args = parser.parse_args( + _normalize_cli_args(sys.argv[1:] if argv is None else argv) + ) + + if args.command == "fix-config": + from xpyd.config_fixer import run_fix_config + + sys.exit( + run_fix_config( + args.config_path, + write=args.write, + interactive=args.interactive, + ) + ) + elif args.command == "proxy": + if args.force and args.init_config is None: + parser.error("--force can only be used with --init-config") + + # --init-config: generate template and exit + if args.init_config is not None: + output_path = Path(args.init_config) + if output_path.exists() and not args.force: + parser.error( + f"config already exists: {output_path}; use --force to overwrite" + ) + try: + generate_config(args.init_config) + except OSError as exc: + parser.exit(1, f"xpyd: failed to write config: {exc}\n") + return None + + # --validate-config: validate and exit + if args.validate_config: + config_path = args.validate_config + try: + config = ProxyConfig.from_yaml(config_path) + print(f"Config is valid: {config_path}") + _print_config_summary(config) + sys.exit(0) + except Exception as exc: + print(f"Config validation failed: {exc}", file=sys.stderr) + sys.exit(1) + + # Resolve config path with precedence + config_path = _resolve_config_path(args) + if config_path is None: + config_path = "./xpyd.yaml" + print( + "No config specified and ./xpyd.yaml was not found; " + "starting config initialization." + ) + generate_config(config_path) + return None + try: + config = ProxyConfig.from_yaml(config_path) + return _apply_cli_overrides(config, args) + except (OSError, ValueError) as exc: + parser.exit(1, f"xpyd: failed to load config: {exc}\n") + else: + parser.print_help() + return None diff --git a/xpyd/completion_handler.py b/xpyd/completion_handler.py new file mode 100644 index 0000000..f4d1f05 --- /dev/null +++ b/xpyd/completion_handler.py @@ -0,0 +1,1166 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Shared execution and helpers for completion and chat completion requests.""" + +from __future__ import annotations + +import json +import logging +import sys +import time +import uuid +from asyncio import CancelledError +from collections.abc import AsyncGenerator, Callable, Mapping + +from fastapi import HTTPException, Request +from fastapi.responses import JSONResponse, StreamingResponse + +from xpyd.errors import INVALID_REQUEST, PROXY_ERROR, error_response +from xpyd.metrics import ( + FirstTokenTracker, + proxy_decode_active_requests, + proxy_decode_requests_total, + proxy_instance_errors_total, + proxy_prefill_active_requests, + proxy_prefill_queue_depth, + proxy_prefill_requests_total, + record_disaggregated_metrics, + track_request_end, + track_request_start, +) +from xpyd.scheduler import Reservation, SchedulingContext + +logger = logging.getLogger("xpyd.proxy") + + +async def P_first_token_generator( + generator_p: AsyncGenerator[bytes, None], + generator_d: AsyncGenerator[bytes, None], + release_prefill: Callable[[], None], + release_decode: Callable[[], None], +) -> AsyncGenerator[bytes, None]: + first_decode = True + + try: + async for chunk in generator_p: + yield chunk + finally: + release_prefill() + + try: + async for chunk in generator_d: + if first_decode: + first_decode = False + continue + yield chunk + finally: + release_decode() + + +async def D_first_token_generator( + generator_p: AsyncGenerator[bytes, None], + generator_d: AsyncGenerator[bytes, None], + release_prefill: Callable[[], None], + release_decode: Callable[[], None], +) -> AsyncGenerator[bytes, None]: + try: + async for _ in generator_p: + continue + finally: + release_prefill() + + try: + async for chunk in generator_d: + yield chunk + finally: + release_decode() + + +_CHAT_ONLY_REQUEST_FIELDS = { + "messages", + "max_completion_tokens", + "tools", + "tool_choice", + "parallel_tool_calls", + "chat_template", + "chat_template_kwargs", + "add_generation_prompt", + "continue_final_message", +} + + +class _RequestReservation: + """Release each P/D scheduling reservation at most once.""" + + def __init__( + self, + prefill: Reservation | None = None, + decode: Reservation | None = None, + ) -> None: + self.prefill = prefill + self.decode = decode + + def release_prefill(self) -> None: + if self.prefill is not None: + self.prefill.release() + + def release_decode(self) -> None: + if self.decode is not None: + self.decode.release() + + def release_all(self) -> None: + self.release_prefill() + self.release_decode() + + +# --------------------------------------------------------------------------- +# Pure helper functions (no server dependency or explicit server param) +# --------------------------------------------------------------------------- + + +def validate_completion_request(request: dict, is_chat: bool) -> JSONResponse | None: + """Validate required fields. Returns JSONResponse on error, None on success.""" + if is_chat: + if "messages" not in request: + return error_response( + "Missing required field: messages", INVALID_REQUEST, 400 + ) + if not isinstance(request["messages"], list): + return error_response("Field messages must be a list", INVALID_REQUEST, 400) + else: + if "prompt" not in request: + return error_response( + "Missing required field: prompt", INVALID_REQUEST, 400 + ) + return None + + +def extract_prompt_info( + request: dict, is_chat: bool, server: "Proxy" # noqa: F821 +) -> tuple[int, int, str]: + """Extract prompt metrics. Returns (total_length, max_tokens, prompt_text).""" + model = request.get("model", "") + if is_chat: + total_length = 0 + prompt_parts = [] + for msg in request["messages"]: + content = msg.get("content") + if content is None: + continue + if isinstance(content, str): + total_length += server.get_total_token_length(content, model) + prompt_parts.append(content) + elif isinstance(content, list): + for part in content: + if isinstance(part, dict) and part.get("type") == "text": + text = part.get("text", "") + total_length += server.get_total_token_length(text, model) + prompt_parts.append(text) + max_tokens = request.get("max_completion_tokens", 0) + if max_tokens == 0: + max_tokens = request.get("max_tokens", 0) + prompt_text = " ".join(prompt_parts) + else: + prompt = request.get("prompt") + total_length = server.get_total_token_length(prompt, model) + max_tokens = request.get("max_tokens", 0) + prompt_text = prompt if isinstance(prompt, str) else str(prompt) + return total_length, max_tokens, prompt_text + + +def build_kv_prepare_request( + request: dict, + is_chat: bool, + disaggregated_mode: str = "direct", +) -> dict: + """Build the KV-prepare request with max_tokens=1.""" + kv_prepare_request = request.copy() + kv_prepare_request["max_tokens"] = 1 + if is_chat: + kv_prepare_request["max_completion_tokens"] = 1 + if disaggregated_mode == "nixl": + kv_prepare_request["stream"] = False + kv_prepare_request.pop("stream_options", None) + kv_prepare_request["kv_transfer_params"] = { + "do_remote_decode": True, + "do_remote_prefill": False, + "remote_engine_id": None, + "remote_block_ids": None, + "remote_host": None, + "remote_port": None, + } + return kv_prepare_request + + +def build_zmq_prepare_request( + request: dict, + prompt_tokens: list[int], + request_id: str, + receiver, + first_token_source: str, + is_chat: bool = False, +) -> dict: + """Build an LMCache sender request for a preselected decode instance.""" + prepare = request.copy() + if is_chat: + for field in _CHAT_ONLY_REQUEST_FIELDS: + prepare.pop(field, None) + prepare["prompt"] = prompt_tokens + prepare["max_tokens"] = 1 + prepare["stream"] = False + prepare.pop("stream_options", None) + transfer_params = { + "disagg_spec": { + "req_id": request_id, + "receiver_host": receiver.host, + "receiver_init_port": receiver.init_ports, + "receiver_alloc_port": receiver.alloc_ports, + } + } + if first_token_source == "prefill": + transfer_params["ret_first_tok"] = True + prepare["kv_transfer_params"] = transfer_params + return prepare + + +def tokenize_zmq_prompt( + request: dict, is_chat: bool, server: "Proxy" # noqa: F821 +) -> list[int]: + """Tokenize the exact prompt used by the LMCache sender and receiver.""" + tokenizer = server.get_tokenizer(request.get("model", "")) + if tokenizer is None: + raise ValueError( + "ZMQ prompt tokenization requires a tokenizer for model " + f"{request.get('model', '')!r}" + ) + if not is_chat: + prompt = request["prompt"] + return list(prompt) if isinstance(prompt, list) else tokenizer.encode(prompt) + + template_kwargs = dict(request.get("chat_template_kwargs") or {}) + template_kwargs["tokenize"] = True + continue_final_message = request.get("continue_final_message", False) + template_kwargs["add_generation_prompt"] = request.get( + "add_generation_prompt", not continue_final_message + ) + if "continue_final_message" in request: + template_kwargs["continue_final_message"] = continue_final_message + if request.get("chat_template") is not None: + template_kwargs["chat_template"] = request["chat_template"] + if request.get("tools") is not None: + template_kwargs["tools"] = request["tools"] + tokenized = tokenizer.apply_chat_template(request["messages"], **template_kwargs) + if isinstance(tokenized, Mapping): + tokenized = tokenized.get("input_ids") + if tokenized is None: + raise ValueError("Chat template output did not contain input_ids") + if hasattr(tokenized, "tolist"): + tokenized = tokenized.tolist() + if ( + isinstance(tokenized, list) + and len(tokenized) == 1 + and isinstance(tokenized[0], list) + ): + tokenized = tokenized[0] + return list(tokenized) + + +def build_zmq_decode_request( + request: dict, + prompt_tokens: list[int], + first_token: int | None, + max_tokens: int, + is_chat: bool, +) -> dict: + """Build a completions request whose prompt exactly matches transferred KV.""" + decode = request.copy() + if is_chat: + for field in _CHAT_ONLY_REQUEST_FIELDS: + decode.pop(field, None) + if max_tokens > 0: + decode["max_tokens"] = max_tokens + decode["prompt"] = list(prompt_tokens) + if first_token is not None: + decode["prompt"].append(first_token) + if max_tokens > 0: + decode["max_tokens"] = max(1, max_tokens - 1) + decode.pop("kv_transfer_params", None) + return decode + + +def _completion_to_chat(output: dict) -> dict: + """Convert a text completion response to OpenAI chat completion shape.""" + result = output.copy() + result["object"] = "chat.completion" + result["choices"] = [ + { + "index": choice.get("index", index), + "message": { + "role": "assistant", + "content": choice.get("text", ""), + }, + "logprobs": choice.get("logprobs"), + "finish_reason": choice.get("finish_reason"), + } + for index, choice in enumerate(output.get("choices", [])) + ] + return result + + +def _completion_chunk_to_chat( + output: dict, + *, + response_metadata: dict | None = None, +) -> dict: + """Convert a text completion stream chunk to chat completion chunk shape.""" + result = output.copy() + if response_metadata: + result.update(response_metadata) + result["object"] = "chat.completion.chunk" + result["choices"] = [ + { + "index": choice.get("index", index), + "delta": {"content": choice.get("text", "")}, + "logprobs": choice.get("logprobs"), + "finish_reason": choice.get("finish_reason"), + } + for index, choice in enumerate(output.get("choices", [])) + ] + return result + + +async def _chat_completion_stream( + completion_generator, + *, + response_metadata: dict | None = None, + include_role: bool = True, +): + """Convert arbitrarily chunked completion SSE data to chat completion SSE.""" + buffer = "" + role_pending = include_role + async for chunk in completion_generator: + buffer += chunk.decode("utf-8") + while "\n\n" in buffer: + event, buffer = buffer.split("\n\n", 1) + for line in event.splitlines(): + if not line.startswith("data:"): + continue + payload = line[5:].strip() + if payload == "[DONE]": + yield b"data: [DONE]\n\n" + continue + output = _completion_chunk_to_chat( + json.loads(payload), + response_metadata=response_metadata, + ) + if role_pending and output["choices"]: + output["choices"][0]["delta"]["role"] = "assistant" + role_pending = False + yield ( + f"data: {json.dumps(output, separators=(',', ':'))}\n\n" + ).encode() + if buffer.strip(): + raise ValueError("Decode response ended with an incomplete SSE event") + + +async def _chat_completion_nonstream(completion_generator): + """Convert a non-streaming text completion response to chat shape.""" + value = b"" + async for chunk in completion_generator: + value += chunk + yield json.dumps( + _completion_to_chat(json.loads(value)), + separators=(",", ":"), + ).encode() + + +def _merge_prefill_token_usage(usage: dict) -> None: + """Move the appended prefill token from prompt usage to completion usage.""" + if usage.get("prompt_tokens", 0) > 0: + usage["prompt_tokens"] -= 1 + usage["completion_tokens"] = usage.get("completion_tokens", 0) + 1 + usage["total_tokens"] = usage.get("prompt_tokens", 0) + usage.get( + "completion_tokens", 0 + ) + + +async def _zmq_stream_usage_generator(decode_generator): + """Correct usage events after the prefill token was appended to the prompt.""" + buffer = "" + async for chunk in decode_generator: + buffer += chunk.decode("utf-8") + while "\n\n" in buffer: + event, buffer = buffer.split("\n\n", 1) + lines = [] + for line in event.splitlines(): + if line.startswith("data:"): + payload = line[5:].strip() + if payload != "[DONE]": + output = json.loads(payload) + if isinstance(output.get("usage"), dict): + _merge_prefill_token_usage(output["usage"]) + line = "data: " + json.dumps(output, separators=(",", ":")) + lines.append(line) + yield ("\n".join(lines) + "\n\n").encode() + if buffer.strip(): + raise ValueError("Decode response ended with an incomplete SSE event") + + +async def _zmq_stream_generator( + prefill_output: dict, + decode_generator, + reservation: _RequestReservation, + is_chat: bool = False, +): + """Return the P token followed by every streamed D token.""" + head: dict = { + "id": prefill_output["id"], + "object": "text_completion", + "created": prefill_output["created"], + "model": prefill_output["model"], + "choices": [ + { + "index": 0, + "text": prefill_output["choices"][0]["text"], + "logprobs": None, + "finish_reason": None, + } + ], + "usage": None, + } + if is_chat: + head = _completion_chunk_to_chat(head) + head["choices"][0]["delta"]["role"] = "assistant" + yield f"data: {json.dumps(head, separators=(',', ':'))}\n\n".encode() + try: + decode_generator = _zmq_stream_usage_generator(decode_generator) + generator = ( + _chat_completion_stream( + decode_generator, + response_metadata={ + "id": prefill_output["id"], + "created": prefill_output["created"], + "model": prefill_output["model"], + }, + include_role=False, + ) + if is_chat + else decode_generator + ) + async for chunk in generator: + yield chunk + finally: + reservation.release_all() + + +async def _zmq_nonstream_generator( + prefill_output: dict, + decode_generator, + reservation: _RequestReservation, + is_chat: bool = False, +): + """Merge the P token into the non-streaming D response.""" + value = b"" + try: + async for chunk in decode_generator: + value += chunk + output = json.loads(value) + output["choices"][0]["text"] = ( + prefill_output["choices"][0]["text"] + output["choices"][0]["text"] + ) + usage = output.get("usage") + if isinstance(usage, dict): + _merge_prefill_token_usage(usage) + if is_chat: + output = _completion_to_chat(output) + yield json.dumps(output, separators=(",", ":")).encode() + finally: + reservation.release_all() + + +async def _zmq_prefill_only_generator( + prefill_output: dict, + reservation: _RequestReservation, + is_chat: bool, + stream: bool, + include_usage: bool, +): + """Return the prefill token when it exhausts the requested token budget.""" + output = prefill_output.copy() + output.pop("kv_transfer_params", None) + try: + if not stream: + if is_chat: + output = _completion_to_chat(output) + yield json.dumps(output, separators=(",", ":")).encode() + return + + choice = output["choices"][0] + head = { + "id": output["id"], + "object": "text_completion", + "created": output["created"], + "model": output["model"], + "choices": [ + { + "index": choice.get("index", 0), + "text": choice.get("text", ""), + "logprobs": choice.get("logprobs"), + "finish_reason": None, + } + ], + "usage": None, + } + tail = { + **head, + "choices": [ + { + "index": choice.get("index", 0), + "text": "", + "logprobs": None, + "finish_reason": choice.get("finish_reason") or "length", + } + ], + } + if is_chat: + head = _completion_chunk_to_chat(head) + head["choices"][0]["delta"]["role"] = "assistant" + tail = _completion_chunk_to_chat(tail) + for chunk in (head, tail): + yield (f"data: {json.dumps(chunk, separators=(',', ':'))}\n\n").encode() + if include_usage and isinstance(output.get("usage"), dict): + usage = { + **tail, + "choices": [], + "usage": output["usage"], + } + yield (f"data: {json.dumps(usage, separators=(',', ':'))}\n\n").encode() + yield b"data: [DONE]\n\n" + finally: + reservation.release_all() + + +async def handle_completion( + endpoint: str, + raw_request: Request, + server: "Proxy", # noqa: F821 + is_chat: bool, +) -> JSONResponse | StreamingResponse: + """Unified completion handler for both /v1/completions and /v1/chat/completions.""" + _metrics_start = track_request_start(endpoint) + t_request_start = time.monotonic() + handler_name = "create_chat_completion" if is_chat else "create_completion" + t_prefill_done = None + decode_tracker = None + reservation = None + try: + try: + request = await raw_request.json() + except (json.JSONDecodeError, ValueError): + track_request_end(endpoint, _metrics_start) + return error_response("Invalid JSON in request body", INVALID_REQUEST, 400) + + error_resp = validate_completion_request(request, is_chat) + if error_resp: + track_request_end(endpoint, _metrics_start) + return error_resp + + prefill_instance = None + decode_instance = None + + start_time = time.time() + total_length, max_tokens, prompt_text = extract_prompt_info( + request, is_chat, server + ) + end_time = time.time() + elapsed_ms = (end_time - start_time) * 1000 + logger.info( + "Completion request received", + extra={ + "endpoint": endpoint, + "prompt_length": total_length, + "max_tokens": max_tokens, + "tokenizer_ms": round(elapsed_ms, 2), + }, + ) + + requested_model = request.get("model", "") + model_label = requested_model or "unknown" + + # Aggregated-role fast path: single forward, no P→D split + if server._is_aggregated_model(requested_model): + return await _handle_aggregated_completion( + endpoint, + request, + raw_request, + server, + requested_model, + total_length, + max_tokens, + prompt_text, + _metrics_start, + handler_name, + ) + zmq_prompt_tokens = None + if server.disaggregated_mode == "zmq": + if server.get_tokenizer(requested_model) is None: + zmq_prompt_tokens = await server.tokenize_on_backend(request, is_chat) + else: + zmq_prompt_tokens = tokenize_zmq_prompt(request, is_chat, server) + total_length = len(zmq_prompt_tokens) + kv_prepare_request = build_kv_prepare_request( + request, + is_chat, + server.disaggregated_mode, + ) + upstream_headers = None + if server.disaggregated_mode == "nixl": + request_id = raw_request.headers.get("x-request-id") or str(uuid.uuid4()) + upstream_headers = {"X-Request-Id": request_id} + + _session_id = ( + raw_request.headers.get("x-session-id") + or request.get("user") + or (raw_request.client.host if raw_request.client else None) + ) + _sched_kwargs = { + "header": raw_request.headers.get("x-session-id"), + "session_id": _session_id, + "user": request.get("user"), + "client_ip": (raw_request.client.host if raw_request.client else None), + "prompt": prompt_text, + "model": request.get("model", ""), + } + + reservation = _RequestReservation() + reservation.prefill = server.reserve( + SchedulingContext( + role="prefill", + request_len=total_length, + max_tokens=1, + **_sched_kwargs, + ) + ) + reservation.decode = server.reserve( + SchedulingContext( + role="decode", + request_len=total_length, + max_tokens=max_tokens, + **_sched_kwargs, + ) + ) + prefill_instance = reservation.prefill.address if reservation.prefill else None + decode_instance = reservation.decode.address if reservation.decode else None + + if prefill_instance is None or decode_instance is None: + logger.warning( + "No available instance", + extra={ + "endpoint": endpoint, + "prompt_length": total_length, + "model": requested_model, + }, + ) + track_request_end(endpoint, _metrics_start) + reservation.release_all() + # Check for unknown model first to return a clean 404 without + # triggering error-path side effects (logging, metrics, etc.) + if requested_model and server.registry is not None: + known_models = server.registry.get_registered_models() + if requested_model not in known_models: + return error_response( + f"The model '{requested_model}' does not exist", + INVALID_REQUEST, + 404, + ) + proxy_instance_errors_total.labels( + instance=str(prefill_instance or decode_instance or "unknown"), + error_type="no_available_instance", + model=model_label, + ).inc() + return error_response( + "No available instance can handle the request", PROXY_ERROR, 503 + ) + + zmq_request_id = None + zmq_wait_for_notification = False + if server.disaggregated_mode == "zmq": + if server.zmq_notifications is None or server.zmq_config is None: + raise HTTPException( + status_code=503, + detail="ZMQ notification listener is not ready", + ) + receiver = server.zmq_config.receivers[decode_instance] + zmq_request_id = str(uuid.uuid4()) + assert zmq_prompt_tokens is not None + kv_prepare_request = build_zmq_prepare_request( + request, + zmq_prompt_tokens, + zmq_request_id, + receiver, + server.first_token_source, + is_chat, + ) + zmq_wait_for_notification = True + await server.zmq_notifications.register(zmq_request_id) + + # Track per-instance request counters + proxy_prefill_requests_total.labels( + prefill_instance=prefill_instance, + decode_instance=decode_instance, + model=model_label, + ).inc() + proxy_prefill_queue_depth.labels( + prefill_instance=prefill_instance, + decode_instance=decode_instance, + model=model_label, + ).set(0) + proxy_decode_requests_total.labels( + prefill_instance=prefill_instance, + decode_instance=decode_instance, + model=model_label, + ).inc() + + # Track active prefill requests + proxy_prefill_active_requests.labels( + prefill_instance=prefill_instance, + decode_instance=decode_instance, + model=model_label, + ).inc() + + value = b"" + try: + async for chunk in server.forward_request( + f"http://{prefill_instance}" + f"{'/v1/completions' if server.disaggregated_mode == 'zmq' else endpoint}", # noqa: E501 + kv_prepare_request, + extra_headers=upstream_headers, + ): + value += chunk + except CancelledError: + if zmq_wait_for_notification and zmq_request_id is not None: + await server.zmq_notifications.discard(zmq_request_id) + proxy_prefill_active_requests.labels( + prefill_instance=prefill_instance, + decode_instance=decode_instance, + model=model_label, + ).dec() + raise + except HTTPException as http_exc: + if zmq_wait_for_notification and zmq_request_id is not None: + await server.zmq_notifications.discard(zmq_request_id) + reservation.release_all() + server._record_failure(prefill_instance, decode_instance) + proxy_instance_errors_total.labels( + instance=prefill_instance, + error_type="prefill_forward_error", + model=model_label, + ).inc() + proxy_prefill_active_requests.labels( + prefill_instance=prefill_instance, + decode_instance=decode_instance, + model=model_label, + ).dec() + raise http_exc + + t_prefill_done = time.monotonic() + # Prefill complete — decrement active prefill, start active decode + proxy_prefill_active_requests.labels( + prefill_instance=prefill_instance, + decode_instance=decode_instance, + model=model_label, + ).dec() + proxy_decode_active_requests.labels( + prefill_instance=prefill_instance, + decode_instance=decode_instance, + model=model_label, + ).inc() + + value = ( + value.strip().decode("utf-8").removesuffix("data: [DONE]").encode("utf-8") + ) + if server.disaggregated_mode == "nixl": + try: + prefill_output = json.loads(value) + except json.JSONDecodeError as exc: + raise HTTPException( + status_code=502, + detail="Prefill response did not contain valid JSON", + ) from exc + kv_transfer_params = prefill_output.get("kv_transfer_params") + if not kv_transfer_params: + raise HTTPException( + status_code=502, + detail="Prefill response did not contain kv_transfer_params", + ) + request = request.copy() + request["kv_transfer_params"] = kv_transfer_params + elif server.disaggregated_mode == "zmq": + prefill_output = None + first_token = None + if server.first_token_source == "prefill": + try: + prefill_output = json.loads(value) + first_token = prefill_output["kv_transfer_params"]["first_tok"] + except (json.JSONDecodeError, KeyError, TypeError) as exc: + assert zmq_request_id is not None + await server.zmq_notifications.discard(zmq_request_id) + raise HTTPException( + status_code=502, + detail=( + "Prefill response did not contain an LMCache " "first token" + ), + ) from exc + receiver = server.zmq_config.receivers[decode_instance] + if zmq_wait_for_notification: + try: + await server.zmq_notifications.wait( + zmq_request_id, + len(receiver.init_ports), + ) + except TimeoutError as exc: + raise HTTPException( + status_code=504, + detail=( + "Timed out waiting for ZMQ notification " + f"{zmq_request_id}" + ), + ) from exc + assert zmq_prompt_tokens is not None + request = build_zmq_decode_request( + request, + zmq_prompt_tokens, + first_token, + max_tokens, + is_chat, + ) + + if server.registry is not None: + server.registry.record_success(prefill_instance) + + async def streaming_response(value): + if value: + yield value + else: + yield b"" + + prefill_only = ( + server.disaggregated_mode == "zmq" + and server.first_token_source == "prefill" + and max_tokens == 1 + ) + if prefill_only: + assert prefill_output is not None + final_generator = _zmq_prefill_only_generator( + prefill_output, + reservation, + is_chat, + request.get("stream", False), + request.get("stream_options", {}).get("include_usage", False), + ) + first_token_from_p = True + else: + generator_p = streaming_response(value) + # server.forward_request() is an async generator; HTTP errors + # surface only when wrapped_generator() iterates it. + generator_d_raw = server.forward_request( + f"http://{decode_instance}" + f"{'/v1/completions' if server.disaggregated_mode == 'zmq' and is_chat else endpoint}", # noqa: E501 + request, + extra_headers=upstream_headers, + ) + decode_tracker = FirstTokenTracker(generator_d_raw) + generator_d = decode_tracker + + if ( + server.disaggregated_mode == "zmq" + and server.first_token_source == "prefill" + ): + assert prefill_output is not None + if request.get("stream", False): + final_generator = _zmq_stream_generator( + prefill_output, + generator_d, + reservation, + is_chat, + ) + else: + final_generator = _zmq_nonstream_generator( + prefill_output, + generator_d, + reservation, + is_chat, + ) + first_token_from_p = True + else: + if server.disaggregated_mode == "zmq" and is_chat: + generator_d = ( + _chat_completion_stream(generator_d) + if request.get("stream", False) + else _chat_completion_nonstream(generator_d) + ) + if request.get("stream", False): + generator_class = server.generator + else: + generator_class = server.d_first_token_generator_class + # Determine if user's first token comes from prefill or decode node. + # P_first_token_generator yields P's token first; + # D_first_token_generator + # discards P's token and yields D's tokens only. + first_token_from_p = ( + server.first_token_source == "prefill" + and generator_class is P_first_token_generator + ) + final_generator = generator_class( + generator_p, + generator_d, + reservation.release_prefill, + reservation.release_decode, + ) + media_type = ( + "text/event-stream" if request.get("stream", False) else "application/json" + ) + + async def wrapped_generator(): + try: + async for chunk in final_generator: + yield chunk + if server.registry is not None and not prefill_only: + server.registry.record_success(decode_instance) + except CancelledError: + logger.warning( + "[0]Client disconnected during %s (CancelledError)", + handler_name, + ) + except HTTPException as http_exc: + reservation.release_all() + server._record_failure(prefill_instance, decode_instance) + proxy_instance_errors_total.labels( + instance=decode_instance, + error_type="decode_forward_error", + model=model_label, + ).inc() + logger.error( + "[1] HTTPException in wrapped_generator: %s", str(http_exc.detail) + ) + raise + except Exception as e: + proxy_instance_errors_total.labels( + instance=decode_instance, + error_type="decode_forward_error", + model=model_label, + ).inc() + logger.error("[1] Exception in wrapped_generator: %s", str(e)) + raise + finally: + reservation.release_all() + if ( + prefill_instance + and decode_instance + and t_prefill_done is not None + and decode_tracker is not None + ): + record_disaggregated_metrics( + prefill_instance=prefill_instance, + decode_instance=decode_instance, + model=model_label, + t_request_start=t_request_start, + t_prefill_done=t_prefill_done, + tracker=decode_tracker, + is_streaming=request.get("stream", False), + first_token_from_prefill=first_token_from_p, + ) + # Decrement decode active gauge (only if decode was started) + if decode_instance and prefill_instance and t_prefill_done is not None: + proxy_decode_active_requests.labels( + prefill_instance=prefill_instance, + decode_instance=decode_instance, + model=model_label, + ).dec() + track_request_end(endpoint, _metrics_start) + + return StreamingResponse(wrapped_generator(), media_type=media_type) + except (HTTPException, CancelledError): + if reservation is not None: + reservation.release_all() + if decode_instance and prefill_instance and t_prefill_done is not None: + proxy_decode_active_requests.labels( + prefill_instance=prefill_instance, + decode_instance=decode_instance, + model=model_label, + ).dec() + track_request_end(endpoint, _metrics_start) + raise + except Exception: + if reservation is not None: + reservation.release_all() + if decode_instance and prefill_instance and t_prefill_done is not None: + proxy_decode_active_requests.labels( + prefill_instance=prefill_instance, + decode_instance=decode_instance, + model=model_label, + ).dec() + track_request_end(endpoint, _metrics_start) + logger.error("Error in %s: %s", handler_name, sys.exc_info()[1]) + return JSONResponse( + {"error": {"message": "Internal proxy error", "type": "proxy_error"}}, + status_code=500, + ) + + +async def _handle_aggregated_completion( + endpoint: str, + request: dict, + raw_request: Request, + server: "Proxy", # noqa: F821 + model: str, + total_length: int, + max_tokens: int, + prompt_text: str, + metrics_start: float, + handler_name: str, +) -> JSONResponse | StreamingResponse: + """Single-pass completion for aggregated-role instances.""" + reservation = server.reserve( + SchedulingContext( + role="aggregated", + model=model, + request_len=total_length, + max_tokens=max_tokens, + header=raw_request.headers.get("x-session-id"), + session_id=request.get("session_id"), + user=request.get("user"), + client_ip=(raw_request.client.host if raw_request.client else None), + prompt=prompt_text, + ) + ) + instance = reservation.address if reservation is not None else None + + logger.info( + "Aggregated %s request", + handler_name, + extra={ + "model": model, + "prompt_length": total_length, + "max_tokens": max_tokens, + "prompt": prompt_text[:200] if prompt_text else "", + "instance": instance, + }, + ) + + if instance is None: + track_request_end(endpoint, metrics_start) + # Check for unknown model + if model and server.registry is not None: + known_models = server.registry.get_registered_models() + if model not in known_models: + return error_response( + f"The model '{model}' does not exist", + INVALID_REQUEST, + 404, + ) + return error_response( + "No available instance can handle the request", + PROXY_ERROR, + 503, + ) + + url = f"http://{instance}{endpoint}" + + if request.get("stream", False): + generator = server.forward_request(url, request) + + async def wrapped(): + _ok = False + try: + async for chunk in generator: + yield chunk + _ok = True + except CancelledError: + _ok = False + logger.warning( + "Client disconnected during aggregated %s (CancelledError)", + handler_name, + ) + except Exception as e: + _ok = False + logger.error("Exception in aggregated stream: %s", str(e)) + if server.registry is not None: + server.registry.record_failure(instance) + raise + finally: + reservation.release() + if _ok and server.registry is not None: + server.registry.record_success(instance) + track_request_end(endpoint, metrics_start) + + return StreamingResponse(wrapped(), media_type="text/event-stream") + else: + # Non-streaming: forward request using server.forward_request() + # for consistent auth, timeouts, and connection handling. + try: + value = b"" + async for chunk in server.forward_request(url, request): + value += chunk + data = json.loads(value) + # Detect error responses from upstream. forward_request yields + # raw bytes without HTTP status, so inspect the parsed JSON. + # OpenAI error format: {"error": {"message": ..., "type": ..., "code": ...}} + # where "code" can be null, a string like "invalid_api_key", or an int. + if "error" in data: + err = data["error"] + err_type = err.get("type", "") + # Map error type to HTTP status code + if err_type == "invalid_request_error": + status_code = 400 + elif err_type == "authentication_error": + status_code = 401 + elif err_type == "not_found_error": + status_code = 404 + elif err_type == "rate_limit_error": + status_code = 429 + else: + status_code = 502 + else: + status_code = 200 + reservation.release() + if status_code < 400 and server.registry is not None: + server.registry.record_success(instance) + elif status_code >= 400 and server.registry is not None: + server.registry.record_failure(instance) + track_request_end(endpoint, metrics_start) + return JSONResponse(data, status_code=status_code) + except HTTPException as http_exc: + reservation.release() + if server.registry is not None: + server.registry.record_failure(instance) + track_request_end(endpoint, metrics_start) + return error_response( + str(http_exc.detail), + PROXY_ERROR, + http_exc.status_code, + ) + except CancelledError: + reservation.release() + raise + except Exception as e: + logger.error("Error in aggregated non-streaming: %s", str(e)) + reservation.release() + if server.registry is not None: + server.registry.record_failure(instance) + track_request_end(endpoint, metrics_start) + return error_response( + "Internal proxy error", + PROXY_ERROR, + 500, + ) + + +async def validate_json_content_type(raw_request: Request) -> None: + """Reject unsupported content types before invoking either completion handler.""" + # Dependencies must raise to short-circuit a request, including custom handlers. + content_type = raw_request.headers.get("content-type", "").lower() + if content_type != "application/json": + raise HTTPException( + status_code=415, + detail="Unsupported Media Type: Only 'application/json' is allowed", + ) diff --git a/xpyd/proxy.py b/xpyd/proxy.py index 72ad23e..ccfccef 100644 --- a/xpyd/proxy.py +++ b/xpyd/proxy.py @@ -12,25 +12,24 @@ non-streaming). """ -import argparse import asyncio import json import logging import os -import sys from collections.abc import AsyncGenerator from pathlib import Path from typing import Any, Callable, Optional import aiohttp import uvicorn -from fastapi import APIRouter, FastAPI, HTTPException, Request -from fastapi.middleware.cors import CORSMiddleware +from fastapi import APIRouter, HTTPException, Request from fastapi.responses import JSONResponse, StreamingResponse from transformers import AutoTokenizer +from xpyd.app import create_app +from xpyd.cli import load_config +from xpyd.completion_handler import D_first_token_generator, P_first_token_generator from xpyd.config import ProxyConfig -from xpyd.discovery import NodeDiscovery from xpyd.errors import INVALID_REQUEST, PROXY_ERROR, SERVER_ERROR, error_response from xpyd.health_monitor import HealthMonitor from xpyd.registry import InstanceRegistry @@ -82,49 +81,6 @@ def format(self, record: logging.LogRecord) -> str: ) -async def P_first_token_generator( - generator_p: AsyncGenerator[bytes, None], - generator_d: AsyncGenerator[bytes, None], - release_prefill: Callable[[], None], - release_decode: Callable[[], None], -) -> AsyncGenerator[bytes, None]: - first_decode = True - - try: - async for chunk in generator_p: - yield chunk - finally: - release_prefill() - - try: - async for chunk in generator_d: - if first_decode: - first_decode = False - continue - yield chunk - finally: - release_decode() - - -async def D_first_token_generator( - generator_p: AsyncGenerator[bytes, None], - generator_d: AsyncGenerator[bytes, None], - release_prefill: Callable[[], None], - release_decode: Callable[[], None], -) -> AsyncGenerator[bytes, None]: - try: - async for _ in generator_p: - continue - finally: - release_prefill() - - try: - async for chunk in generator_d: - yield chunk - finally: - release_decode() - - class Proxy: def __init__( @@ -950,94 +906,7 @@ def __init__( ) def run_server(self) -> None: - discovery = NodeDiscovery( - prefill_instances=self._all_prefill, - decode_instances=self._all_decode, - probe_interval=self.config.probe_interval_seconds, - wait_timeout=self.config.wait_timeout_seconds, - heartbeat_interval=self.config.heartbeat_interval_seconds, - registry=self.registry, - aggregated_instances=self._all_aggregated, - on_model_discovered=lambda model: asyncio.to_thread( - self.proxy_instance.ensure_tokenizer, model - ), - ) - self.proxy_instance.discovery = discovery - self.proxy_instance.health_monitor = self.health_monitor - - app = FastAPI() - app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_credentials=False, - allow_methods=["*"], - allow_headers=["*"], - ) - - @app.middleware("http") - async def _check_readiness(request: Request, call_next): - # Allow health/status/metrics endpoints through always - path = request.url.path - if path in ( - "/health", - "/ping", - "/status", - "/metrics", - "/instances/add", - "/instances/remove", - ) or path.startswith("/status/"): - return await call_next(request) - if not discovery.is_ready: - return error_response("Waiting for backend nodes", PROXY_ERROR, 503) - return await call_next(request) - - @app.on_event("startup") - async def _start_discovery(): - if self.config.disaggregated_mode == "zmq": - from xpyd.zmq_notifications import ZmqNotificationListener - - zmq_config = self.config.zmq - assert zmq_config is not None - listener = ZmqNotificationListener( - zmq_config.host, - zmq_config.port, - zmq_config.notification_timeout_seconds, - ) - await listener.start() - self.proxy_instance.zmq_notifications = listener - await discovery.start() - if self.health_monitor: - await self.health_monitor.start() - - @app.on_event("shutdown") - async def _stop_discovery(): - await discovery.stop() - if self.proxy_instance.zmq_notifications is not None: - await self.proxy_instance.zmq_notifications.stop() - if self.health_monitor: - await self.health_monitor.stop() - - @app.get("/status/instances") - async def _instance_status(): - """Return per-instance health and circuit breaker state.""" - result: dict[str, list] = { - "prefill_instances": [], - "decode_instances": [], - "aggregated_instances": [], - } - for info in self.registry.get_all_instances(): - result[f"{info.role}_instances"].append( - { - "address": info.address, - "status": info.status.value, - "circuit": info.circuit_breaker_state.value, - "active_requests": info.active_request_count, - "last_check": info.last_health_check, - } - ) - return JSONResponse(result) - - app.include_router(self.proxy_instance.router) + app = create_app(self.config, self.proxy_instance, self.health_monitor) config = uvicorn.Config( app, host=self.config.host, @@ -1049,267 +918,12 @@ async def _instance_status(): server.run() -_VERSION = "1.8.0" - - -def _valid_port(value: str) -> int: - """Parse a TCP port for argparse with a concise user-facing error.""" - try: - port = int(value) - except ValueError as exc: - raise argparse.ArgumentTypeError("port must be an integer") from exc - if not 1 <= port <= 65535: - raise argparse.ArgumentTypeError("port must be between 1 and 65535") - return port - - -def _build_parser(): - """Build the subcommand argument parser for the proxy CLI.""" - parser = argparse.ArgumentParser( - prog="xpyd", - description="xPyD — lightweight disaggregated proxy server", - ) - parser.add_argument( - "--version", - "-V", - action="version", - version=f"%(prog)s {_VERSION}", - ) - - subparsers = parser.add_subparsers(dest="command") - - proxy_parser = subparsers.add_parser( - "proxy", - prog="xpyd [proxy]", - help="Start the proxy server (default command)", - ) - proxy_parser.add_argument( - "--config", - "-c", - type=str, - default=None, - help="Path to YAML configuration file", - ) - proxy_parser.add_argument( - "--validate-config", - type=str, - default=None, - metavar="FILE", - help="Validate YAML config and exit (no server start)", - ) - proxy_parser.add_argument( - "--init-config", - nargs="?", - const="./xpyd.yaml", - default=None, - metavar="PATH", - help="Generate xpyd.yaml, optionally through an interactive wizard " - "(default path: ./xpyd.yaml)", - ) - proxy_parser.add_argument( - "--force", - action="store_true", - help="Overwrite an existing --init-config destination", - ) - proxy_parser.add_argument( - "--port", - type=_valid_port, - default=None, - help="Override the port from config", - ) - proxy_parser.add_argument( - "--log-level", - choices=("debug", "info", "warning", "error"), - default=None, - dest="log_level", - help="Override log level: debug|info|warning|error", - ) - proxy_parser.add_argument( - "--disaggregated-mode", - choices=("direct", "nixl", "zmq"), - default=None, - help="disaggregated transfer mode (default: config value or direct)", - ) - proxy_parser.add_argument( - "--first-token-source", - choices=("prefill", "decode"), - default=None, - help="Backend that provides the first client-visible token", - ) - - # fix-config subcommand - fix_parser = subparsers.add_parser( - "fix-config", - help="Auto-fix common config mistakes", - ) - fix_parser.add_argument( - "config_path", - type=str, - help="Path to YAML configuration file to fix", - ) - fix_parser.add_argument( - "--write", - action="store_true", - default=False, - help="Write fixes back to the file (creates timestamped .bak backup). " - "Note: does not preserve YAML comments or formatting.", - ) - fix_parser.add_argument( - "--interactive", - action="store_true", - default=False, - help="Interactively acknowledge ambiguous suggestions one by one", - ) - - return parser - - -def _normalize_cli_args(argv: list[str]) -> list[str]: - """Treat omitted subcommands as the default ``proxy`` command.""" - if argv and argv[0] == "proxy": - if len(argv) > 1 and argv[1] in {"fix-config", "--version", "-V"}: - return argv[1:] - return argv - if argv and argv[0] in {"fix-config", "--version", "-V"}: - return argv - return ["proxy", *argv] - - -def _resolve_config_path(args): - """Resolve the config file path: --config > XPYD_CONFIG env > ./xpyd.yaml. - - Returns the path string, or ``None`` when no config can be found. - """ - if args.config: - return args.config - env_config = os.environ.get("XPYD_CONFIG") - if env_config: - return env_config - default_path = "./xpyd.yaml" - if os.path.exists(default_path): - print( - "No config specified; found ./xpyd.yaml and using it.", - flush=True, - ) - return default_path - return None - - -def _print_config_summary(config: ProxyConfig) -> None: - """Print topology-aware details for a validated configuration.""" - if config.instances is None: - topology = "disaggregated" if config.prefill else "decode-only" - print(f" topology: {topology}") - if config.prefill: - print(f" transfer: {config.disaggregated_mode}") - print(f" model: {config.model}") - print(f" prefill: {len(config.prefill)} instances") - print(f" decode: {len(config.decode)} instances") - else: - roles = {instance.role for instance in config.instances} - topology = "aggregated" if roles == {"aggregated"} else "disaggregated" - print(f" topology: {topology}") - if topology == "disaggregated": - print(f" transfer: {config.disaggregated_mode}") - models = sorted( - {instance.model for instance in config.instances if instance.model} - ) - print(f" models: {', '.join(models) if models else 'auto-detected'}") - for role in ("aggregated", "prefill", "decode"): - count = sum(instance.role == role for instance in config.instances) - if count: - print(f" {role}: {count} instances") - print(f" port: {config.port}") - print(f" log_level: {config.log_level}") - - -def _apply_cli_overrides(config: ProxyConfig, args) -> ProxyConfig: - """Apply explicitly supplied CLI values and revalidate the result.""" - overrides = { - field: value - for field in ( - "port", - "log_level", - "disaggregated_mode", - "first_token_source", - ) - if (value := getattr(args, field, None)) is not None - } - if not overrides: - return config - - model_schedulers = config._model_schedulers.copy() - updated = ProxyConfig.model_validate({**config.model_dump(), **overrides}) - updated._model_schedulers = model_schedulers - return updated - - def main() -> None: """Entry point for the ``xpyd`` CLI.""" - from xpyd.init_config import generate_config - - parser = _build_parser() - args = parser.parse_args(_normalize_cli_args(sys.argv[1:])) - - if args.command == "fix-config": - from xpyd.config_fixer import run_fix_config - - sys.exit( - run_fix_config( - args.config_path, - write=args.write, - interactive=args.interactive, - ) - ) - elif args.command == "proxy": - if args.force and args.init_config is None: - parser.error("--force can only be used with --init-config") - - # --init-config: generate template and exit - if args.init_config is not None: - output_path = Path(args.init_config) - if output_path.exists() and not args.force: - parser.error( - f"config already exists: {output_path}; use --force to overwrite" - ) - try: - generate_config(args.init_config) - except OSError as exc: - parser.exit(1, f"xpyd: failed to write config: {exc}\n") - return - - # --validate-config: validate and exit - if args.validate_config: - config_path = args.validate_config - try: - config = ProxyConfig.from_yaml(config_path) - print(f"Config is valid: {config_path}") - _print_config_summary(config) - sys.exit(0) - except Exception as exc: - print(f"Config validation failed: {exc}", file=sys.stderr) - sys.exit(1) - - # Resolve config path with precedence - config_path = _resolve_config_path(args) - if config_path is None: - config_path = "./xpyd.yaml" - print( - "No config specified and ./xpyd.yaml was not found; " - "starting config initialization." - ) - generate_config(config_path) - return - try: - config = ProxyConfig.from_yaml(config_path) - config = _apply_cli_overrides(config, args) - except (OSError, ValueError) as exc: - parser.exit(1, f"xpyd: failed to load config: {exc}\n") - + config = load_config() + if config is not None: proxy_server = ProxyServer(config=config) proxy_server.run_server() - else: - parser.print_help() if __name__ == "__main__": diff --git a/xpyd/routes/__init__.py b/xpyd/routes/__init__.py index db39b49..f0ef4f8 100644 --- a/xpyd/routes/__init__.py +++ b/xpyd/routes/__init__.py @@ -1,17 +1,44 @@ # SPDX-License-Identifier: Apache-2.0 """Route registration aggregator for MicroDisaggregatedProxy.""" +from typing import TYPE_CHECKING + from fastapi import APIRouter -from .admin import register as register_admin +from .chat_completions import register as register_chat_completions from .completions import register as register_completions -from .forward import register as register_proxy +from .detokenize import register as register_detokenize +from .embeddings import register as register_embeddings from .health import register as register_health +from .instances import register as register_instances +from .invocations import register as register_invocations +from .metrics import register as register_metrics +from .models import register as register_models +from .pooling import register as register_pooling +from .rerank import register as register_rerank +from .score import register as register_score +from .status import register as register_status +from .tokenize import register as register_tokenize +from .version import register as register_version + +if TYPE_CHECKING: + from xpyd.proxy import Proxy -def register_routes(router: APIRouter, server) -> None: +def register_routes(router: APIRouter, server: "Proxy") -> None: """Register all route handlers on *router*, using *server* for state.""" register_completions(router, server) - register_proxy(router, server) - register_admin(router, server) + register_chat_completions(router, server) + register_tokenize(router, server) + register_detokenize(router, server) + register_embeddings(router, server) + register_pooling(router, server) + register_score(router, server) + register_rerank(router, server) + register_invocations(router, server) + register_status(router, server) + register_instances(router, server) register_health(router, server) + register_models(router, server) + register_version(router, server) + register_metrics(router, server) diff --git a/xpyd/routes/chat_completions.py b/xpyd/routes/chat_completions.py new file mode 100644 index 0000000..c2f2098 --- /dev/null +++ b/xpyd/routes/chat_completions.py @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The /v1/chat/completions endpoint.""" + +from typing import TYPE_CHECKING + +from fastapi import APIRouter, Depends, Request + +from xpyd.completion_handler import handle_completion, validate_json_content_type + +if TYPE_CHECKING: + from xpyd.proxy import Proxy + + +def register(router: APIRouter, server: "Proxy") -> None: + async def create_chat_completion(raw_request: Request): + return await handle_completion( + "/v1/chat/completions", raw_request, server, is_chat=True + ) + + router.post( + "/v1/chat/completions", + dependencies=[Depends(validate_json_content_type)], + )(server.custom_create_chat_completion or create_chat_completion) + router.options("/v1/chat/completions")(lambda: None) diff --git a/xpyd/routes/completions.py b/xpyd/routes/completions.py index e612e2d..289e473 100644 --- a/xpyd/routes/completions.py +++ b/xpyd/routes/completions.py @@ -1,1164 +1,24 @@ # SPDX-License-Identifier: Apache-2.0 -"""Completion route handlers.""" +"""The /v1/completions endpoint.""" -from __future__ import annotations - -import json -import logging -import sys -import time -import uuid -from asyncio import CancelledError -from collections.abc import Mapping from typing import TYPE_CHECKING -from fastapi import APIRouter, Depends, HTTPException, Request -from fastapi.responses import JSONResponse, StreamingResponse +from fastapi import APIRouter, Depends, Request -from xpyd.errors import INVALID_REQUEST, PROXY_ERROR, error_response -from xpyd.scheduler import Reservation, SchedulingContext +from xpyd.completion_handler import handle_completion, validate_json_content_type if TYPE_CHECKING: from xpyd.proxy import Proxy -from xpyd.metrics import ( - FirstTokenTracker, - proxy_decode_active_requests, - proxy_decode_requests_total, - proxy_instance_errors_total, - proxy_prefill_active_requests, - proxy_prefill_queue_depth, - proxy_prefill_requests_total, - record_disaggregated_metrics, - track_request_end, - track_request_start, -) - -logger = logging.getLogger("xpyd.proxy") - -_CHAT_ONLY_REQUEST_FIELDS = { - "messages", - "max_completion_tokens", - "tools", - "tool_choice", - "parallel_tool_calls", - "chat_template", - "chat_template_kwargs", - "add_generation_prompt", - "continue_final_message", -} - - -class _RequestReservation: - """Release each P/D scheduling reservation at most once.""" - - def __init__( - self, - prefill: Reservation | None = None, - decode: Reservation | None = None, - ) -> None: - self.prefill = prefill - self.decode = decode - - def release_prefill(self) -> None: - if self.prefill is not None: - self.prefill.release() - - def release_decode(self) -> None: - if self.decode is not None: - self.decode.release() - - def release_all(self) -> None: - self.release_prefill() - self.release_decode() - - -# --------------------------------------------------------------------------- -# Pure helper functions (no server dependency or explicit server param) -# --------------------------------------------------------------------------- - - -def validate_completion_request(request: dict, is_chat: bool) -> JSONResponse | None: - """Validate required fields. Returns JSONResponse on error, None on success.""" - if is_chat: - if "messages" not in request: - return error_response( - "Missing required field: messages", INVALID_REQUEST, 400 - ) - if not isinstance(request["messages"], list): - return error_response("Field messages must be a list", INVALID_REQUEST, 400) - else: - if "prompt" not in request: - return error_response( - "Missing required field: prompt", INVALID_REQUEST, 400 - ) - return None - - -def extract_prompt_info( - request: dict, is_chat: bool, server: Proxy -) -> tuple[int, int, str]: - """Extract prompt metrics. Returns (total_length, max_tokens, prompt_text).""" - model = request.get("model", "") - if is_chat: - total_length = 0 - prompt_parts = [] - for msg in request["messages"]: - content = msg.get("content") - if content is None: - continue - if isinstance(content, str): - total_length += server.get_total_token_length(content, model) - prompt_parts.append(content) - elif isinstance(content, list): - for part in content: - if isinstance(part, dict) and part.get("type") == "text": - text = part.get("text", "") - total_length += server.get_total_token_length(text, model) - prompt_parts.append(text) - max_tokens = request.get("max_completion_tokens", 0) - if max_tokens == 0: - max_tokens = request.get("max_tokens", 0) - prompt_text = " ".join(prompt_parts) - else: - prompt = request.get("prompt") - total_length = server.get_total_token_length(prompt, model) - max_tokens = request.get("max_tokens", 0) - prompt_text = prompt if isinstance(prompt, str) else str(prompt) - return total_length, max_tokens, prompt_text - - -def build_kv_prepare_request( - request: dict, - is_chat: bool, - disaggregated_mode: str = "direct", -) -> dict: - """Build the KV-prepare request with max_tokens=1.""" - kv_prepare_request = request.copy() - kv_prepare_request["max_tokens"] = 1 - if is_chat: - kv_prepare_request["max_completion_tokens"] = 1 - if disaggregated_mode == "nixl": - kv_prepare_request["stream"] = False - kv_prepare_request.pop("stream_options", None) - kv_prepare_request["kv_transfer_params"] = { - "do_remote_decode": True, - "do_remote_prefill": False, - "remote_engine_id": None, - "remote_block_ids": None, - "remote_host": None, - "remote_port": None, - } - return kv_prepare_request - - -def build_zmq_prepare_request( - request: dict, - prompt_tokens: list[int], - request_id: str, - receiver, - first_token_source: str, - is_chat: bool = False, -) -> dict: - """Build an LMCache sender request for a preselected decode instance.""" - prepare = request.copy() - if is_chat: - for field in _CHAT_ONLY_REQUEST_FIELDS: - prepare.pop(field, None) - prepare["prompt"] = prompt_tokens - prepare["max_tokens"] = 1 - prepare["stream"] = False - prepare.pop("stream_options", None) - transfer_params = { - "disagg_spec": { - "req_id": request_id, - "receiver_host": receiver.host, - "receiver_init_port": receiver.init_ports, - "receiver_alloc_port": receiver.alloc_ports, - } - } - if first_token_source == "prefill": - transfer_params["ret_first_tok"] = True - prepare["kv_transfer_params"] = transfer_params - return prepare - - -def tokenize_zmq_prompt(request: dict, is_chat: bool, server: Proxy) -> list[int]: - """Tokenize the exact prompt used by the LMCache sender and receiver.""" - tokenizer = server.get_tokenizer(request.get("model", "")) - if tokenizer is None: - raise ValueError( - "ZMQ prompt tokenization requires a tokenizer for model " - f"{request.get('model', '')!r}" - ) - if not is_chat: - prompt = request["prompt"] - return list(prompt) if isinstance(prompt, list) else tokenizer.encode(prompt) - - template_kwargs = dict(request.get("chat_template_kwargs") or {}) - template_kwargs["tokenize"] = True - continue_final_message = request.get("continue_final_message", False) - template_kwargs["add_generation_prompt"] = request.get( - "add_generation_prompt", not continue_final_message - ) - if "continue_final_message" in request: - template_kwargs["continue_final_message"] = continue_final_message - if request.get("chat_template") is not None: - template_kwargs["chat_template"] = request["chat_template"] - if request.get("tools") is not None: - template_kwargs["tools"] = request["tools"] - tokenized = tokenizer.apply_chat_template(request["messages"], **template_kwargs) - if isinstance(tokenized, Mapping): - tokenized = tokenized.get("input_ids") - if tokenized is None: - raise ValueError("Chat template output did not contain input_ids") - if hasattr(tokenized, "tolist"): - tokenized = tokenized.tolist() - if ( - isinstance(tokenized, list) - and len(tokenized) == 1 - and isinstance(tokenized[0], list) - ): - tokenized = tokenized[0] - return list(tokenized) - - -def build_zmq_decode_request( - request: dict, - prompt_tokens: list[int], - first_token: int | None, - max_tokens: int, - is_chat: bool, -) -> dict: - """Build a completions request whose prompt exactly matches transferred KV.""" - decode = request.copy() - if is_chat: - for field in _CHAT_ONLY_REQUEST_FIELDS: - decode.pop(field, None) - if max_tokens > 0: - decode["max_tokens"] = max_tokens - decode["prompt"] = list(prompt_tokens) - if first_token is not None: - decode["prompt"].append(first_token) - if max_tokens > 0: - decode["max_tokens"] = max(1, max_tokens - 1) - decode.pop("kv_transfer_params", None) - return decode - - -def _completion_to_chat(output: dict) -> dict: - """Convert a text completion response to OpenAI chat completion shape.""" - result = output.copy() - result["object"] = "chat.completion" - result["choices"] = [ - { - "index": choice.get("index", index), - "message": { - "role": "assistant", - "content": choice.get("text", ""), - }, - "logprobs": choice.get("logprobs"), - "finish_reason": choice.get("finish_reason"), - } - for index, choice in enumerate(output.get("choices", [])) - ] - return result - - -def _completion_chunk_to_chat( - output: dict, - *, - response_metadata: dict | None = None, -) -> dict: - """Convert a text completion stream chunk to chat completion chunk shape.""" - result = output.copy() - if response_metadata: - result.update(response_metadata) - result["object"] = "chat.completion.chunk" - result["choices"] = [ - { - "index": choice.get("index", index), - "delta": {"content": choice.get("text", "")}, - "logprobs": choice.get("logprobs"), - "finish_reason": choice.get("finish_reason"), - } - for index, choice in enumerate(output.get("choices", [])) - ] - return result - - -async def _chat_completion_stream( - completion_generator, - *, - response_metadata: dict | None = None, - include_role: bool = True, -): - """Convert arbitrarily chunked completion SSE data to chat completion SSE.""" - buffer = "" - role_pending = include_role - async for chunk in completion_generator: - buffer += chunk.decode("utf-8") - while "\n\n" in buffer: - event, buffer = buffer.split("\n\n", 1) - for line in event.splitlines(): - if not line.startswith("data:"): - continue - payload = line[5:].strip() - if payload == "[DONE]": - yield b"data: [DONE]\n\n" - continue - output = _completion_chunk_to_chat( - json.loads(payload), - response_metadata=response_metadata, - ) - if role_pending and output["choices"]: - output["choices"][0]["delta"]["role"] = "assistant" - role_pending = False - yield ( - f"data: {json.dumps(output, separators=(',', ':'))}\n\n" - ).encode() - if buffer.strip(): - raise ValueError("Decode response ended with an incomplete SSE event") - - -async def _chat_completion_nonstream(completion_generator): - """Convert a non-streaming text completion response to chat shape.""" - value = b"" - async for chunk in completion_generator: - value += chunk - yield json.dumps( - _completion_to_chat(json.loads(value)), - separators=(",", ":"), - ).encode() - - -def _merge_prefill_token_usage(usage: dict) -> None: - """Move the appended prefill token from prompt usage to completion usage.""" - if usage.get("prompt_tokens", 0) > 0: - usage["prompt_tokens"] -= 1 - usage["completion_tokens"] = usage.get("completion_tokens", 0) + 1 - usage["total_tokens"] = usage.get("prompt_tokens", 0) + usage.get( - "completion_tokens", 0 - ) - - -async def _zmq_stream_usage_generator(decode_generator): - """Correct usage events after the prefill token was appended to the prompt.""" - buffer = "" - async for chunk in decode_generator: - buffer += chunk.decode("utf-8") - while "\n\n" in buffer: - event, buffer = buffer.split("\n\n", 1) - lines = [] - for line in event.splitlines(): - if line.startswith("data:"): - payload = line[5:].strip() - if payload != "[DONE]": - output = json.loads(payload) - if isinstance(output.get("usage"), dict): - _merge_prefill_token_usage(output["usage"]) - line = "data: " + json.dumps(output, separators=(",", ":")) - lines.append(line) - yield ("\n".join(lines) + "\n\n").encode() - if buffer.strip(): - raise ValueError("Decode response ended with an incomplete SSE event") - - -async def _zmq_stream_generator( - prefill_output: dict, - decode_generator, - reservation: _RequestReservation, - is_chat: bool = False, -): - """Return the P token followed by every streamed D token.""" - head: dict = { - "id": prefill_output["id"], - "object": "text_completion", - "created": prefill_output["created"], - "model": prefill_output["model"], - "choices": [ - { - "index": 0, - "text": prefill_output["choices"][0]["text"], - "logprobs": None, - "finish_reason": None, - } - ], - "usage": None, - } - if is_chat: - head = _completion_chunk_to_chat(head) - head["choices"][0]["delta"]["role"] = "assistant" - yield f"data: {json.dumps(head, separators=(',', ':'))}\n\n".encode() - try: - decode_generator = _zmq_stream_usage_generator(decode_generator) - generator = ( - _chat_completion_stream( - decode_generator, - response_metadata={ - "id": prefill_output["id"], - "created": prefill_output["created"], - "model": prefill_output["model"], - }, - include_role=False, - ) - if is_chat - else decode_generator - ) - async for chunk in generator: - yield chunk - finally: - reservation.release_all() - - -async def _zmq_nonstream_generator( - prefill_output: dict, - decode_generator, - reservation: _RequestReservation, - is_chat: bool = False, -): - """Merge the P token into the non-streaming D response.""" - value = b"" - try: - async for chunk in decode_generator: - value += chunk - output = json.loads(value) - output["choices"][0]["text"] = ( - prefill_output["choices"][0]["text"] + output["choices"][0]["text"] - ) - usage = output.get("usage") - if isinstance(usage, dict): - _merge_prefill_token_usage(usage) - if is_chat: - output = _completion_to_chat(output) - yield json.dumps(output, separators=(",", ":")).encode() - finally: - reservation.release_all() - - -async def _zmq_prefill_only_generator( - prefill_output: dict, - reservation: _RequestReservation, - is_chat: bool, - stream: bool, - include_usage: bool, -): - """Return the prefill token when it exhausts the requested token budget.""" - output = prefill_output.copy() - output.pop("kv_transfer_params", None) - try: - if not stream: - if is_chat: - output = _completion_to_chat(output) - yield json.dumps(output, separators=(",", ":")).encode() - return - - choice = output["choices"][0] - head = { - "id": output["id"], - "object": "text_completion", - "created": output["created"], - "model": output["model"], - "choices": [ - { - "index": choice.get("index", 0), - "text": choice.get("text", ""), - "logprobs": choice.get("logprobs"), - "finish_reason": None, - } - ], - "usage": None, - } - tail = { - **head, - "choices": [ - { - "index": choice.get("index", 0), - "text": "", - "logprobs": None, - "finish_reason": choice.get("finish_reason") or "length", - } - ], - } - if is_chat: - head = _completion_chunk_to_chat(head) - head["choices"][0]["delta"]["role"] = "assistant" - tail = _completion_chunk_to_chat(tail) - for chunk in (head, tail): - yield (f"data: {json.dumps(chunk, separators=(',', ':'))}\n\n").encode() - if include_usage and isinstance(output.get("usage"), dict): - usage = { - **tail, - "choices": [], - "usage": output["usage"], - } - yield (f"data: {json.dumps(usage, separators=(',', ':'))}\n\n").encode() - yield b"data: [DONE]\n\n" - finally: - reservation.release_all() - - -async def handle_completion( - endpoint: str, raw_request: Request, server: Proxy, is_chat: bool -) -> JSONResponse | StreamingResponse: - """Unified completion handler for both /v1/completions and /v1/chat/completions.""" - _metrics_start = track_request_start(endpoint) - t_request_start = time.monotonic() - handler_name = "create_chat_completion" if is_chat else "create_completion" - t_prefill_done = None - decode_tracker = None - reservation = None - try: - try: - request = await raw_request.json() - except (json.JSONDecodeError, ValueError): - track_request_end(endpoint, _metrics_start) - return error_response("Invalid JSON in request body", INVALID_REQUEST, 400) - - error_resp = validate_completion_request(request, is_chat) - if error_resp: - track_request_end(endpoint, _metrics_start) - return error_resp - - prefill_instance = None - decode_instance = None - - start_time = time.time() - total_length, max_tokens, prompt_text = extract_prompt_info( - request, is_chat, server - ) - end_time = time.time() - elapsed_ms = (end_time - start_time) * 1000 - logger.info( - "Completion request received", - extra={ - "endpoint": endpoint, - "prompt_length": total_length, - "max_tokens": max_tokens, - "tokenizer_ms": round(elapsed_ms, 2), - }, - ) - - requested_model = request.get("model", "") - model_label = requested_model or "unknown" - - # Aggregated-role fast path: single forward, no P→D split - if server._is_aggregated_model(requested_model): - return await _handle_aggregated_completion( - endpoint, - request, - raw_request, - server, - requested_model, - total_length, - max_tokens, - prompt_text, - _metrics_start, - handler_name, - ) - zmq_prompt_tokens = None - if server.disaggregated_mode == "zmq": - if server.get_tokenizer(requested_model) is None: - zmq_prompt_tokens = await server.tokenize_on_backend(request, is_chat) - else: - zmq_prompt_tokens = tokenize_zmq_prompt(request, is_chat, server) - total_length = len(zmq_prompt_tokens) - kv_prepare_request = build_kv_prepare_request( - request, - is_chat, - server.disaggregated_mode, - ) - upstream_headers = None - if server.disaggregated_mode == "nixl": - request_id = raw_request.headers.get("x-request-id") or str(uuid.uuid4()) - upstream_headers = {"X-Request-Id": request_id} - - _session_id = ( - raw_request.headers.get("x-session-id") - or request.get("user") - or (raw_request.client.host if raw_request.client else None) - ) - _sched_kwargs = { - "header": raw_request.headers.get("x-session-id"), - "session_id": _session_id, - "user": request.get("user"), - "client_ip": (raw_request.client.host if raw_request.client else None), - "prompt": prompt_text, - "model": request.get("model", ""), - } - - reservation = _RequestReservation() - reservation.prefill = server.reserve( - SchedulingContext( - role="prefill", - request_len=total_length, - max_tokens=1, - **_sched_kwargs, - ) - ) - reservation.decode = server.reserve( - SchedulingContext( - role="decode", - request_len=total_length, - max_tokens=max_tokens, - **_sched_kwargs, - ) - ) - prefill_instance = reservation.prefill.address if reservation.prefill else None - decode_instance = reservation.decode.address if reservation.decode else None - - if prefill_instance is None or decode_instance is None: - logger.warning( - "No available instance", - extra={ - "endpoint": endpoint, - "prompt_length": total_length, - "model": requested_model, - }, - ) - track_request_end(endpoint, _metrics_start) - reservation.release_all() - # Check for unknown model first to return a clean 404 without - # triggering error-path side effects (logging, metrics, etc.) - if requested_model and server.registry is not None: - known_models = server.registry.get_registered_models() - if requested_model not in known_models: - return error_response( - f"The model '{requested_model}' does not exist", - INVALID_REQUEST, - 404, - ) - proxy_instance_errors_total.labels( - instance=str(prefill_instance or decode_instance or "unknown"), - error_type="no_available_instance", - model=model_label, - ).inc() - return error_response( - "No available instance can handle the request", PROXY_ERROR, 503 - ) - - zmq_request_id = None - zmq_wait_for_notification = False - if server.disaggregated_mode == "zmq": - if server.zmq_notifications is None or server.zmq_config is None: - raise HTTPException( - status_code=503, - detail="ZMQ notification listener is not ready", - ) - receiver = server.zmq_config.receivers[decode_instance] - zmq_request_id = str(uuid.uuid4()) - assert zmq_prompt_tokens is not None - kv_prepare_request = build_zmq_prepare_request( - request, - zmq_prompt_tokens, - zmq_request_id, - receiver, - server.first_token_source, - is_chat, - ) - zmq_wait_for_notification = True - await server.zmq_notifications.register(zmq_request_id) - - # Track per-instance request counters - proxy_prefill_requests_total.labels( - prefill_instance=prefill_instance, - decode_instance=decode_instance, - model=model_label, - ).inc() - proxy_prefill_queue_depth.labels( - prefill_instance=prefill_instance, - decode_instance=decode_instance, - model=model_label, - ).set(0) - proxy_decode_requests_total.labels( - prefill_instance=prefill_instance, - decode_instance=decode_instance, - model=model_label, - ).inc() - - # Track active prefill requests - proxy_prefill_active_requests.labels( - prefill_instance=prefill_instance, - decode_instance=decode_instance, - model=model_label, - ).inc() - - value = b"" - try: - async for chunk in server.forward_request( - f"http://{prefill_instance}" - f"{'/v1/completions' if server.disaggregated_mode == 'zmq' else endpoint}", # noqa: E501 - kv_prepare_request, - extra_headers=upstream_headers, - ): - value += chunk - except CancelledError: - if zmq_wait_for_notification and zmq_request_id is not None: - await server.zmq_notifications.discard(zmq_request_id) - proxy_prefill_active_requests.labels( - prefill_instance=prefill_instance, - decode_instance=decode_instance, - model=model_label, - ).dec() - raise - except HTTPException as http_exc: - if zmq_wait_for_notification and zmq_request_id is not None: - await server.zmq_notifications.discard(zmq_request_id) - reservation.release_all() - server._record_failure(prefill_instance, decode_instance) - proxy_instance_errors_total.labels( - instance=prefill_instance, - error_type="prefill_forward_error", - model=model_label, - ).inc() - proxy_prefill_active_requests.labels( - prefill_instance=prefill_instance, - decode_instance=decode_instance, - model=model_label, - ).dec() - raise http_exc - - t_prefill_done = time.monotonic() - # Prefill complete — decrement active prefill, start active decode - proxy_prefill_active_requests.labels( - prefill_instance=prefill_instance, - decode_instance=decode_instance, - model=model_label, - ).dec() - proxy_decode_active_requests.labels( - prefill_instance=prefill_instance, - decode_instance=decode_instance, - model=model_label, - ).inc() - - value = ( - value.strip().decode("utf-8").removesuffix("data: [DONE]").encode("utf-8") - ) - if server.disaggregated_mode == "nixl": - try: - prefill_output = json.loads(value) - except json.JSONDecodeError as exc: - raise HTTPException( - status_code=502, - detail="Prefill response did not contain valid JSON", - ) from exc - kv_transfer_params = prefill_output.get("kv_transfer_params") - if not kv_transfer_params: - raise HTTPException( - status_code=502, - detail="Prefill response did not contain kv_transfer_params", - ) - request = request.copy() - request["kv_transfer_params"] = kv_transfer_params - elif server.disaggregated_mode == "zmq": - prefill_output = None - first_token = None - if server.first_token_source == "prefill": - try: - prefill_output = json.loads(value) - first_token = prefill_output["kv_transfer_params"]["first_tok"] - except (json.JSONDecodeError, KeyError, TypeError) as exc: - assert zmq_request_id is not None - await server.zmq_notifications.discard(zmq_request_id) - raise HTTPException( - status_code=502, - detail=( - "Prefill response did not contain an LMCache " "first token" - ), - ) from exc - receiver = server.zmq_config.receivers[decode_instance] - if zmq_wait_for_notification: - try: - await server.zmq_notifications.wait( - zmq_request_id, - len(receiver.init_ports), - ) - except TimeoutError as exc: - raise HTTPException( - status_code=504, - detail=( - "Timed out waiting for ZMQ notification " - f"{zmq_request_id}" - ), - ) from exc - assert zmq_prompt_tokens is not None - request = build_zmq_decode_request( - request, - zmq_prompt_tokens, - first_token, - max_tokens, - is_chat, - ) - - if server.registry is not None: - server.registry.record_success(prefill_instance) - - async def streaming_response(value): - if value: - yield value - else: - yield b"" - - prefill_only = ( - server.disaggregated_mode == "zmq" - and server.first_token_source == "prefill" - and max_tokens == 1 - ) - if prefill_only: - assert prefill_output is not None - final_generator = _zmq_prefill_only_generator( - prefill_output, - reservation, - is_chat, - request.get("stream", False), - request.get("stream_options", {}).get("include_usage", False), - ) - first_token_from_p = True - else: - generator_p = streaming_response(value) - # server.forward_request() is an async generator; HTTP errors - # surface only when wrapped_generator() iterates it. - generator_d_raw = server.forward_request( - f"http://{decode_instance}" - f"{'/v1/completions' if server.disaggregated_mode == 'zmq' and is_chat else endpoint}", # noqa: E501 - request, - extra_headers=upstream_headers, - ) - decode_tracker = FirstTokenTracker(generator_d_raw) - generator_d = decode_tracker - - if ( - server.disaggregated_mode == "zmq" - and server.first_token_source == "prefill" - ): - assert prefill_output is not None - if request.get("stream", False): - final_generator = _zmq_stream_generator( - prefill_output, - generator_d, - reservation, - is_chat, - ) - else: - final_generator = _zmq_nonstream_generator( - prefill_output, - generator_d, - reservation, - is_chat, - ) - first_token_from_p = True - else: - if server.disaggregated_mode == "zmq" and is_chat: - generator_d = ( - _chat_completion_stream(generator_d) - if request.get("stream", False) - else _chat_completion_nonstream(generator_d) - ) - if request.get("stream", False): - generator_class = server.generator - else: - generator_class = server.d_first_token_generator_class - # Determine if user's first token comes from prefill or decode node. - # P_first_token_generator yields P's token first; - # D_first_token_generator - # discards P's token and yields D's tokens only. - from xpyd.proxy import P_first_token_generator - - first_token_from_p = ( - server.first_token_source == "prefill" - and generator_class is P_first_token_generator - ) - final_generator = generator_class( - generator_p, - generator_d, - reservation.release_prefill, - reservation.release_decode, - ) - media_type = ( - "text/event-stream" if request.get("stream", False) else "application/json" - ) - - async def wrapped_generator(): - try: - async for chunk in final_generator: - yield chunk - if server.registry is not None and not prefill_only: - server.registry.record_success(decode_instance) - except CancelledError: - logger.warning( - "[0]Client disconnected during %s (CancelledError)", - handler_name, - ) - except HTTPException as http_exc: - reservation.release_all() - server._record_failure(prefill_instance, decode_instance) - proxy_instance_errors_total.labels( - instance=decode_instance, - error_type="decode_forward_error", - model=model_label, - ).inc() - logger.error( - "[1] HTTPException in wrapped_generator: %s", str(http_exc.detail) - ) - raise - except Exception as e: - proxy_instance_errors_total.labels( - instance=decode_instance, - error_type="decode_forward_error", - model=model_label, - ).inc() - logger.error("[1] Exception in wrapped_generator: %s", str(e)) - raise - finally: - reservation.release_all() - if ( - prefill_instance - and decode_instance - and t_prefill_done is not None - and decode_tracker is not None - ): - record_disaggregated_metrics( - prefill_instance=prefill_instance, - decode_instance=decode_instance, - model=model_label, - t_request_start=t_request_start, - t_prefill_done=t_prefill_done, - tracker=decode_tracker, - is_streaming=request.get("stream", False), - first_token_from_prefill=first_token_from_p, - ) - # Decrement decode active gauge (only if decode was started) - if decode_instance and prefill_instance and t_prefill_done is not None: - proxy_decode_active_requests.labels( - prefill_instance=prefill_instance, - decode_instance=decode_instance, - model=model_label, - ).dec() - track_request_end(endpoint, _metrics_start) - - return StreamingResponse(wrapped_generator(), media_type=media_type) - except (HTTPException, CancelledError): - if reservation is not None: - reservation.release_all() - if decode_instance and prefill_instance and t_prefill_done is not None: - proxy_decode_active_requests.labels( - prefill_instance=prefill_instance, - decode_instance=decode_instance, - model=model_label, - ).dec() - track_request_end(endpoint, _metrics_start) - raise - except Exception: - if reservation is not None: - reservation.release_all() - if decode_instance and prefill_instance and t_prefill_done is not None: - proxy_decode_active_requests.labels( - prefill_instance=prefill_instance, - decode_instance=decode_instance, - model=model_label, - ).dec() - track_request_end(endpoint, _metrics_start) - logger.error("Error in %s: %s", handler_name, sys.exc_info()[1]) - return JSONResponse( - {"error": {"message": "Internal proxy error", "type": "proxy_error"}}, - status_code=500, - ) - - -async def _handle_aggregated_completion( - endpoint: str, - request: dict, - raw_request: Request, - server: Proxy, - model: str, - total_length: int, - max_tokens: int, - prompt_text: str, - metrics_start: float, - handler_name: str, -) -> JSONResponse | StreamingResponse: - """Single-pass completion for aggregated-role instances.""" - reservation = server.reserve( - SchedulingContext( - role="aggregated", - model=model, - request_len=total_length, - max_tokens=max_tokens, - header=raw_request.headers.get("x-session-id"), - session_id=request.get("session_id"), - user=request.get("user"), - client_ip=(raw_request.client.host if raw_request.client else None), - prompt=prompt_text, - ) - ) - instance = reservation.address if reservation is not None else None - - logger.info( - "Aggregated %s request", - handler_name, - extra={ - "model": model, - "prompt_length": total_length, - "max_tokens": max_tokens, - "prompt": prompt_text[:200] if prompt_text else "", - "instance": instance, - }, - ) - - if instance is None: - track_request_end(endpoint, metrics_start) - # Check for unknown model - if model and server.registry is not None: - known_models = server.registry.get_registered_models() - if model not in known_models: - return error_response( - f"The model '{model}' does not exist", - INVALID_REQUEST, - 404, - ) - return error_response( - "No available instance can handle the request", - PROXY_ERROR, - 503, - ) - - url = f"http://{instance}{endpoint}" - - if request.get("stream", False): - generator = server.forward_request(url, request) - - async def wrapped(): - _ok = False - try: - async for chunk in generator: - yield chunk - _ok = True - except CancelledError: - _ok = False - logger.warning( - "Client disconnected during aggregated %s (CancelledError)", - handler_name, - ) - except Exception as e: - _ok = False - logger.error("Exception in aggregated stream: %s", str(e)) - if server.registry is not None: - server.registry.record_failure(instance) - raise - finally: - reservation.release() - if _ok and server.registry is not None: - server.registry.record_success(instance) - track_request_end(endpoint, metrics_start) - - return StreamingResponse(wrapped(), media_type="text/event-stream") - else: - # Non-streaming: forward request using server.forward_request() - # for consistent auth, timeouts, and connection handling. - try: - value = b"" - async for chunk in server.forward_request(url, request): - value += chunk - data = json.loads(value) - # Detect error responses from upstream. forward_request yields - # raw bytes without HTTP status, so inspect the parsed JSON. - # OpenAI error format: {"error": {"message": ..., "type": ..., "code": ...}} - # where "code" can be null, a string like "invalid_api_key", or an int. - if "error" in data: - err = data["error"] - err_type = err.get("type", "") - # Map error type to HTTP status code - if err_type == "invalid_request_error": - status_code = 400 - elif err_type == "authentication_error": - status_code = 401 - elif err_type == "not_found_error": - status_code = 404 - elif err_type == "rate_limit_error": - status_code = 429 - else: - status_code = 502 - else: - status_code = 200 - reservation.release() - if status_code < 400 and server.registry is not None: - server.registry.record_success(instance) - elif status_code >= 400 and server.registry is not None: - server.registry.record_failure(instance) - track_request_end(endpoint, metrics_start) - return JSONResponse(data, status_code=status_code) - except HTTPException as http_exc: - reservation.release() - if server.registry is not None: - server.registry.record_failure(instance) - track_request_end(endpoint, metrics_start) - return error_response( - str(http_exc.detail), - PROXY_ERROR, - http_exc.status_code, - ) - except CancelledError: - reservation.release() - raise - except Exception as e: - logger.error("Error in aggregated non-streaming: %s", str(e)) - reservation.release() - if server.registry is not None: - server.registry.record_failure(instance) - track_request_end(endpoint, metrics_start) - return error_response( - "Internal proxy error", - PROXY_ERROR, - 500, - ) - - -# --------------------------------------------------------------------------- -# Route registration -# --------------------------------------------------------------------------- - - -def register(router: APIRouter, server: Proxy) -> None: - """Register completion routes on *router*.""" - - async def _validate_json(raw_request: Request): - # HTTPException is intentional here: FastAPI Depends() dependencies - # cannot return a JSONResponse to short-circuit the request; only - # raising an exception aborts the dependency chain. - content_type = raw_request.headers.get("content-type", "").lower() - if content_type != "application/json": - raise HTTPException( - status_code=415, - detail="Unsupported Media Type: Only 'application/json' is allowed", - ) +def register(router: APIRouter, server: "Proxy") -> None: async def create_completion(raw_request: Request): return await handle_completion( "/v1/completions", raw_request, server, is_chat=False ) - async def create_chat_completion(raw_request: Request): - return await handle_completion( - "/v1/chat/completions", raw_request, server, is_chat=True - ) - router.post( "/v1/completions", - dependencies=[Depends(_validate_json)], - )( - server.custom_create_completion - if server.custom_create_completion - else create_completion - ) - - router.post( - "/v1/chat/completions", - dependencies=[Depends(_validate_json)], - )( - server.custom_create_chat_completion - if server.custom_create_chat_completion - else create_chat_completion - ) - + dependencies=[Depends(validate_json_content_type)], + )(server.custom_create_completion or create_completion) router.options("/v1/completions")(lambda: None) - router.options("/v1/chat/completions")(lambda: None) diff --git a/xpyd/routes/detokenize.py b/xpyd/routes/detokenize.py new file mode 100644 index 0000000..e3601df --- /dev/null +++ b/xpyd/routes/detokenize.py @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The /detokenize endpoint.""" + +from typing import TYPE_CHECKING + +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse + +if TYPE_CHECKING: + from xpyd.proxy import Proxy + + +def register(router: APIRouter, server: "Proxy") -> None: + async def post_detokenize(request: Request): + return await server.post_to_instance( + request, "/detokenize", {"model": "", "tokens": []} + ) + + router.post("/detokenize", response_class=JSONResponse)(post_detokenize) + router.options("/detokenize")(lambda: None) diff --git a/xpyd/routes/embeddings.py b/xpyd/routes/embeddings.py new file mode 100644 index 0000000..89cc5b0 --- /dev/null +++ b/xpyd/routes/embeddings.py @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The /v1/embeddings endpoint.""" + +from typing import TYPE_CHECKING + +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse + +if TYPE_CHECKING: + from xpyd.proxy import Proxy + + +def register(router: APIRouter, server: "Proxy") -> None: + async def post_embeddings(request: Request): + return await server.post_to_instance( + request, "/v1/embeddings", {"model": "", "input": ""} + ) + + router.post("/v1/embeddings", response_class=JSONResponse)(post_embeddings) + router.options("/v1/embeddings")(lambda: None) diff --git a/xpyd/routes/forward.py b/xpyd/routes/forward.py deleted file mode 100644 index 79a0775..0000000 --- a/xpyd/routes/forward.py +++ /dev/null @@ -1,85 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""Proxy / forward-to-instance route handlers.""" - -from fastapi import APIRouter, Request -from fastapi.responses import JSONResponse - - -def register(router: APIRouter, server) -> None: - """Register proxy-forward routes on *router*.""" - - async def post_tokenize(request: Request): - return await server.post_to_instance( - request, "/tokenize", {"model": "", "prompt": ""} - ) - - async def post_detokenize(request: Request): - return await server.post_to_instance( - request, "/detokenize", {"model": "", "tokens": []} - ) - - async def post_embeddings(request: Request): - return await server.post_to_instance( - request, "/v1/embeddings", {"model": "", "input": ""} - ) - - async def post_pooling(request: Request): - return await server.post_to_instance( - request, "/pooling", {"model": "", "messages": ""} - ) - - async def post_score(request: Request): - return await server.post_to_instance( - request, - "/score", - {"model": "", "text_1": "", "text_2": "", "predictions": ""}, - ) - - async def post_scorev1(request: Request): - return await server.post_to_instance( - request, - "/v1/score", - {"model": "", "text_1": "", "text_2": "", "predictions": ""}, - ) - - async def post_rerank(request: Request): - return await server.post_to_instance( - request, "/rerank", {"model": "", "query": "", "documents": ""} - ) - - async def post_rerankv1(request: Request): - return await server.post_to_instance( - request, "/v1/rerank", {"model": "", "query": "", "documents": ""} - ) - - async def post_rerankv2(request: Request): - return await server.post_to_instance( - request, "/v2/rerank", {"model": "", "query": "", "documents": ""} - ) - - async def post_invocations(request: Request): - return await server.post_to_instance( - request, "/invocations", {"model": "", "prompt": ""} - ) - - router.post("/tokenize", response_class=JSONResponse)(post_tokenize) - router.post("/detokenize", response_class=JSONResponse)(post_detokenize) - router.post("/v1/embeddings", response_class=JSONResponse)(post_embeddings) - router.post("/pooling", response_class=JSONResponse)(post_pooling) - router.post("/score", response_class=JSONResponse)(post_score) - router.post("/v1/score", response_class=JSONResponse)(post_scorev1) - router.post("/rerank", response_class=JSONResponse)(post_rerank) - router.post("/v1/rerank", response_class=JSONResponse)(post_rerankv1) - router.post("/v2/rerank", response_class=JSONResponse)(post_rerankv2) - router.post("/invocations", response_class=JSONResponse)(post_invocations) - - router.options("/tokenize")(lambda: None) - router.options("/detokenize")(lambda: None) - router.options("/v1/embeddings")(lambda: None) - router.options("/pooling")(lambda: None) - router.options("/score")(lambda: None) - router.options("/v1/score")(lambda: None) - router.options("/rerank")(lambda: None) - router.options("/v1/rerank")(lambda: None) - router.options("/v2/rerank")(lambda: None) - router.options("/invocations")(lambda: None) diff --git a/xpyd/routes/health.py b/xpyd/routes/health.py index f23ddee..fc206ca 100644 --- a/xpyd/routes/health.py +++ b/xpyd/routes/health.py @@ -1,17 +1,16 @@ # SPDX-License-Identifier: Apache-2.0 -"""Health, info, and metrics route handlers.""" +"""The /health and /ping endpoints.""" -from fastapi import APIRouter -from fastapi.responses import JSONResponse, PlainTextResponse, Response +from typing import TYPE_CHECKING -from xpyd.metrics import get_metrics +from fastapi import APIRouter +from fastapi.responses import PlainTextResponse -# Fixed epoch timestamp for model listing (avoids changing on every request) -_MODEL_CREATED_EPOCH = 0 +if TYPE_CHECKING: + from xpyd.proxy import Proxy -def register(router: APIRouter, server) -> None: - """Register health/info routes on *router*.""" +def register(router: APIRouter, server: "Proxy") -> None: async def get_health(): return await server.get_from_instance("/health", is_full_instancelist=1) @@ -19,43 +18,8 @@ async def get_health(): async def get_ping(): return await server.get_from_instance("/ping", is_full_instancelist=1) - async def get_models(): - # Multi-model: return all registered models from registry - if server.registry is not None: - models = server.registry.get_registered_models() - data = [ - { - "id": m, - "object": "model", - "created": _MODEL_CREATED_EPOCH, - "owned_by": "system", - } - for m in models - ] - return JSONResponse( - content={"object": "list", "data": data}, - status_code=200, - ) - # No registry (should not happen in normal operation) — forward - return await server.get_from_instance("/v1/models") - - async def get_version(): - return await server.get_from_instance("/version") - - async def get_metrics_endpoint(): - return Response( - content=get_metrics(), - media_type="text/plain; version=0.0.4; charset=utf-8", - ) - router.get("/health", response_class=PlainTextResponse)(get_health) router.get("/ping", response_class=PlainTextResponse)(get_ping) router.post("/ping", response_class=PlainTextResponse)(get_ping) - router.get("/v1/models", response_class=JSONResponse)(get_models) - router.get("/version", response_class=JSONResponse)(get_version) - router.get("/metrics")(get_metrics_endpoint) - router.options("/health")(lambda: None) router.options("/ping")(lambda: None) - router.options("/v1/models")(lambda: None) - router.options("/version")(lambda: None) diff --git a/xpyd/routes/admin.py b/xpyd/routes/instances.py similarity index 90% rename from xpyd/routes/admin.py rename to xpyd/routes/instances.py index 7c22116..df97015 100644 --- a/xpyd/routes/admin.py +++ b/xpyd/routes/instances.py @@ -1,20 +1,24 @@ # SPDX-License-Identifier: Apache-2.0 -"""Admin route handlers.""" +"""Instance administration endpoints.""" import ipaddress import logging import os +from typing import TYPE_CHECKING from fastapi import APIRouter, Header, Request from fastapi.responses import JSONResponse from xpyd.errors import INVALID_REQUEST, SERVER_ERROR, error_response +if TYPE_CHECKING: + from xpyd.proxy import Proxy + logger = logging.getLogger("xpyd.proxy") -def register(router: APIRouter, server) -> None: - """Register admin routes on *router*.""" +def register(router: APIRouter, server: "Proxy") -> None: + """Register instance administration routes on *router*.""" def _authenticate_api_key(x_api_key: str): """Validate admin API key. Returns error_response or None.""" @@ -27,14 +31,6 @@ def _authenticate_api_key(x_api_key: str): return error_response("Forbidden: Invalid API Key", INVALID_REQUEST, 403) return None - async def get_status(): - return { - "prefill_node_count": len(server.prefill_instances), - "decode_node_count": len(server.decode_instances), - "prefill_nodes": server.prefill_instances, - "decode_nodes": server.decode_instances, - } - async def add_instance_endpoint( request: Request, x_api_key: str = Header(...), # noqa: B008 - FastAPI dependency idiom @@ -129,9 +125,7 @@ async def remove_instance_endpoint( logger.exception("Error in remove_instance_endpoint") return error_response(f"Internal error: {exc}", SERVER_ERROR, 500) - router.get("/status", response_class=JSONResponse)(get_status) router.post("/instances/add")(add_instance_endpoint) router.post("/instances/remove")(remove_instance_endpoint) - router.options("/status")(lambda: None) router.options("/instances/add")(lambda: None) router.options("/instances/remove")(lambda: None) diff --git a/xpyd/routes/invocations.py b/xpyd/routes/invocations.py new file mode 100644 index 0000000..d6f0a68 --- /dev/null +++ b/xpyd/routes/invocations.py @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The /invocations endpoint.""" + +from typing import TYPE_CHECKING + +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse + +if TYPE_CHECKING: + from xpyd.proxy import Proxy + + +def register(router: APIRouter, server: "Proxy") -> None: + async def post_invocations(request: Request): + return await server.post_to_instance( + request, "/invocations", {"model": "", "prompt": ""} + ) + + router.post("/invocations", response_class=JSONResponse)(post_invocations) + router.options("/invocations")(lambda: None) diff --git a/xpyd/routes/metrics.py b/xpyd/routes/metrics.py new file mode 100644 index 0000000..7087ec4 --- /dev/null +++ b/xpyd/routes/metrics.py @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The /metrics endpoint.""" + +from typing import TYPE_CHECKING + +from fastapi import APIRouter +from fastapi.responses import Response + +from xpyd.metrics import get_metrics + +if TYPE_CHECKING: + from xpyd.proxy import Proxy + + +def register(router: APIRouter, server: "Proxy") -> None: + async def get_metrics_endpoint(): + return Response( + content=get_metrics(), + media_type="text/plain; version=0.0.4; charset=utf-8", + ) + + router.get("/metrics")(get_metrics_endpoint) diff --git a/xpyd/routes/models.py b/xpyd/routes/models.py new file mode 100644 index 0000000..d5a2686 --- /dev/null +++ b/xpyd/routes/models.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The /v1/models endpoint.""" + +from typing import TYPE_CHECKING + +from fastapi import APIRouter +from fastapi.responses import JSONResponse + +if TYPE_CHECKING: + from xpyd.proxy import Proxy + +# Fixed epoch timestamp for model listing (avoids changing on every request). +_MODEL_CREATED_EPOCH = 0 + + +def register(router: APIRouter, server: "Proxy") -> None: + async def get_models(): + if server.registry is not None: + models = server.registry.get_registered_models() + data = [ + { + "id": model, + "object": "model", + "created": _MODEL_CREATED_EPOCH, + "owned_by": "system", + } + for model in models + ] + return JSONResponse( + content={"object": "list", "data": data}, + status_code=200, + ) + return await server.get_from_instance("/v1/models") + + router.get("/v1/models", response_class=JSONResponse)(get_models) + router.options("/v1/models")(lambda: None) diff --git a/xpyd/routes/pooling.py b/xpyd/routes/pooling.py new file mode 100644 index 0000000..3177424 --- /dev/null +++ b/xpyd/routes/pooling.py @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The /pooling endpoint.""" + +from typing import TYPE_CHECKING + +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse + +if TYPE_CHECKING: + from xpyd.proxy import Proxy + + +def register(router: APIRouter, server: "Proxy") -> None: + async def post_pooling(request: Request): + return await server.post_to_instance( + request, "/pooling", {"model": "", "messages": ""} + ) + + router.post("/pooling", response_class=JSONResponse)(post_pooling) + router.options("/pooling")(lambda: None) diff --git a/xpyd/routes/rerank.py b/xpyd/routes/rerank.py new file mode 100644 index 0000000..3be59ff --- /dev/null +++ b/xpyd/routes/rerank.py @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The /rerank, /v1/rerank and /v2/rerank endpoints.""" + +from typing import TYPE_CHECKING + +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse + +if TYPE_CHECKING: + from xpyd.proxy import Proxy + + +def register(router: APIRouter, server: "Proxy") -> None: + async def post_rerank(request: Request): + return await server.post_to_instance( + request, "/rerank", {"model": "", "query": "", "documents": ""} + ) + + async def post_rerankv1(request: Request): + return await server.post_to_instance( + request, "/v1/rerank", {"model": "", "query": "", "documents": ""} + ) + + async def post_rerankv2(request: Request): + return await server.post_to_instance( + request, "/v2/rerank", {"model": "", "query": "", "documents": ""} + ) + + router.post("/rerank", response_class=JSONResponse)(post_rerank) + router.post("/v1/rerank", response_class=JSONResponse)(post_rerankv1) + router.post("/v2/rerank", response_class=JSONResponse)(post_rerankv2) + router.options("/rerank")(lambda: None) + router.options("/v1/rerank")(lambda: None) + router.options("/v2/rerank")(lambda: None) diff --git a/xpyd/routes/score.py b/xpyd/routes/score.py new file mode 100644 index 0000000..6f22da3 --- /dev/null +++ b/xpyd/routes/score.py @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The /score and /v1/score endpoints.""" + +from typing import TYPE_CHECKING + +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse + +if TYPE_CHECKING: + from xpyd.proxy import Proxy + + +def register(router: APIRouter, server: "Proxy") -> None: + async def post_score(request: Request): + return await server.post_to_instance( + request, + "/score", + {"model": "", "text_1": "", "text_2": "", "predictions": ""}, + ) + + async def post_scorev1(request: Request): + return await server.post_to_instance( + request, + "/v1/score", + {"model": "", "text_1": "", "text_2": "", "predictions": ""}, + ) + + router.post("/score", response_class=JSONResponse)(post_score) + router.post("/v1/score", response_class=JSONResponse)(post_scorev1) + router.options("/score")(lambda: None) + router.options("/v1/score")(lambda: None) diff --git a/xpyd/routes/status.py b/xpyd/routes/status.py new file mode 100644 index 0000000..6c4d839 --- /dev/null +++ b/xpyd/routes/status.py @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Cluster and per-instance status endpoints.""" + +from typing import TYPE_CHECKING + +from fastapi import APIRouter +from fastapi.responses import JSONResponse + +if TYPE_CHECKING: + from xpyd.proxy import Proxy + + +def register(router: APIRouter, server: "Proxy") -> None: + async def get_status(): + return { + "prefill_node_count": len(server.prefill_instances), + "decode_node_count": len(server.decode_instances), + "prefill_nodes": server.prefill_instances, + "decode_nodes": server.decode_instances, + } + + async def _instance_status(): + result: dict[str, list] = { + "prefill_instances": [], + "decode_instances": [], + "aggregated_instances": [], + } + for info in server.registry.get_all_instances(): + result[f"{info.role}_instances"].append( + { + "address": info.address, + "status": info.status.value, + "circuit": info.circuit_breaker_state.value, + "active_requests": info.active_request_count, + "last_check": info.last_health_check, + } + ) + return JSONResponse(result) + + router.get("/status", response_class=JSONResponse)(get_status) + router.get("/status/instances")(_instance_status) + router.options("/status")(lambda: None) diff --git a/xpyd/routes/tokenize.py b/xpyd/routes/tokenize.py new file mode 100644 index 0000000..af23692 --- /dev/null +++ b/xpyd/routes/tokenize.py @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The /tokenize endpoint.""" + +from typing import TYPE_CHECKING + +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse + +if TYPE_CHECKING: + from xpyd.proxy import Proxy + + +def register(router: APIRouter, server: "Proxy") -> None: + async def post_tokenize(request: Request): + return await server.post_to_instance( + request, "/tokenize", {"model": "", "prompt": ""} + ) + + router.post("/tokenize", response_class=JSONResponse)(post_tokenize) + router.options("/tokenize")(lambda: None) diff --git a/xpyd/routes/version.py b/xpyd/routes/version.py new file mode 100644 index 0000000..07d9953 --- /dev/null +++ b/xpyd/routes/version.py @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The /version endpoint.""" + +from typing import TYPE_CHECKING + +from fastapi import APIRouter +from fastapi.responses import JSONResponse + +if TYPE_CHECKING: + from xpyd.proxy import Proxy + + +def register(router: APIRouter, server: "Proxy") -> None: + async def get_version(): + return await server.get_from_instance("/version") + + router.get("/version", response_class=JSONResponse)(get_version) + router.options("/version")(lambda: None)