From f4260eafe6405588f15b3b969facd7f1fb43a6a2 Mon Sep 17 00:00:00 2001 From: Tony Lin Date: Thu, 29 Oct 2026 03:21:16 +0000 Subject: [PATCH 1/2] refactor: inject explicit dependencies into routes and completion handlers Signed-off-by: Tony Lin --- docs/architecture.md | 55 +++-- tests/unit/routes/test_admin_routes.py | 12 +- tests/unit/routes/test_completion_routes.py | 55 ++--- tests/unit/routes/test_forward_routes.py | 2 +- tests/unit/routes/test_health_routes.py | 14 +- tests/unit/test_aggregated_completion.py | 38 +++- tests/unit/test_app.py | 101 +++++++-- tests/unit/test_completion_handler.py | 208 +++++++++++-------- xpyd/app.py | 53 ++--- xpyd/completion_handler.py | 218 +++++++++++++------- xpyd/interfaces.py | 44 ++++ xpyd/proxy.py | 127 ++++++++++-- xpyd/routes/__init__.py | 71 +++++-- xpyd/routes/chat_completions.py | 21 +- xpyd/routes/completions.py | 21 +- xpyd/routes/detokenize.py | 11 +- xpyd/routes/embeddings.py | 11 +- xpyd/routes/health.py | 11 +- xpyd/routes/instances.py | 16 +- xpyd/routes/invocations.py | 11 +- xpyd/routes/metrics.py | 7 +- xpyd/routes/models.py | 18 +- xpyd/routes/pooling.py | 11 +- xpyd/routes/rerank.py | 13 +- xpyd/routes/score.py | 11 +- xpyd/routes/status.py | 25 ++- xpyd/routes/tokenize.py | 11 +- xpyd/routes/version.py | 9 +- 28 files changed, 774 insertions(+), 431 deletions(-) create mode 100644 xpyd/interfaces.py diff --git a/docs/architecture.md b/docs/architecture.md index 9dd3187..dcf1f52 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -55,10 +55,15 @@ The existing `xpyd.proxy:main` CLI entry point remains supported. configuration-only commands. `load_config()` returns a `ProxyConfig` when a server should start, or `None` when a configuration command completes without starting one. It does not import or construct the proxy runtime. -- `xpyd/app.py` exposes `create_app(config, proxy, health_monitor=None)`. It - assembles FastAPI around an existing proxy, mounts its router, and owns CORS, - readiness checks, and the discovery/health-monitor/ZMQ-listener lifecycle. - Constructing the app does not start a network listener or background services. +- `xpyd/app.py` exposes + `create_app(router, *, discovery, health_monitor=None, zmq_notifications=None)`. + It mounts the supplied router and owns CORS, readiness checks, and the + background-service lifecycle. It does not receive a `Proxy`, create runtime + dependencies, or write state back to a coordinator. +- `ProxyServer` constructs discovery, the optional health monitor and ZMQ + listener, and shares the same listener with completion execution and app + lifecycle management. `ProxyServer.create_app()` wires the application without + starting a network listener or background services. Backends may be offline at startup. Discovery starts in the background, inference returns 503 until ready, and health/status/metrics/admin endpoints remain @@ -66,18 +71,31 @@ accessible. ### Endpoint Modules (`xpyd/routes/`) -Each logical endpoint has a module with `register(router, server)`. +Each logical endpoint has a module with `register(router, *, ...)`, taking only +the callbacks or data that endpoint needs. `routes/__init__.py` registers them all. URL aliases share their logical module: `score.py` handles `/score` and `/v1/score`, `rerank.py` handles all three rerank paths, and `health.py` handles `/health` and `/ping`. `status.py` serves both `/status` and `/status/instances`; `instances.py` handles addition and removal. Existing OPTIONS handlers stay beside their endpoints. -`completions.py` and `chat_completions.py` are separate HTTP entry points. Both -delegate to `xpyd/completion_handler.py`, which contains shared validation, -request preparation, aggregated/P-D execution, response conversion, first-token -generators, metrics, and reservation cleanup. Passthrough endpoints reuse -`Proxy.post_to_instance()`. +`completions.py` and `chat_completions.py` are separate HTTP entry points. Each +receives an async `execute(request)` callback and an optional custom handler. +Passthrough endpoints receive a `forward(request, path, template)` callback; +health/version endpoints receive a fetch callback. Model/status endpoints use +query callbacks and the relevant live membership lists. Instance administration +receives only add/remove callbacks; metrics needs no injected runtime object. + +`xpyd/completion_handler.py` takes explicit keyword-only operations for scheduling, +forwarding, tokenization, topology lookup, model lookup and outcome recording, +plus `CompletionOptions` and an optional notification interface. Options contain +only transfer configuration, not service objects. The aggregated sub-handler +receives only scheduling, forwarding, model lookup and outcome callbacks; pure +prompt helpers receive only a token-counting function or tokenizer. + +`xpyd/interfaces.py` defines narrow callable/tokenizer/notification contracts. +There is no `ProxyProtocol`, generic service container, or all-purpose context +passed through request handling. Runtime imports are one-way: @@ -86,13 +104,20 @@ proxy -> cli -> config proxy -> app -> discovery / health_monitor proxy -> routes -> completion_handler proxy -> completion_handler +routes / completion_handler -> interfaces ``` -App assembly, routes, and completion helpers receive the proxy object rather -than importing its implementation at runtime. References to `Proxy` for type -annotations are guarded by `TYPE_CHECKING`. Completion helpers do not import -routes or app assembly. Resource ownership during streaming is unchanged: -request completion and reservation release remain tied to response consumption. +Only the composition code in `proxy.py` selects concrete implementations and +binds endpoint-specific completion callbacks. It currently supplies bound +runtime methods, but the receiving modules depend only on the declared callable +contracts: independent functions or alternative services can be supplied instead. +Routes, app assembly, and completion helpers neither receive a `Proxy`/server +object nor import that implementation, including for type annotations. + +Completion helpers do not import routes or app assembly. Resource ownership +during streaming is unchanged: request completion and reservation release remain +tied to response consumption. Membership lists remain live so runtime +addition/removal is reflected in status responses. ### Scheduling Policies diff --git a/tests/unit/routes/test_admin_routes.py b/tests/unit/routes/test_admin_routes.py index ba454ad..891f057 100644 --- a/tests/unit/routes/test_admin_routes.py +++ b/tests/unit/routes/test_admin_routes.py @@ -16,8 +16,16 @@ def register(router, server): - instances.register(router, server) - status.register(router, server) + instances.register( + router, + add_instance=server.add_instance, + remove_instance=server.drain_and_remove_instance, + ) + status.register( + router, + prefill_instances=server.prefill_instances, + decode_instances=server.decode_instances, + ) API_KEY = "s3cr3t" diff --git a/tests/unit/routes/test_completion_routes.py b/tests/unit/routes/test_completion_routes.py index c6d763e..71f5bc3 100644 --- a/tests/unit/routes/test_completion_routes.py +++ b/tests/unit/routes/test_completion_routes.py @@ -1,27 +1,23 @@ # SPDX-License-Identifier: Apache-2.0 """Completion entry points share execution without coupling route modules.""" -from types import SimpleNamespace -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock import pytest -from fastapi import Request +from fastapi import APIRouter, FastAPI, Request from fastapi.responses import JSONResponse from fastapi.testclient import TestClient from xpyd.routes import chat_completions, completions -from ._helpers import build_app - @pytest.fixture( params=[ - (completions, "/v1/completions", False, "custom_create_completion"), + (completions, "/v1/completions", False), ( chat_completions, "/v1/chat/completions", True, - "custom_create_chat_completion", ), ], ids=["completions", "chat-completions"], @@ -31,42 +27,48 @@ def endpoint(request): def test_default_entry_delegates_to_shared_handler(endpoint): - module, path, is_chat, _ = endpoint - server = SimpleNamespace( - custom_create_completion=None, - custom_create_chat_completion=None, - ) + module, path, is_chat = endpoint handler = AsyncMock(return_value=JSONResponse({"shared": True}, status_code=202)) - with patch.object(module, "handle_completion", handler): - client = TestClient(build_app(module.register, server)) - response = client.post(path, json={"model": "example"}) + router = APIRouter() + module.register(router, execute=handler) + app = FastAPI() + app.include_router(router) + client = TestClient(app) + response = client.post(path, json={"model": "example"}) assert response.status_code == 202 assert response.json() == {"shared": True} handler.assert_awaited_once() - assert handler.call_args.args[0] == path - assert isinstance(handler.call_args.args[1], Request) - assert handler.call_args.args[2] is server - assert handler.call_args.kwargs == {"is_chat": is_chat} + assert len(handler.call_args.args) == 1 + assert isinstance(handler.call_args.args[0], Request) + assert handler.call_args.args[0].url.path == path + assert handler.call_args.kwargs == {} assert client.options(path).status_code == 200 other_path = "/v1/completions" if is_chat else "/v1/chat/completions" assert client.post(other_path, json={}).status_code == 404 -def test_custom_handler_keeps_content_type_validation(endpoint): - module, path, _, custom_attribute = endpoint +@pytest.mark.parametrize("synchronous", [False, True]) +def test_custom_handler_keeps_content_type_validation(endpoint, synchronous): + module, path, _ = endpoint calls = [] async def custom(request: Request): calls.append(await request.json()) return JSONResponse({"custom": True}) - server = SimpleNamespace( - custom_create_completion=None, - custom_create_chat_completion=None, + def sync_custom(request: Request): + calls.append({"model": "example"}) + return JSONResponse({"custom": True}) + + execute = AsyncMock() + router = APIRouter() + module.register( + router, execute=execute, custom_handler=sync_custom if synchronous else custom ) - setattr(server, custom_attribute, custom) - client = TestClient(build_app(module.register, server)) + app = FastAPI() + app.include_router(router) + client = TestClient(app) assert ( client.post( @@ -79,3 +81,4 @@ async def custom(request: Request): assert response.status_code == 200 assert response.json() == {"custom": True} assert calls == [{"model": "example"}] + execute.assert_not_called() diff --git a/tests/unit/routes/test_forward_routes.py b/tests/unit/routes/test_forward_routes.py index 96a2fba..4d80075 100644 --- a/tests/unit/routes/test_forward_routes.py +++ b/tests/unit/routes/test_forward_routes.py @@ -33,7 +33,7 @@ def register(router, server): rerank, invocations, ): - module.register(router, server) + module.register(router, forward=server.post_to_instance) # (route, forwarded path, expected json template, a valid request body) diff --git a/tests/unit/routes/test_health_routes.py b/tests/unit/routes/test_health_routes.py index 58d4e9e..38120b1 100644 --- a/tests/unit/routes/test_health_routes.py +++ b/tests/unit/routes/test_health_routes.py @@ -22,8 +22,18 @@ def register(router, server): - for module in (health, models, version, metrics): - module.register(router, server) + health.register(router, fetch=server.get_from_instance) + version.register(router, fetch=server.get_from_instance) + models.register( + router, + fetch=server.get_from_instance, + get_registered_models=( + server.registry.get_registered_models + if server.registry is not None + else None + ), + ) + metrics.register(router) class _HealthServer: diff --git a/tests/unit/test_aggregated_completion.py b/tests/unit/test_aggregated_completion.py index cddd14d..ba43993 100644 --- a/tests/unit/test_aggregated_completion.py +++ b/tests/unit/test_aggregated_completion.py @@ -6,7 +6,7 @@ import pytest -from xpyd.completion_handler import handle_completion +from xpyd.completion_handler import CompletionOptions, handle_completion from xpyd.proxy import Proxy from xpyd.registry import InstanceRegistry from xpyd.scheduler import SchedulingContext, default_registry @@ -32,6 +32,21 @@ def make_proxy(strategy="roundrobin", override=None, registry=True): ) +def completion_arguments(proxy): + return { + "options": CompletionOptions(), + "reserve": proxy.reserve, + "forward_request": proxy.forward_request, + "count_tokens": proxy.get_total_token_length, + "get_tokenizer": proxy.get_tokenizer, + "tokenize_on_backend": proxy.tokenize_on_backend, + "is_aggregated_model": proxy._is_aggregated_model, + "get_registered_models": proxy.registry.get_registered_models, + "record_success": proxy.registry.record_success, + "record_failure": proxy.registry.record_failure, + } + + def test_model_topology_detection(): proxy = make_proxy() assert proxy._is_aggregated_model(MODEL) @@ -152,7 +167,12 @@ async def forward(*args, **kwargs): patch.object(proxy.registry, "record_success") as success, patch.object(proxy.registry, "record_failure") as failed, ): - response = await handle_completion("/v1/completions", request, proxy, False) + response = await handle_completion( + request, + endpoint="/v1/completions", + is_chat=False, + **completion_arguments(proxy), + ) if failure is RuntimeError: with pytest.raises(RuntimeError): _ = [chunk async for chunk in response.body_iterator] @@ -176,7 +196,12 @@ async def forward(*args, **kwargs): proxy.forward_request = forward with patch("xpyd.completion_handler.track_request_end") as ended: with pytest.raises(asyncio.CancelledError): - await handle_completion("/v1/completions", request, proxy, False) + await handle_completion( + request, + endpoint="/v1/completions", + is_chat=False, + **completion_arguments(proxy), + ) ended.assert_called_once() assert all(proxy.registry.get_active_requests(address) == 0 for address in NODES) @@ -195,7 +220,12 @@ async def forward(*args, **kwargs): proxy.forward_request = forward with patch.object(proxy.registry, "record_success") as success: - response = await handle_completion("/v1/completions", request, proxy, False) + response = await handle_completion( + request, + endpoint="/v1/completions", + is_chat=False, + **completion_arguments(proxy), + ) await response.body_iterator.__anext__() await response.body_iterator.aclose() success.assert_not_called() diff --git a/tests/unit/test_app.py b/tests/unit/test_app.py index 56e20f9..d81bb9b 100644 --- a/tests/unit/test_app.py +++ b/tests/unit/test_app.py @@ -24,10 +24,23 @@ def test_support_modules_do_not_import_proxy_runtime(): "-c", "\n".join( [ - "import importlib, sys", + "import importlib, inspect, pkgutil, sys, typing", "for name in ('xpyd.cli', 'xpyd.app', 'xpyd.routes',", - " 'xpyd.completion_handler'):", - " importlib.import_module(name)", + " 'xpyd.completion_handler', 'xpyd.interfaces'):", + " module = importlib.import_module(name)", + " assert 'xpyd.proxy' not in sys.modules, name", + " modules = [module]", + " if hasattr(module, '__path__'):", + " modules += [importlib.import_module(item.name)", + " for item in pkgutil.iter_modules(", + " module.__path__, module.__name__ + '.')]", + " for item in modules:", + " for value in vars(item).values():", + " if (inspect.isfunction(value)", + " and value.__module__ == item.__name__):", + " typing.get_type_hints(value)", + " parameters = inspect.signature(value).parameters", + " assert not {'proxy', 'server'} & set(parameters)", " assert 'xpyd.proxy' not in sys.modules, name", ] ), @@ -78,8 +91,8 @@ async def forward(url, data, **kwargs): yield json.dumps({"choices": [{"text": "ok"}]}).encode() proxy.forward_request = forward - app = create_app(config, proxy, server.health_monitor) - discovery = proxy.discovery + app = server.create_app() + discovery = server.discovery monitor = server.health_monitor assert discovery._task is None assert monitor._task is None @@ -226,24 +239,19 @@ def test_zmq_lifecycle_order(): health_check={"enabled": True}, ) server = ProxyServer(config) - proxy = server.proxy_instance - app = create_app(config, proxy, server.health_monitor) events = [] listener = Mock( start=AsyncMock(side_effect=lambda: events.append("listener:start")), stop=AsyncMock(side_effect=lambda: events.append("listener:stop")), ) with ( - patch( - "xpyd.zmq_notifications.ZmqNotificationListener", return_value=listener - ) as factory, patch.object( - proxy.discovery, + server.discovery, "start", AsyncMock(side_effect=lambda: events.append("discovery:start")), ), patch.object( - proxy.discovery, + server.discovery, "stop", AsyncMock(side_effect=lambda: events.append("discovery:stop")), ), @@ -258,13 +266,15 @@ def test_zmq_lifecycle_order(): AsyncMock(side_effect=lambda: events.append("health:stop")), ), ): + app = create_app( + server.proxy_instance.router, + discovery=server.discovery, + health_monitor=server.health_monitor, + zmq_notifications=listener, + ) assert events == [] with TestClient(app): - assert proxy.zmq_notifications is listener assert events == ["listener:start", "discovery:start", "health:start"] - factory.assert_called_once_with( - config.zmq.host, config.zmq.port, config.zmq.notification_timeout_seconds - ) assert events == [ "listener:start", "discovery:start", @@ -273,3 +283,62 @@ def test_zmq_lifecycle_order(): "listener:stop", "health:stop", ] + + +def test_zmq_route_uses_composed_listener(): + config = ProxyConfig( + model="example", + prefill=["127.0.0.1:18100"], + decode=["127.0.0.1:18200"], + scheduling="roundrobin", + disaggregated_mode="zmq", + zmq={ + "receivers": { + "127.0.0.1:18200": { + "host": "127.0.0.1", + "init_ports": [7300], + "alloc_ports": [7400], + } + } + }, + ) + server = ProxyServer(config) + server.discovery.healthy_prefill.update(config.prefill) + server.discovery.healthy_decode.update(config.decode) + listener = server.zmq_notifications + forwarded = [] + + async def forward(url, data, **kwargs): + forwarded.append((url, data)) + yield b'{"choices":[{"text":"ok"}]}' + + server.proxy_instance.forward_request = forward + server.proxy_instance.tokenize_on_backend = AsyncMock(return_value=[1, 2]) + with ( + patch.object(server.discovery, "start", AsyncMock()), + patch.object(server.discovery, "stop", AsyncMock()), + patch.object(listener, "start", AsyncMock()) as start, + patch.object(listener, "stop", AsyncMock()) as stop, + patch.object(listener, "register", AsyncMock()) as register, + patch.object(listener, "wait", AsyncMock()) as wait, + ): + with TestClient(server.create_app()) as client: + response = client.post( + "/v1/completions", + json={"model": "example", "prompt": "hello", "max_tokens": 2}, + ) + assert response.status_code == 200, response.text + assert response.json()["choices"][0]["text"] == "ok" + start.assert_awaited_once() + stop.assert_awaited_once() + request_id = forwarded[0][1]["kv_transfer_params"]["disagg_spec"]["req_id"] + register.assert_awaited_once_with(request_id) + wait.assert_awaited_once_with(request_id, 1) + assert [url for url, _ in forwarded] == [ + "http://127.0.0.1:18100/v1/completions", + "http://127.0.0.1:18200/v1/completions", + ] + assert all( + server.registry.get_active_requests(address) == 0 + for address in config.prefill + config.decode + ) diff --git a/tests/unit/test_completion_handler.py b/tests/unit/test_completion_handler.py index b118aa6..2ec842e 100644 --- a/tests/unit/test_completion_handler.py +++ b/tests/unit/test_completion_handler.py @@ -9,6 +9,7 @@ from starlette.responses import JSONResponse from xpyd.completion_handler import ( + CompletionOptions, _chat_completion_nonstream, _chat_completion_stream, _RequestReservation, @@ -22,6 +23,7 @@ tokenize_zmq_prompt, validate_completion_request, ) +from xpyd.config import ZmqConfig, ZmqReceiverConfig from xpyd.metrics import REGISTRY from xpyd.scheduler import RoundRobinSchedulingPolicy, Scheduler, SchedulingContext @@ -40,15 +42,32 @@ def reservations(request_len=5): @pytest.fixture -def server(): - """Create a mock server with minimal attributes.""" - srv = MagicMock() - srv.get_total_token_length = MagicMock( +def count_tokens(): + return MagicMock( side_effect=lambda x, model="": len(x) if isinstance(x, str) else 0 ) - srv.get_tokenizer = MagicMock(side_effect=lambda model="": srv.tokenizer) - srv._is_aggregated_model = MagicMock(return_value=False) - return srv + + +@pytest.fixture +def tokenizer(): + return MagicMock() + + +@pytest.fixture +def dependencies(count_tokens, tokenizer): + return { + "options": CompletionOptions(), + "reserve": MagicMock(return_value=None), + "count_tokens": count_tokens, + "get_tokenizer": MagicMock(return_value=tokenizer), + "tokenize_on_backend": AsyncMock(return_value=[1, 2, 3]), + "forward_request": MagicMock(), + "is_aggregated_model": MagicMock(return_value=False), + "get_registered_models": MagicMock(return_value=["model"]), + "record_success": MagicMock(), + "record_failure": MagicMock(), + "notifications": AsyncMock(), + } class TestValidateCompletionRequest: @@ -83,29 +102,33 @@ def test_chat_messages_not_list(self): class TestExtractPromptInfo: """Tests for extract_prompt_info.""" - def test_completion_string_prompt(self, server): + def test_completion_string_prompt(self, count_tokens): total_length, max_tokens, prompt_text = extract_prompt_info( - {"prompt": "hello world", "max_tokens": 50}, is_chat=False, server=server + {"prompt": "hello world", "max_tokens": 50}, + is_chat=False, + count_tokens=count_tokens, ) assert total_length == 11 # len("hello world") assert max_tokens == 50 assert prompt_text == "hello world" - def test_completion_list_prompt(self, server): + def test_completion_list_prompt(self, count_tokens): total_length, max_tokens, prompt_text = extract_prompt_info( - {"prompt": [1, 2, 3], "max_tokens": 10}, is_chat=False, server=server + {"prompt": [1, 2, 3], "max_tokens": 10}, + is_chat=False, + count_tokens=count_tokens, ) assert total_length == 0 # MagicMock returns 0 for non-str assert max_tokens == 10 assert prompt_text == "[1, 2, 3]" - def test_completion_default_max_tokens(self, server): + def test_completion_default_max_tokens(self, count_tokens): _, max_tokens, _ = extract_prompt_info( - {"prompt": "test"}, is_chat=False, server=server + {"prompt": "test"}, is_chat=False, count_tokens=count_tokens ) assert max_tokens == 0 - def test_chat_basic(self, server): + def test_chat_basic(self, count_tokens): total_length, max_tokens, prompt_text = extract_prompt_info( { "messages": [ @@ -115,14 +138,14 @@ def test_chat_basic(self, server): "max_tokens": 100, }, is_chat=True, - server=server, + count_tokens=count_tokens, ) assert total_length == len("You are helpful") + len("Hello") assert max_tokens == 100 assert "You are helpful" in prompt_text assert "Hello" in prompt_text - def test_chat_max_completion_tokens_priority(self, server): + def test_chat_max_completion_tokens_priority(self, count_tokens): _, max_tokens, _ = extract_prompt_info( { "messages": [{"role": "user", "content": "hi"}], @@ -130,22 +153,22 @@ def test_chat_max_completion_tokens_priority(self, server): "max_tokens": 100, }, is_chat=True, - server=server, + count_tokens=count_tokens, ) assert max_tokens == 200 - def test_chat_fallback_to_max_tokens(self, server): + def test_chat_fallback_to_max_tokens(self, count_tokens): _, max_tokens, _ = extract_prompt_info( { "messages": [{"role": "user", "content": "hi"}], "max_tokens": 100, }, is_chat=True, - server=server, + count_tokens=count_tokens, ) assert max_tokens == 100 - def test_chat_mixed_content_types(self, server): + def test_chat_mixed_content_types(self, count_tokens): """Non-string content should be excluded from prompt_text.""" _, _, prompt_text = extract_prompt_info( { @@ -156,13 +179,13 @@ def test_chat_mixed_content_types(self, server): ], }, is_chat=True, - server=server, + count_tokens=count_tokens, ) assert "hello" in prompt_text assert "world" in prompt_text assert "None" not in prompt_text - def test_chat_null_content_zero_length(self, server): + def test_chat_null_content_zero_length(self, count_tokens): """Messages with None content should contribute 0 to total_length.""" total_length, _, _ = extract_prompt_info( { @@ -172,11 +195,11 @@ def test_chat_null_content_zero_length(self, server): ], }, is_chat=True, - server=server, + count_tokens=count_tokens, ) assert total_length == 2 # len("hi") via mock - def test_chat_multimodal_content_array(self, server): + def test_chat_multimodal_content_array(self, count_tokens): """Multimodal content (list of parts) should extract text parts only.""" total_length, _, prompt_text = extract_prompt_info( { @@ -194,14 +217,14 @@ def test_chat_multimodal_content_array(self, server): ], }, is_chat=True, - server=server, + count_tokens=count_tokens, ) assert total_length == len("What is this?") assert "What is this?" in prompt_text - def test_completion_token_ids(self, server): + def test_completion_token_ids(self, count_tokens): """Flat list of ints (already tokenized) should return its length.""" - server.get_total_token_length = MagicMock( + count_tokens = MagicMock( side_effect=lambda x, model="": ( len(x) if isinstance(x, (str, list)) else 0 ) @@ -209,7 +232,7 @@ def test_completion_token_ids(self, server): total_length, _, _ = extract_prompt_info( {"prompt": [101, 102, 103]}, is_chat=False, - server=server, + count_tokens=count_tokens, ) assert total_length == 3 @@ -343,8 +366,8 @@ def test_chat_becomes_completion_request(self, receiver): class TestZmqChatHelpers: - def test_chat_template_tokenization(self, server): - server.tokenizer.apply_chat_template.return_value = { + def test_chat_template_tokenization(self, tokenizer): + tokenizer.apply_chat_template.return_value = { "input_ids": [1, 2, 3], "attention_mask": [1, 1, 1], } @@ -355,8 +378,8 @@ def test_chat_template_tokenization(self, server): "tools": [{"type": "function"}], } - assert tokenize_zmq_prompt(request, True, server) == [1, 2, 3] - server.tokenizer.apply_chat_template.assert_called_once_with( + assert tokenize_zmq_prompt(request, True, tokenizer) == [1, 2, 3] + tokenizer.apply_chat_template.assert_called_once_with( request["messages"], foo="bar", tokenize=True, @@ -365,31 +388,31 @@ def test_chat_template_tokenization(self, server): tools=request["tools"], ) - def test_continue_final_message_disables_generation_prompt(self, server): - server.tokenizer.apply_chat_template.return_value = [1, 2, 3] + def test_continue_final_message_disables_generation_prompt(self, tokenizer): + tokenizer.apply_chat_template.return_value = [1, 2, 3] request = { "messages": [{"role": "assistant", "content": "partial"}], "continue_final_message": True, } - tokenize_zmq_prompt(request, True, server) + tokenize_zmq_prompt(request, True, tokenizer) - server.tokenizer.apply_chat_template.assert_called_once_with( + tokenizer.apply_chat_template.assert_called_once_with( request["messages"], tokenize=True, add_generation_prompt=False, continue_final_message=True, ) - def test_chat_template_single_batch_is_unwrapped(self, server): - server.tokenizer.apply_chat_template.return_value = { + def test_chat_template_single_batch_is_unwrapped(self, tokenizer): + tokenizer.apply_chat_template.return_value = { "input_ids": [[1, 2, 3]], } result = tokenize_zmq_prompt( {"messages": [{"role": "user", "content": "hi"}]}, True, - server, + tokenizer, ) assert result == [1, 2, 3] @@ -542,7 +565,7 @@ class TestHandleCompletion: """Integration-level tests for handle_completion.""" @pytest.mark.asyncio - async def test_invalid_json(self, server): + async def test_invalid_json(self, dependencies): raw_request = AsyncMock() raw_request.json = AsyncMock(side_effect=ValueError("bad json")) @@ -551,7 +574,7 @@ async def test_invalid_json(self, server): patch("xpyd.completion_handler.track_request_end") as track_end, ): result = await handle_completion( - "/v1/completions", raw_request, server, is_chat=False + raw_request, endpoint="/v1/completions", is_chat=False, **dependencies ) assert isinstance(result, JSONResponse) @@ -559,36 +582,42 @@ async def test_invalid_json(self, server): track_end.assert_called_once_with("/v1/completions", 0) @pytest.mark.asyncio - async def test_second_selection_failure_releases_prefill(self, server): + async def test_second_selection_failure_releases_prefill(self, dependencies): request = MagicMock(headers={}, client=None) request.json = AsyncMock(return_value={"prompt": "hello"}) leases = reservations() leases[1].release() - server.reserve.side_effect = [leases[0], RuntimeError("selection failed")] - result = await handle_completion("/v1/completions", request, server, False) + dependencies["reserve"].side_effect = [ + leases[0], + RuntimeError("selection failed"), + ] + result = await handle_completion( + request, endpoint="/v1/completions", is_chat=False, **dependencies + ) assert result.status_code == 500 assert leases[0]._released @pytest.mark.asyncio - async def test_prefill_cancellation_releases_both_nodes(self, server): + async def test_prefill_cancellation_releases_both_nodes(self, dependencies): request = MagicMock(headers={}, client=None) request.json = AsyncMock(return_value={"model": "model", "prompt": "hello"}) leases = reservations() - server.reserve.side_effect = leases - server.disaggregated_mode = "direct" + dependencies["reserve"].side_effect = leases async def forward(*args, **kwargs): raise asyncio.CancelledError() yield b"" # pragma: no cover - server.forward_request = forward + dependencies["forward_request"] = forward with pytest.raises(asyncio.CancelledError): - await handle_completion("/v1/completions", request, server, False) + await handle_completion( + request, endpoint="/v1/completions", is_chat=False, **dependencies + ) assert all(lease._released for lease in leases) - server.registry.record_success.assert_not_called() + dependencies["record_success"].assert_not_called() @pytest.mark.asyncio - async def test_missing_required_field(self, server): + async def test_missing_required_field(self, dependencies): raw_request = AsyncMock() raw_request.json = AsyncMock(return_value={}) @@ -597,7 +626,7 @@ async def test_missing_required_field(self, server): patch("xpyd.completion_handler.track_request_end") as track_end, ): result = await handle_completion( - "/v1/completions", raw_request, server, is_chat=False + raw_request, endpoint="/v1/completions", is_chat=False, **dependencies ) assert isinstance(result, JSONResponse) @@ -605,13 +634,13 @@ async def test_missing_required_field(self, server): track_end.assert_called_once_with("/v1/completions", 0) @pytest.mark.asyncio - async def test_no_available_instance(self, server): + async def test_no_available_instance(self, dependencies): raw_request = AsyncMock() raw_request.json = AsyncMock(return_value={"prompt": "hello"}) raw_request.headers = {} raw_request.client = None - server.reserve = MagicMock(return_value=None) + dependencies["reserve"] = MagicMock(return_value=None) labels = { "instance": "unknown", "error_type": "no_available_instance", @@ -625,7 +654,7 @@ async def test_no_available_instance(self, server): patch("xpyd.completion_handler.logger"), ): result = await handle_completion( - "/v1/completions", raw_request, server, is_chat=False + raw_request, endpoint="/v1/completions", is_chat=False, **dependencies ) assert isinstance(result, JSONResponse) @@ -636,7 +665,7 @@ async def test_no_available_instance(self, server): == before + 1 ) - def test_request_reservation_releases_each_node_once(self, server): + def test_request_reservation_releases_each_node_once(self, dependencies): leases = reservations(10) reservation = _RequestReservation(*leases) @@ -647,7 +676,7 @@ def test_request_reservation_releases_each_node_once(self, server): assert all(value == 0 for value in leases[0]._runtime._active.values()) @pytest.mark.asyncio - async def test_nixl_prefill_parse_error_releases_reservations(self, server): + async def test_nixl_prefill_parse_error_releases_reservations(self, dependencies): raw_request = AsyncMock() raw_request.json = AsyncMock( return_value={ @@ -658,26 +687,25 @@ async def test_nixl_prefill_parse_error_releases_reservations(self, server): ) raw_request.headers = {} raw_request.client = None - server.disaggregated_mode = "nixl" + dependencies["options"] = CompletionOptions(disaggregated_mode="nixl") leases = reservations() - server.reserve = MagicMock(side_effect=leases) - server._record_failure = MagicMock() + dependencies["reserve"] = MagicMock(side_effect=leases) async def forward(*_args, **_kwargs): yield b"not-json" - server.forward_request = forward + dependencies["forward_request"] = forward with pytest.raises(HTTPException, match="valid JSON"): await handle_completion( - "/v1/completions", raw_request, server, is_chat=False + raw_request, endpoint="/v1/completions", is_chat=False, **dependencies ) assert all(lease._released for lease in leases) - server.registry.record_success.assert_not_called() + dependencies["record_success"].assert_not_called() @pytest.mark.asyncio - async def test_unknown_aggregated_model_ends_metrics(self, server): + async def test_unknown_aggregated_model_ends_metrics(self, dependencies): raw_request = AsyncMock() raw_request.json = AsyncMock( return_value={ @@ -689,16 +717,16 @@ async def test_unknown_aggregated_model_ends_metrics(self, server): raw_request.headers = {} raw_request.client = None - server._is_aggregated_model.return_value = True - server.reserve.return_value = None - server.registry.get_registered_models.return_value = ["known-model"] + dependencies["is_aggregated_model"].return_value = True + dependencies["reserve"].return_value = None + dependencies["get_registered_models"].return_value = ["known-model"] with ( patch("xpyd.completion_handler.track_request_start", return_value=0), patch("xpyd.completion_handler.track_request_end") as track_end, ): result = await handle_completion( - "/v1/completions", raw_request, server, is_chat=False + raw_request, endpoint="/v1/completions", is_chat=False, **dependencies ) assert isinstance(result, JSONResponse) @@ -709,9 +737,9 @@ async def test_unknown_aggregated_model_ends_metrics(self, server): @pytest.mark.parametrize("source", ["decode", "prefill"]) @pytest.mark.parametrize("stream", [False, True]) @pytest.mark.parametrize("max_tokens", [1, 2]) - async def test_zmq_chat_completion_modes(self, server, source, stream, max_tokens): - from xpyd.proxy import D_first_token_generator - + async def test_zmq_chat_completion_modes( + self, dependencies, tokenizer, source, stream, max_tokens + ): request = { "model": "model", "messages": [{"role": "user", "content": "hi"}], @@ -723,24 +751,25 @@ async def test_zmq_chat_completion_modes(self, server, source, stream, max_token raw_request.headers = {} raw_request.client = None - receiver = MagicMock(host="receiver") - receiver.init_ports = [7300] - receiver.alloc_ports = [7400] - server.disaggregated_mode = "zmq" - server.first_token_source = source - server.tokenizer.apply_chat_template.return_value = { + receiver = ZmqReceiverConfig( + host="receiver", init_ports=[7300], alloc_ports=[7400] + ) + dependencies["options"] = CompletionOptions( + disaggregated_mode="zmq", + first_token_source=source, + zmq=ZmqConfig(receivers={"decode:8000": receiver}), + ) + tokenizer.apply_chat_template.return_value = { "input_ids": [1, 2, 3], "attention_mask": [1, 1, 1], } leases = reservations(3) - server.reserve = MagicMock(side_effect=leases) - server.zmq_config.receivers = {"decode:8000": receiver} - server.zmq_notifications.register = AsyncMock() - server.zmq_notifications.wait = AsyncMock() - server._record_failure = MagicMock() - server.registry = None - server.generator = D_first_token_generator - server.d_first_token_generator_class = D_first_token_generator + dependencies["reserve"] = MagicMock(side_effect=leases) + dependencies["notifications"].register = AsyncMock() + dependencies["notifications"].wait = AsyncMock() + dependencies["get_registered_models"] = None + dependencies["record_success"] = None + dependencies["record_failure"] = None forwarded = [] async def forward(url, data, **kwargs): @@ -793,14 +822,17 @@ async def forward(url, data, **kwargs): } ).encode() - server.forward_request = forward + dependencies["forward_request"] = forward with ( patch("xpyd.completion_handler.track_request_start", return_value=0), patch("xpyd.completion_handler.track_request_end"), patch("xpyd.completion_handler.record_disaggregated_metrics"), ): response = await handle_completion( - "/v1/chat/completions", raw_request, server, is_chat=True + raw_request, + endpoint="/v1/chat/completions", + is_chat=True, + **dependencies, ) body = b"".join([chunk async for chunk in response.body_iterator]) @@ -810,7 +842,7 @@ async def forward(url, data, **kwargs): else [] ) assert [call[0] for call in forwarded] == expected_urls - assert server.reserve.call_count == 2 + assert dependencies["reserve"].call_count == 2 assert all(lease._released for lease in leases) assert "messages" not in forwarded[0][1] assert forwarded[0][1]["prompt"] == [1, 2, 3] diff --git a/xpyd/app.py b/xpyd/app.py index fc1e2f7..dbc1170 100644 --- a/xpyd/app.py +++ b/xpyd/app.py @@ -1,42 +1,23 @@ # SPDX-License-Identifier: Apache-2.0 """FastAPI application assembly and background service lifecycle.""" -import asyncio - -from fastapi import FastAPI, Request +from fastapi import APIRouter, FastAPI, Request from fastapi.middleware.cors import CORSMiddleware -from xpyd.config import ProxyConfig from xpyd.discovery import NodeDiscovery from xpyd.errors import PROXY_ERROR, error_response from xpyd.health_monitor import HealthMonitor +from xpyd.zmq_notifications import ZmqNotificationListener def create_app( - config: ProxyConfig, - proxy: "Proxy", # noqa: F821 + router: APIRouter, + *, + discovery: NodeDiscovery, health_monitor: HealthMonitor | None = None, + zmq_notifications: ZmqNotificationListener | None = None, ) -> FastAPI: - """Assemble an app around an existing proxy without starting a listener.""" - discovery = NodeDiscovery( - prefill_instances=proxy.prefill_instances, - decode_instances=proxy.decode_instances, - probe_interval=config.probe_interval_seconds, - wait_timeout=config.wait_timeout_seconds, - heartbeat_interval=config.heartbeat_interval_seconds, - registry=proxy.registry, - aggregated_instances=[ - address - for instances in proxy.aggregated_instances.values() - for address in instances - ], - on_model_discovered=lambda model: asyncio.to_thread( - proxy.ensure_tokenizer, model - ), - ) - proxy.discovery = discovery - proxy.health_monitor = health_monitor - + """Mount a router and manage explicitly supplied background services.""" app = FastAPI() app.add_middleware( CORSMiddleware, @@ -65,18 +46,8 @@ async def _check_readiness(request: Request, call_next): @app.on_event("startup") async def _start_discovery(): - if config.disaggregated_mode == "zmq": - from xpyd.zmq_notifications import ZmqNotificationListener - - zmq_config = config.zmq - assert zmq_config is not None - listener = ZmqNotificationListener( - zmq_config.host, - zmq_config.port, - zmq_config.notification_timeout_seconds, - ) - await listener.start() - proxy.zmq_notifications = listener + if zmq_notifications is not None: + await zmq_notifications.start() await discovery.start() if health_monitor: await health_monitor.start() @@ -84,10 +55,10 @@ async def _start_discovery(): @app.on_event("shutdown") async def _stop_discovery(): await discovery.stop() - if proxy.zmq_notifications is not None: - await proxy.zmq_notifications.stop() + if zmq_notifications is not None: + await zmq_notifications.stop() if health_monitor: await health_monitor.stop() - app.include_router(proxy.router) + app.include_router(router) return app diff --git a/xpyd/completion_handler.py b/xpyd/completion_handler.py index f4d1f05..846a007 100644 --- a/xpyd/completion_handler.py +++ b/xpyd/completion_handler.py @@ -9,12 +9,21 @@ import time import uuid from asyncio import CancelledError -from collections.abc import AsyncGenerator, Callable, Mapping +from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping +from dataclasses import dataclass from fastapi import HTTPException, Request from fastapi.responses import JSONResponse, StreamingResponse +from xpyd.config import ZmqConfig, ZmqReceiverConfig from xpyd.errors import INVALID_REQUEST, PROXY_ERROR, error_response +from xpyd.interfaces import ( + CountTokens, + ForwardRequest, + Notifications, + TokenizeOnBackend, + Tokenizer, +) from xpyd.metrics import ( FirstTokenTracker, proxy_decode_active_requests, @@ -31,10 +40,35 @@ logger = logging.getLogger("xpyd.proxy") +FirstTokenGenerator = Callable[ + [ + AsyncIterator[bytes], + AsyncIterator[bytes], + Callable[[], None], + Callable[[], None], + ], + AsyncGenerator[bytes, None], +] + + +@dataclass(frozen=True) +class CompletionOptions: + disaggregated_mode: str = "direct" + first_token_source: str = "decode" + zmq: ZmqConfig | None = None + + +def _record_failures( + record_failure: Callable[[str], None] | None, *instances: str +) -> None: + if record_failure is not None: + for instance in instances: + record_failure(instance) + async def P_first_token_generator( - generator_p: AsyncGenerator[bytes, None], - generator_d: AsyncGenerator[bytes, None], + generator_p: AsyncIterator[bytes], + generator_d: AsyncIterator[bytes], release_prefill: Callable[[], None], release_decode: Callable[[], None], ) -> AsyncGenerator[bytes, None]: @@ -57,8 +91,8 @@ async def P_first_token_generator( async def D_first_token_generator( - generator_p: AsyncGenerator[bytes, None], - generator_d: AsyncGenerator[bytes, None], + generator_p: AsyncIterator[bytes], + generator_d: AsyncIterator[bytes], release_prefill: Callable[[], None], release_decode: Callable[[], None], ) -> AsyncGenerator[bytes, None]: @@ -113,7 +147,7 @@ def release_all(self) -> None: # --------------------------------------------------------------------------- -# Pure helper functions (no server dependency or explicit server param) +# Request preparation helpers # --------------------------------------------------------------------------- @@ -135,7 +169,7 @@ def validate_completion_request(request: dict, is_chat: bool) -> JSONResponse | def extract_prompt_info( - request: dict, is_chat: bool, server: "Proxy" # noqa: F821 + request: dict, is_chat: bool, count_tokens: CountTokens ) -> tuple[int, int, str]: """Extract prompt metrics. Returns (total_length, max_tokens, prompt_text).""" model = request.get("model", "") @@ -147,13 +181,13 @@ def extract_prompt_info( if content is None: continue if isinstance(content, str): - total_length += server.get_total_token_length(content, model) + total_length += count_tokens(content, model) prompt_parts.append(content) elif isinstance(content, list): for part in content: if isinstance(part, dict) and part.get("type") == "text": text = part.get("text", "") - total_length += server.get_total_token_length(text, model) + total_length += count_tokens(text, model) prompt_parts.append(text) max_tokens = request.get("max_completion_tokens", 0) if max_tokens == 0: @@ -161,7 +195,7 @@ def extract_prompt_info( prompt_text = " ".join(prompt_parts) else: prompt = request.get("prompt") - total_length = server.get_total_token_length(prompt, model) + total_length = count_tokens(prompt, model) max_tokens = request.get("max_tokens", 0) prompt_text = prompt if isinstance(prompt, str) else str(prompt) return total_length, max_tokens, prompt_text @@ -195,7 +229,7 @@ def build_zmq_prepare_request( request: dict, prompt_tokens: list[int], request_id: str, - receiver, + receiver: ZmqReceiverConfig, first_token_source: str, is_chat: bool = False, ) -> dict: @@ -223,10 +257,9 @@ def build_zmq_prepare_request( def tokenize_zmq_prompt( - request: dict, is_chat: bool, server: "Proxy" # noqa: F821 + request: dict, is_chat: bool, tokenizer: Tokenizer | None ) -> list[int]: """Tokenize the exact prompt used by the LMCache sender and receiver.""" - tokenizer = server.get_tokenizer(request.get("model", "")) if tokenizer is None: raise ValueError( "ZMQ prompt tokenization requires a tokenizer for model " @@ -540,10 +573,23 @@ async def _zmq_prefill_only_generator( async def handle_completion( - endpoint: str, raw_request: Request, - server: "Proxy", # noqa: F821 + *, + endpoint: str, is_chat: bool, + reserve: Callable[[SchedulingContext], Reservation | None], + forward_request: ForwardRequest, + count_tokens: CountTokens, + get_tokenizer: Callable[[str], Tokenizer | None], + tokenize_on_backend: TokenizeOnBackend, + is_aggregated_model: Callable[[str], bool], + options: CompletionOptions, + get_registered_models: Callable[[], list[str]] | None = None, + record_success: Callable[[str], None] | None = None, + record_failure: Callable[[str], None] | None = None, + notifications: Notifications | None = None, + stream_generator: FirstTokenGenerator | None = None, + nonstream_generator: FirstTokenGenerator = D_first_token_generator, ) -> JSONResponse | StreamingResponse: """Unified completion handler for both /v1/completions and /v1/chat/completions.""" _metrics_start = track_request_start(endpoint) @@ -569,7 +615,7 @@ async def handle_completion( start_time = time.time() total_length, max_tokens, prompt_text = extract_prompt_info( - request, is_chat, server + request, is_chat, count_tokens ) end_time = time.time() elapsed_ms = (end_time - start_time) * 1000 @@ -587,33 +633,38 @@ async def handle_completion( model_label = requested_model or "unknown" # Aggregated-role fast path: single forward, no P→D split - if server._is_aggregated_model(requested_model): + if is_aggregated_model(requested_model): return await _handle_aggregated_completion( endpoint, request, raw_request, - server, requested_model, total_length, max_tokens, prompt_text, _metrics_start, handler_name, + reserve=reserve, + forward_request=forward_request, + get_registered_models=get_registered_models, + record_success=record_success, + record_failure=record_failure, ) zmq_prompt_tokens = None - if server.disaggregated_mode == "zmq": - if server.get_tokenizer(requested_model) is None: - zmq_prompt_tokens = await server.tokenize_on_backend(request, is_chat) + if options.disaggregated_mode == "zmq": + tokenizer = get_tokenizer(requested_model) + if tokenizer is None: + zmq_prompt_tokens = await tokenize_on_backend(request, is_chat) else: - zmq_prompt_tokens = tokenize_zmq_prompt(request, is_chat, server) + zmq_prompt_tokens = tokenize_zmq_prompt(request, is_chat, tokenizer) total_length = len(zmq_prompt_tokens) kv_prepare_request = build_kv_prepare_request( request, is_chat, - server.disaggregated_mode, + options.disaggregated_mode, ) upstream_headers = None - if server.disaggregated_mode == "nixl": + if options.disaggregated_mode == "nixl": request_id = raw_request.headers.get("x-request-id") or str(uuid.uuid4()) upstream_headers = {"X-Request-Id": request_id} @@ -632,7 +683,7 @@ async def handle_completion( } reservation = _RequestReservation() - reservation.prefill = server.reserve( + reservation.prefill = reserve( SchedulingContext( role="prefill", request_len=total_length, @@ -640,7 +691,7 @@ async def handle_completion( **_sched_kwargs, ) ) - reservation.decode = server.reserve( + reservation.decode = reserve( SchedulingContext( role="decode", request_len=total_length, @@ -664,8 +715,8 @@ async def handle_completion( reservation.release_all() # Check for unknown model first to return a clean 404 without # triggering error-path side effects (logging, metrics, etc.) - if requested_model and server.registry is not None: - known_models = server.registry.get_registered_models() + if requested_model and get_registered_models is not None: + known_models = get_registered_models() if requested_model not in known_models: return error_response( f"The model '{requested_model}' does not exist", @@ -683,13 +734,13 @@ async def handle_completion( zmq_request_id = None zmq_wait_for_notification = False - if server.disaggregated_mode == "zmq": - if server.zmq_notifications is None or server.zmq_config is None: + if options.disaggregated_mode == "zmq": + if notifications is None or options.zmq is None: raise HTTPException( status_code=503, detail="ZMQ notification listener is not ready", ) - receiver = server.zmq_config.receivers[decode_instance] + receiver = options.zmq.receivers[decode_instance] zmq_request_id = str(uuid.uuid4()) assert zmq_prompt_tokens is not None kv_prepare_request = build_zmq_prepare_request( @@ -697,11 +748,11 @@ async def handle_completion( zmq_prompt_tokens, zmq_request_id, receiver, - server.first_token_source, + options.first_token_source, is_chat, ) zmq_wait_for_notification = True - await server.zmq_notifications.register(zmq_request_id) + await notifications.register(zmq_request_id) # Track per-instance request counters proxy_prefill_requests_total.labels( @@ -729,16 +780,16 @@ async def handle_completion( value = b"" try: - async for chunk in server.forward_request( + async for chunk in forward_request( f"http://{prefill_instance}" - f"{'/v1/completions' if server.disaggregated_mode == 'zmq' else endpoint}", # noqa: E501 + f"{'/v1/completions' if options.disaggregated_mode == 'zmq' else endpoint}", # noqa: E501 kv_prepare_request, extra_headers=upstream_headers, ): value += chunk except CancelledError: if zmq_wait_for_notification and zmq_request_id is not None: - await server.zmq_notifications.discard(zmq_request_id) + await notifications.discard(zmq_request_id) proxy_prefill_active_requests.labels( prefill_instance=prefill_instance, decode_instance=decode_instance, @@ -747,9 +798,9 @@ async def handle_completion( raise except HTTPException as http_exc: if zmq_wait_for_notification and zmq_request_id is not None: - await server.zmq_notifications.discard(zmq_request_id) + await notifications.discard(zmq_request_id) reservation.release_all() - server._record_failure(prefill_instance, decode_instance) + _record_failures(record_failure, prefill_instance, decode_instance) proxy_instance_errors_total.labels( instance=prefill_instance, error_type="prefill_forward_error", @@ -778,7 +829,7 @@ async def handle_completion( value = ( value.strip().decode("utf-8").removesuffix("data: [DONE]").encode("utf-8") ) - if server.disaggregated_mode == "nixl": + if options.disaggregated_mode == "nixl": try: prefill_output = json.loads(value) except json.JSONDecodeError as exc: @@ -794,26 +845,26 @@ async def handle_completion( ) request = request.copy() request["kv_transfer_params"] = kv_transfer_params - elif server.disaggregated_mode == "zmq": + elif options.disaggregated_mode == "zmq": prefill_output = None first_token = None - if server.first_token_source == "prefill": + if options.first_token_source == "prefill": try: prefill_output = json.loads(value) first_token = prefill_output["kv_transfer_params"]["first_tok"] except (json.JSONDecodeError, KeyError, TypeError) as exc: assert zmq_request_id is not None - await server.zmq_notifications.discard(zmq_request_id) + await notifications.discard(zmq_request_id) raise HTTPException( status_code=502, detail=( "Prefill response did not contain an LMCache " "first token" ), ) from exc - receiver = server.zmq_config.receivers[decode_instance] + receiver = options.zmq.receivers[decode_instance] if zmq_wait_for_notification: try: - await server.zmq_notifications.wait( + await notifications.wait( zmq_request_id, len(receiver.init_ports), ) @@ -834,8 +885,8 @@ async def handle_completion( is_chat, ) - if server.registry is not None: - server.registry.record_success(prefill_instance) + if record_success is not None: + record_success(prefill_instance) async def streaming_response(value): if value: @@ -844,8 +895,8 @@ async def streaming_response(value): yield b"" prefill_only = ( - server.disaggregated_mode == "zmq" - and server.first_token_source == "prefill" + options.disaggregated_mode == "zmq" + and options.first_token_source == "prefill" and max_tokens == 1 ) if prefill_only: @@ -860,11 +911,11 @@ async def streaming_response(value): first_token_from_p = True else: generator_p = streaming_response(value) - # server.forward_request() is an async generator; HTTP errors + # forward_request() is an async generator; HTTP errors # surface only when wrapped_generator() iterates it. - generator_d_raw = server.forward_request( + generator_d_raw = forward_request( f"http://{decode_instance}" - f"{'/v1/completions' if server.disaggregated_mode == 'zmq' and is_chat else endpoint}", # noqa: E501 + f"{'/v1/completions' if options.disaggregated_mode == 'zmq' and is_chat else endpoint}", # noqa: E501 request, extra_headers=upstream_headers, ) @@ -872,8 +923,8 @@ async def streaming_response(value): generator_d = decode_tracker if ( - server.disaggregated_mode == "zmq" - and server.first_token_source == "prefill" + options.disaggregated_mode == "zmq" + and options.first_token_source == "prefill" ): assert prefill_output is not None if request.get("stream", False): @@ -892,22 +943,26 @@ async def streaming_response(value): ) first_token_from_p = True else: - if server.disaggregated_mode == "zmq" and is_chat: + if options.disaggregated_mode == "zmq" and is_chat: generator_d = ( _chat_completion_stream(generator_d) if request.get("stream", False) else _chat_completion_nonstream(generator_d) ) if request.get("stream", False): - generator_class = server.generator + generator_class = stream_generator or ( + P_first_token_generator + if options.first_token_source == "prefill" + else D_first_token_generator + ) else: - generator_class = server.d_first_token_generator_class + generator_class = nonstream_generator # Determine if user's first token comes from prefill or decode node. # P_first_token_generator yields P's token first; # D_first_token_generator # discards P's token and yields D's tokens only. first_token_from_p = ( - server.first_token_source == "prefill" + options.first_token_source == "prefill" and generator_class is P_first_token_generator ) final_generator = generator_class( @@ -924,8 +979,8 @@ async def wrapped_generator(): try: async for chunk in final_generator: yield chunk - if server.registry is not None and not prefill_only: - server.registry.record_success(decode_instance) + if record_success is not None and not prefill_only: + record_success(decode_instance) except CancelledError: logger.warning( "[0]Client disconnected during %s (CancelledError)", @@ -933,7 +988,7 @@ async def wrapped_generator(): ) except HTTPException as http_exc: reservation.release_all() - server._record_failure(prefill_instance, decode_instance) + _record_failures(record_failure, prefill_instance, decode_instance) proxy_instance_errors_total.labels( instance=decode_instance, error_type="decode_forward_error", @@ -1011,16 +1066,21 @@ async def _handle_aggregated_completion( endpoint: str, request: dict, raw_request: Request, - server: "Proxy", # noqa: F821 model: str, total_length: int, max_tokens: int, prompt_text: str, metrics_start: float, handler_name: str, + *, + reserve: Callable[[SchedulingContext], Reservation | None], + forward_request: ForwardRequest, + get_registered_models: Callable[[], list[str]] | None, + record_success: Callable[[str], None] | None, + record_failure: Callable[[str], None] | None, ) -> JSONResponse | StreamingResponse: """Single-pass completion for aggregated-role instances.""" - reservation = server.reserve( + reservation = reserve( SchedulingContext( role="aggregated", model=model, @@ -1050,8 +1110,8 @@ async def _handle_aggregated_completion( if instance is None: track_request_end(endpoint, metrics_start) # Check for unknown model - if model and server.registry is not None: - known_models = server.registry.get_registered_models() + if model and get_registered_models is not None: + known_models = get_registered_models() if model not in known_models: return error_response( f"The model '{model}' does not exist", @@ -1067,7 +1127,7 @@ async def _handle_aggregated_completion( url = f"http://{instance}{endpoint}" if request.get("stream", False): - generator = server.forward_request(url, request) + generator = forward_request(url, request) async def wrapped(): _ok = False @@ -1084,22 +1144,22 @@ async def wrapped(): except Exception as e: _ok = False logger.error("Exception in aggregated stream: %s", str(e)) - if server.registry is not None: - server.registry.record_failure(instance) + if record_failure is not None: + record_failure(instance) raise finally: reservation.release() - if _ok and server.registry is not None: - server.registry.record_success(instance) + if _ok and record_success is not None: + record_success(instance) track_request_end(endpoint, metrics_start) return StreamingResponse(wrapped(), media_type="text/event-stream") else: - # Non-streaming: forward request using server.forward_request() + # Non-streaming: forward request using forward_request() # for consistent auth, timeouts, and connection handling. try: value = b"" - async for chunk in server.forward_request(url, request): + async for chunk in forward_request(url, request): value += chunk data = json.loads(value) # Detect error responses from upstream. forward_request yields @@ -1123,16 +1183,16 @@ async def wrapped(): else: status_code = 200 reservation.release() - if status_code < 400 and server.registry is not None: - server.registry.record_success(instance) - elif status_code >= 400 and server.registry is not None: - server.registry.record_failure(instance) + if status_code < 400 and record_success is not None: + record_success(instance) + elif status_code >= 400 and record_failure is not None: + record_failure(instance) track_request_end(endpoint, metrics_start) return JSONResponse(data, status_code=status_code) except HTTPException as http_exc: reservation.release() - if server.registry is not None: - server.registry.record_failure(instance) + if record_failure is not None: + record_failure(instance) track_request_end(endpoint, metrics_start) return error_response( str(http_exc.detail), @@ -1145,8 +1205,8 @@ async def wrapped(): except Exception as e: logger.error("Error in aggregated non-streaming: %s", str(e)) reservation.release() - if server.registry is not None: - server.registry.record_failure(instance) + if record_failure is not None: + record_failure(instance) track_request_end(endpoint, metrics_start) return error_response( "Internal proxy error", diff --git a/xpyd/interfaces.py b/xpyd/interfaces.py new file mode 100644 index 0000000..ef03492 --- /dev/null +++ b/xpyd/interfaces.py @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Narrow callable contracts used by HTTP endpoints and request execution.""" + +from collections.abc import AsyncIterator, Awaitable, Callable +from typing import Any, Protocol + +from fastapi import Request +from fastapi.responses import JSONResponse, Response + +AsyncRequestHandler = Callable[[Request], Awaitable[Response]] +RequestHandler = Callable[[Request], Response | Awaitable[Response]] +PostToInstance = Callable[[Request, str, dict[str, Any]], Awaitable[JSONResponse]] +CountTokens = Callable[[Any, str], int] +TokenizeOnBackend = Callable[[dict[str, Any], bool], Awaitable[list[int]]] + + +class GetFromInstance(Protocol): + async def __call__( + self, path: str, is_full_instancelist: int = 0 + ) -> JSONResponse: ... + + +class ForwardRequest(Protocol): + def __call__( + self, + url: str, + data: dict[str, Any], + *, + extra_headers: dict[str, str] | None = None, + ) -> AsyncIterator[bytes]: ... + + +class Tokenizer(Protocol): + def encode(self, text: str) -> list[int]: ... + + def apply_chat_template(self, messages: list[dict], **kwargs: Any) -> Any: ... + + +class Notifications(Protocol): + async def register(self, request_id: str) -> None: ... + + async def wait(self, request_id: str, expected: int) -> None: ... + + async def discard(self, request_id: str) -> None: ... diff --git a/xpyd/proxy.py b/xpyd/proxy.py index ccfccef..b75ea6b 100644 --- a/xpyd/proxy.py +++ b/xpyd/proxy.py @@ -17,21 +17,29 @@ import logging import os from collections.abc import AsyncGenerator +from functools import partial from pathlib import Path -from typing import Any, Callable, Optional +from typing import Any, Optional import aiohttp import uvicorn -from fastapi import APIRouter, HTTPException, Request -from fastapi.responses import JSONResponse, StreamingResponse +from fastapi import APIRouter, FastAPI, HTTPException, Request +from fastapi.responses import JSONResponse from transformers import AutoTokenizer from xpyd.app import create_app from xpyd.cli import load_config -from xpyd.completion_handler import D_first_token_generator, P_first_token_generator +from xpyd.completion_handler import ( + CompletionOptions, + D_first_token_generator, + P_first_token_generator, + handle_completion, +) from xpyd.config import ProxyConfig +from xpyd.discovery import NodeDiscovery from xpyd.errors import INVALID_REQUEST, PROXY_ERROR, SERVER_ERROR, error_response from xpyd.health_monitor import HealthMonitor +from xpyd.interfaces import Notifications, RequestHandler from xpyd.registry import InstanceRegistry from xpyd.routes import register_routes from xpyd.scheduler import ( @@ -41,6 +49,7 @@ SchedulingPolicy, default_registry, ) +from xpyd.zmq_notifications import ZmqNotificationListener class _ExtraFormatter(logging.Formatter): @@ -89,12 +98,8 @@ def __init__( decode_instances: list[str], model: str, scheduling_policy: SchedulingPolicy, - custom_create_completion: Optional[ - Callable[[Request], StreamingResponse] - ] = None, - custom_create_chat_completion: Optional[ - Callable[[Request], StreamingResponse] - ] = None, + custom_create_completion: RequestHandler | None = None, + custom_create_chat_completion: RequestHandler | None = None, first_token_source: str = "decode", registry: Optional[InstanceRegistry] = None, aggregated_instances: Optional[dict[str, list[str]]] = None, @@ -104,6 +109,7 @@ def __init__( zmq_config=None, scheduling_options: Optional[dict[str, Any]] = None, model_policy_options: Optional[dict[str, dict[str, Any]]] = None, + zmq_notifications: Notifications | None = None, ): self.prefill_instances = prefill_instances self.decode_instances = decode_instances @@ -121,7 +127,7 @@ def __init__( self.disaggregated_mode = disaggregated_mode self.first_token_source = first_token_source self.zmq_config = zmq_config - self.zmq_notifications = None + self.zmq_notifications = zmq_notifications self._aggregated_policies: dict[str, SchedulingPolicy] = {} self._scheduling_options = scheduling_options or {} self._model_policy_options = model_policy_options or {} @@ -131,8 +137,6 @@ def __init__( self.health_monitor = None self.discovery = None self._instance_mutation_lock = asyncio.Lock() - self.router = APIRouter() - self.setup_routes() self.generator = ( P_first_token_generator if first_token_source == "prefill" @@ -140,6 +144,7 @@ def __init__( ) self.d_first_token_generator_class = D_first_token_generator self.tokenizer = None + self.setup_routes() def ensure_tokenizer(self, model: str) -> bool: """Load a model tokenizer or mark the model for round-robin fallback.""" @@ -271,7 +276,60 @@ def reserve(self, context: SchedulingContext) -> Optional[Reservation]: return self._scheduler.reserve(policy, context, instances) def setup_routes(self) -> None: - register_routes(self.router, self) + """Bind runtime operations at the composition boundary.""" + completion = partial( + handle_completion, + reserve=self.reserve, + forward_request=self.forward_request, + count_tokens=self.get_total_token_length, + get_tokenizer=self.get_tokenizer, + tokenize_on_backend=self.tokenize_on_backend, + is_aggregated_model=self._is_aggregated_model, + options=CompletionOptions( + disaggregated_mode=self.disaggregated_mode, + first_token_source=self.first_token_source, + zmq=self.zmq_config, + ), + get_registered_models=( + self.registry.get_registered_models + if self.registry is not None + else None + ), + record_success=( + self.registry.record_success if self.registry is not None else None + ), + record_failure=( + self.registry.record_failure if self.registry is not None else None + ), + notifications=self.zmq_notifications, + stream_generator=self.generator, + nonstream_generator=self.d_first_token_generator_class, + ) + router = APIRouter() + register_routes( + router, + completion=partial(completion, endpoint="/v1/completions", is_chat=False), + chat_completion=partial( + completion, endpoint="/v1/chat/completions", is_chat=True + ), + forward=self.post_to_instance, + fetch=self.get_from_instance, + add_instance=self.add_instance, + remove_instance=self.drain_and_remove_instance, + prefill_instances=self.prefill_instances, + decode_instances=self.decode_instances, + get_instances=( + self.registry.get_all_instances if self.registry is not None else None + ), + get_registered_models=( + self.registry.get_registered_models + if self.registry is not None + else None + ), + custom_completion=self.custom_create_completion, + custom_chat_completion=self.custom_create_chat_completion, + ) + self.router = router async def forward_request( self, @@ -751,8 +809,8 @@ class ProxyServer: def __init__( self, config: ProxyConfig, - create_completion: Optional[Callable[[Request], StreamingResponse]] = None, - create_chat_completion: Optional[Callable[[Request], StreamingResponse]] = None, + create_completion: RequestHandler | None = None, + create_chat_completion: RequestHandler | None = None, ): self.config = config self.port = config.port @@ -887,6 +945,16 @@ def __init__( all_decode=all_decode, ) + self.zmq_notifications = None + if config.disaggregated_mode == "zmq": + zmq_config = config.zmq + assert zmq_config is not None + self.zmq_notifications = ZmqNotificationListener( + zmq_config.host, + zmq_config.port, + zmq_config.notification_timeout_seconds, + ) + self.proxy_instance = Proxy( prefill_instances=all_prefill, decode_instances=all_decode, @@ -903,10 +971,35 @@ def __init__( zmq_config=config.zmq, scheduling_options=config.scheduling_config.get(config.scheduling, {}), model_policy_options=config.scheduling_config, + zmq_notifications=self.zmq_notifications, + ) + + self.discovery = NodeDiscovery( + prefill_instances=self._all_prefill, + decode_instances=self._all_decode, + probe_interval=config.probe_interval_seconds, + wait_timeout=config.wait_timeout_seconds, + heartbeat_interval=config.heartbeat_interval_seconds, + registry=self.registry, + aggregated_instances=self._all_aggregated, + on_model_discovered=lambda model: asyncio.to_thread( + self.proxy_instance.ensure_tokenizer, model + ), + ) + self.proxy_instance.discovery = self.discovery + self.proxy_instance.health_monitor = self.health_monitor + + def create_app(self) -> FastAPI: + self.proxy_instance.setup_routes() + return create_app( + self.proxy_instance.router, + discovery=self.discovery, + health_monitor=self.health_monitor, + zmq_notifications=self.zmq_notifications, ) def run_server(self) -> None: - app = create_app(self.config, self.proxy_instance, self.health_monitor) + app = self.create_app() config = uvicorn.Config( app, host=self.config.host, diff --git a/xpyd/routes/__init__.py b/xpyd/routes/__init__.py index f0ef4f8..aa7ef42 100644 --- a/xpyd/routes/__init__.py +++ b/xpyd/routes/__init__.py @@ -1,10 +1,18 @@ # SPDX-License-Identifier: Apache-2.0 """Route registration aggregator for MicroDisaggregatedProxy.""" -from typing import TYPE_CHECKING +from collections.abc import Awaitable, Callable from fastapi import APIRouter +from xpyd.interfaces import ( + AsyncRequestHandler, + GetFromInstance, + PostToInstance, + RequestHandler, +) +from xpyd.registry import InstanceInfo + from .chat_completions import register as register_chat_completions from .completions import register as register_completions from .detokenize import register as register_detokenize @@ -21,24 +29,45 @@ from .tokenize import register as register_tokenize from .version import register as register_version -if TYPE_CHECKING: - from xpyd.proxy import Proxy - -def register_routes(router: APIRouter, server: "Proxy") -> None: - """Register all route handlers on *router*, using *server* for state.""" - register_completions(router, server) - register_chat_completions(router, server) - register_tokenize(router, server) - register_detokenize(router, server) - register_embeddings(router, server) - register_pooling(router, server) - register_score(router, server) - register_rerank(router, server) - register_invocations(router, server) - register_status(router, server) - register_instances(router, server) - register_health(router, server) - register_models(router, server) - register_version(router, server) - register_metrics(router, server) +def register_routes( + router: APIRouter, + *, + completion: AsyncRequestHandler, + chat_completion: AsyncRequestHandler, + forward: PostToInstance, + fetch: GetFromInstance, + add_instance: Callable[[str, str], Awaitable[bool]], + remove_instance: Callable[[str, str, float], Awaitable[None]], + prefill_instances: list[str], + decode_instances: list[str], + get_instances: Callable[[], list[InstanceInfo]] | None = None, + get_registered_models: Callable[[], list[str]] | None = None, + custom_completion: RequestHandler | None = None, + custom_chat_completion: RequestHandler | None = None, +) -> None: + """Wire each endpoint to only the operations and data it needs.""" + register_completions(router, execute=completion, custom_handler=custom_completion) + register_chat_completions( + router, execute=chat_completion, custom_handler=custom_chat_completion + ) + register_tokenize(router, forward=forward) + register_detokenize(router, forward=forward) + register_embeddings(router, forward=forward) + register_pooling(router, forward=forward) + register_score(router, forward=forward) + register_rerank(router, forward=forward) + register_invocations(router, forward=forward) + register_status( + router, + prefill_instances=prefill_instances, + decode_instances=decode_instances, + get_instances=get_instances, + ) + register_instances( + router, add_instance=add_instance, remove_instance=remove_instance + ) + register_health(router, fetch=fetch) + register_models(router, fetch=fetch, get_registered_models=get_registered_models) + register_version(router, fetch=fetch) + register_metrics(router) diff --git a/xpyd/routes/chat_completions.py b/xpyd/routes/chat_completions.py index c2f2098..0033d33 100644 --- a/xpyd/routes/chat_completions.py +++ b/xpyd/routes/chat_completions.py @@ -1,24 +1,23 @@ # SPDX-License-Identifier: Apache-2.0 """The /v1/chat/completions endpoint.""" -from typing import TYPE_CHECKING - from fastapi import APIRouter, Depends, Request -from xpyd.completion_handler import handle_completion, validate_json_content_type - -if TYPE_CHECKING: - from xpyd.proxy import Proxy +from xpyd.completion_handler import validate_json_content_type +from xpyd.interfaces import AsyncRequestHandler, RequestHandler -def register(router: APIRouter, server: "Proxy") -> None: +def register( + router: APIRouter, + *, + execute: AsyncRequestHandler, + custom_handler: RequestHandler | None = None, +) -> None: async def create_chat_completion(raw_request: Request): - return await handle_completion( - "/v1/chat/completions", raw_request, server, is_chat=True - ) + return await execute(raw_request) router.post( "/v1/chat/completions", dependencies=[Depends(validate_json_content_type)], - )(server.custom_create_chat_completion or create_chat_completion) + )(custom_handler or create_chat_completion) router.options("/v1/chat/completions")(lambda: None) diff --git a/xpyd/routes/completions.py b/xpyd/routes/completions.py index 289e473..96d8c52 100644 --- a/xpyd/routes/completions.py +++ b/xpyd/routes/completions.py @@ -1,24 +1,23 @@ # SPDX-License-Identifier: Apache-2.0 """The /v1/completions endpoint.""" -from typing import TYPE_CHECKING - from fastapi import APIRouter, Depends, Request -from xpyd.completion_handler import handle_completion, validate_json_content_type - -if TYPE_CHECKING: - from xpyd.proxy import Proxy +from xpyd.completion_handler import validate_json_content_type +from xpyd.interfaces import AsyncRequestHandler, RequestHandler -def register(router: APIRouter, server: "Proxy") -> None: +def register( + router: APIRouter, + *, + execute: AsyncRequestHandler, + custom_handler: RequestHandler | None = None, +) -> None: async def create_completion(raw_request: Request): - return await handle_completion( - "/v1/completions", raw_request, server, is_chat=False - ) + return await execute(raw_request) router.post( "/v1/completions", dependencies=[Depends(validate_json_content_type)], - )(server.custom_create_completion or create_completion) + )(custom_handler or create_completion) router.options("/v1/completions")(lambda: None) diff --git a/xpyd/routes/detokenize.py b/xpyd/routes/detokenize.py index e3601df..f87d07b 100644 --- a/xpyd/routes/detokenize.py +++ b/xpyd/routes/detokenize.py @@ -1,20 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 """The /detokenize endpoint.""" -from typing import TYPE_CHECKING - from fastapi import APIRouter, Request from fastapi.responses import JSONResponse -if TYPE_CHECKING: - from xpyd.proxy import Proxy +from xpyd.interfaces import PostToInstance -def register(router: APIRouter, server: "Proxy") -> None: +def register(router: APIRouter, *, forward: PostToInstance) -> None: async def post_detokenize(request: Request): - return await server.post_to_instance( - request, "/detokenize", {"model": "", "tokens": []} - ) + return await forward(request, "/detokenize", {"model": "", "tokens": []}) router.post("/detokenize", response_class=JSONResponse)(post_detokenize) router.options("/detokenize")(lambda: None) diff --git a/xpyd/routes/embeddings.py b/xpyd/routes/embeddings.py index 89cc5b0..be4ebea 100644 --- a/xpyd/routes/embeddings.py +++ b/xpyd/routes/embeddings.py @@ -1,20 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 """The /v1/embeddings endpoint.""" -from typing import TYPE_CHECKING - from fastapi import APIRouter, Request from fastapi.responses import JSONResponse -if TYPE_CHECKING: - from xpyd.proxy import Proxy +from xpyd.interfaces import PostToInstance -def register(router: APIRouter, server: "Proxy") -> None: +def register(router: APIRouter, *, forward: PostToInstance) -> None: async def post_embeddings(request: Request): - return await server.post_to_instance( - request, "/v1/embeddings", {"model": "", "input": ""} - ) + return await forward(request, "/v1/embeddings", {"model": "", "input": ""}) router.post("/v1/embeddings", response_class=JSONResponse)(post_embeddings) router.options("/v1/embeddings")(lambda: None) diff --git a/xpyd/routes/health.py b/xpyd/routes/health.py index fc206ca..c7edb87 100644 --- a/xpyd/routes/health.py +++ b/xpyd/routes/health.py @@ -1,22 +1,19 @@ # SPDX-License-Identifier: Apache-2.0 """The /health and /ping endpoints.""" -from typing import TYPE_CHECKING - from fastapi import APIRouter from fastapi.responses import PlainTextResponse -if TYPE_CHECKING: - from xpyd.proxy import Proxy +from xpyd.interfaces import GetFromInstance -def register(router: APIRouter, server: "Proxy") -> None: +def register(router: APIRouter, *, fetch: GetFromInstance) -> None: async def get_health(): - return await server.get_from_instance("/health", is_full_instancelist=1) + return await fetch("/health", is_full_instancelist=1) async def get_ping(): - return await server.get_from_instance("/ping", is_full_instancelist=1) + return await fetch("/ping", is_full_instancelist=1) router.get("/health", response_class=PlainTextResponse)(get_health) router.get("/ping", response_class=PlainTextResponse)(get_ping) diff --git a/xpyd/routes/instances.py b/xpyd/routes/instances.py index df97015..84bd018 100644 --- a/xpyd/routes/instances.py +++ b/xpyd/routes/instances.py @@ -4,20 +4,22 @@ import ipaddress import logging import os -from typing import TYPE_CHECKING +from collections.abc import Awaitable, Callable from fastapi import APIRouter, Header, Request from fastapi.responses import JSONResponse from xpyd.errors import INVALID_REQUEST, SERVER_ERROR, error_response -if TYPE_CHECKING: - from xpyd.proxy import Proxy - logger = logging.getLogger("xpyd.proxy") -def register(router: APIRouter, server: "Proxy") -> None: +def register( + router: APIRouter, + *, + add_instance: Callable[[str, str], Awaitable[bool]], + remove_instance: Callable[[str, str, float], Awaitable[None]], +) -> None: """Register instance administration routes on *router*.""" def _authenticate_api_key(x_api_key: str): @@ -62,7 +64,7 @@ async def add_instance_endpoint( except Exception: return error_response("Invalid instance address", INVALID_REQUEST, 400) - is_valid = await server.add_instance(instance_type, instance) + is_valid = await add_instance(instance_type, instance) if not is_valid: return error_response( "Instance validation failed", INVALID_REQUEST, 400 @@ -105,7 +107,7 @@ async def remove_instance_endpoint( 400, ) - await server.drain_and_remove_instance( + await remove_instance( instance_type, instance, float(timeout_seconds), diff --git a/xpyd/routes/invocations.py b/xpyd/routes/invocations.py index d6f0a68..bf8e205 100644 --- a/xpyd/routes/invocations.py +++ b/xpyd/routes/invocations.py @@ -1,20 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 """The /invocations endpoint.""" -from typing import TYPE_CHECKING - from fastapi import APIRouter, Request from fastapi.responses import JSONResponse -if TYPE_CHECKING: - from xpyd.proxy import Proxy +from xpyd.interfaces import PostToInstance -def register(router: APIRouter, server: "Proxy") -> None: +def register(router: APIRouter, *, forward: PostToInstance) -> None: async def post_invocations(request: Request): - return await server.post_to_instance( - request, "/invocations", {"model": "", "prompt": ""} - ) + return await forward(request, "/invocations", {"model": "", "prompt": ""}) router.post("/invocations", response_class=JSONResponse)(post_invocations) router.options("/invocations")(lambda: None) diff --git a/xpyd/routes/metrics.py b/xpyd/routes/metrics.py index 7087ec4..b513b3b 100644 --- a/xpyd/routes/metrics.py +++ b/xpyd/routes/metrics.py @@ -1,18 +1,13 @@ # SPDX-License-Identifier: Apache-2.0 """The /metrics endpoint.""" -from typing import TYPE_CHECKING - from fastapi import APIRouter from fastapi.responses import Response from xpyd.metrics import get_metrics -if TYPE_CHECKING: - from xpyd.proxy import Proxy - -def register(router: APIRouter, server: "Proxy") -> None: +def register(router: APIRouter) -> None: async def get_metrics_endpoint(): return Response( content=get_metrics(), diff --git a/xpyd/routes/models.py b/xpyd/routes/models.py index d5a2686..8a20494 100644 --- a/xpyd/routes/models.py +++ b/xpyd/routes/models.py @@ -1,22 +1,26 @@ # SPDX-License-Identifier: Apache-2.0 """The /v1/models endpoint.""" -from typing import TYPE_CHECKING +from collections.abc import Callable from fastapi import APIRouter from fastapi.responses import JSONResponse -if TYPE_CHECKING: - from xpyd.proxy import Proxy +from xpyd.interfaces import GetFromInstance # Fixed epoch timestamp for model listing (avoids changing on every request). _MODEL_CREATED_EPOCH = 0 -def register(router: APIRouter, server: "Proxy") -> None: +def register( + router: APIRouter, + *, + fetch: GetFromInstance, + get_registered_models: Callable[[], list[str]] | None = None, +) -> None: async def get_models(): - if server.registry is not None: - models = server.registry.get_registered_models() + if get_registered_models is not None: + models = get_registered_models() data = [ { "id": model, @@ -30,7 +34,7 @@ async def get_models(): content={"object": "list", "data": data}, status_code=200, ) - return await server.get_from_instance("/v1/models") + return await fetch("/v1/models") router.get("/v1/models", response_class=JSONResponse)(get_models) router.options("/v1/models")(lambda: None) diff --git a/xpyd/routes/pooling.py b/xpyd/routes/pooling.py index 3177424..b1f9334 100644 --- a/xpyd/routes/pooling.py +++ b/xpyd/routes/pooling.py @@ -1,20 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 """The /pooling endpoint.""" -from typing import TYPE_CHECKING - from fastapi import APIRouter, Request from fastapi.responses import JSONResponse -if TYPE_CHECKING: - from xpyd.proxy import Proxy +from xpyd.interfaces import PostToInstance -def register(router: APIRouter, server: "Proxy") -> None: +def register(router: APIRouter, *, forward: PostToInstance) -> None: async def post_pooling(request: Request): - return await server.post_to_instance( - request, "/pooling", {"model": "", "messages": ""} - ) + return await forward(request, "/pooling", {"model": "", "messages": ""}) router.post("/pooling", response_class=JSONResponse)(post_pooling) router.options("/pooling")(lambda: None) diff --git a/xpyd/routes/rerank.py b/xpyd/routes/rerank.py index 3be59ff..85c183d 100644 --- a/xpyd/routes/rerank.py +++ b/xpyd/routes/rerank.py @@ -1,28 +1,25 @@ # SPDX-License-Identifier: Apache-2.0 """The /rerank, /v1/rerank and /v2/rerank endpoints.""" -from typing import TYPE_CHECKING - from fastapi import APIRouter, Request from fastapi.responses import JSONResponse -if TYPE_CHECKING: - from xpyd.proxy import Proxy +from xpyd.interfaces import PostToInstance -def register(router: APIRouter, server: "Proxy") -> None: +def register(router: APIRouter, *, forward: PostToInstance) -> None: async def post_rerank(request: Request): - return await server.post_to_instance( + return await forward( request, "/rerank", {"model": "", "query": "", "documents": ""} ) async def post_rerankv1(request: Request): - return await server.post_to_instance( + return await forward( request, "/v1/rerank", {"model": "", "query": "", "documents": ""} ) async def post_rerankv2(request: Request): - return await server.post_to_instance( + return await forward( request, "/v2/rerank", {"model": "", "query": "", "documents": ""} ) diff --git a/xpyd/routes/score.py b/xpyd/routes/score.py index 6f22da3..9e33f0a 100644 --- a/xpyd/routes/score.py +++ b/xpyd/routes/score.py @@ -1,25 +1,22 @@ # SPDX-License-Identifier: Apache-2.0 """The /score and /v1/score endpoints.""" -from typing import TYPE_CHECKING - from fastapi import APIRouter, Request from fastapi.responses import JSONResponse -if TYPE_CHECKING: - from xpyd.proxy import Proxy +from xpyd.interfaces import PostToInstance -def register(router: APIRouter, server: "Proxy") -> None: +def register(router: APIRouter, *, forward: PostToInstance) -> None: async def post_score(request: Request): - return await server.post_to_instance( + return await forward( request, "/score", {"model": "", "text_1": "", "text_2": "", "predictions": ""}, ) async def post_scorev1(request: Request): - return await server.post_to_instance( + return await forward( request, "/v1/score", {"model": "", "text_1": "", "text_2": "", "predictions": ""}, diff --git a/xpyd/routes/status.py b/xpyd/routes/status.py index 6c4d839..cf12a5d 100644 --- a/xpyd/routes/status.py +++ b/xpyd/routes/status.py @@ -1,31 +1,38 @@ # SPDX-License-Identifier: Apache-2.0 """Cluster and per-instance status endpoints.""" -from typing import TYPE_CHECKING +from collections.abc import Callable from fastapi import APIRouter from fastapi.responses import JSONResponse -if TYPE_CHECKING: - from xpyd.proxy import Proxy +from xpyd.registry import InstanceInfo -def register(router: APIRouter, server: "Proxy") -> None: +def register( + router: APIRouter, + *, + prefill_instances: list[str], + decode_instances: list[str], + get_instances: Callable[[], list[InstanceInfo]] | None = None, +) -> None: async def get_status(): return { - "prefill_node_count": len(server.prefill_instances), - "decode_node_count": len(server.decode_instances), - "prefill_nodes": server.prefill_instances, - "decode_nodes": server.decode_instances, + "prefill_node_count": len(prefill_instances), + "decode_node_count": len(decode_instances), + "prefill_nodes": prefill_instances, + "decode_nodes": decode_instances, } async def _instance_status(): + if get_instances is None: + raise RuntimeError("Instance registry is unavailable.") result: dict[str, list] = { "prefill_instances": [], "decode_instances": [], "aggregated_instances": [], } - for info in server.registry.get_all_instances(): + for info in get_instances(): result[f"{info.role}_instances"].append( { "address": info.address, diff --git a/xpyd/routes/tokenize.py b/xpyd/routes/tokenize.py index af23692..2784715 100644 --- a/xpyd/routes/tokenize.py +++ b/xpyd/routes/tokenize.py @@ -1,20 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 """The /tokenize endpoint.""" -from typing import TYPE_CHECKING - from fastapi import APIRouter, Request from fastapi.responses import JSONResponse -if TYPE_CHECKING: - from xpyd.proxy import Proxy +from xpyd.interfaces import PostToInstance -def register(router: APIRouter, server: "Proxy") -> None: +def register(router: APIRouter, *, forward: PostToInstance) -> None: async def post_tokenize(request: Request): - return await server.post_to_instance( - request, "/tokenize", {"model": "", "prompt": ""} - ) + return await forward(request, "/tokenize", {"model": "", "prompt": ""}) router.post("/tokenize", response_class=JSONResponse)(post_tokenize) router.options("/tokenize")(lambda: None) diff --git a/xpyd/routes/version.py b/xpyd/routes/version.py index 07d9953..785fcec 100644 --- a/xpyd/routes/version.py +++ b/xpyd/routes/version.py @@ -1,18 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 """The /version endpoint.""" -from typing import TYPE_CHECKING - from fastapi import APIRouter from fastapi.responses import JSONResponse -if TYPE_CHECKING: - from xpyd.proxy import Proxy +from xpyd.interfaces import GetFromInstance -def register(router: APIRouter, server: "Proxy") -> None: +def register(router: APIRouter, *, fetch: GetFromInstance) -> None: async def get_version(): - return await server.get_from_instance("/version") + return await fetch("/version") router.get("/version", response_class=JSONResponse)(get_version) router.options("/version")(lambda: None) From be36f5fbae17baee4e892f77c4f7f1ce71ef44f7 Mon Sep 17 00:00:00 2001 From: Tony Lin Date: Thu, 29 Oct 2026 11:34:28 +0800 Subject: [PATCH 2/2] fix: use pass bodies in Protocol stubs to clear py/ineffectual-statement CodeQL flags the standalone ellipsis expression in the new Protocol method bodies (alerts 47-53). 'pass' is equivalent for typing purposes and does not trip py/ineffectual-statement. --- xpyd/interfaces.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/xpyd/interfaces.py b/xpyd/interfaces.py index ef03492..99fdd10 100644 --- a/xpyd/interfaces.py +++ b/xpyd/interfaces.py @@ -15,9 +15,8 @@ class GetFromInstance(Protocol): - async def __call__( - self, path: str, is_full_instancelist: int = 0 - ) -> JSONResponse: ... + async def __call__(self, path: str, is_full_instancelist: int = 0) -> JSONResponse: + pass class ForwardRequest(Protocol): @@ -27,18 +26,24 @@ def __call__( data: dict[str, Any], *, extra_headers: dict[str, str] | None = None, - ) -> AsyncIterator[bytes]: ... + ) -> AsyncIterator[bytes]: + pass class Tokenizer(Protocol): - def encode(self, text: str) -> list[int]: ... + def encode(self, text: str) -> list[int]: + pass - def apply_chat_template(self, messages: list[dict], **kwargs: Any) -> Any: ... + def apply_chat_template(self, messages: list[dict], **kwargs: Any) -> Any: + pass class Notifications(Protocol): - async def register(self, request_id: str) -> None: ... + async def register(self, request_id: str) -> None: + pass - async def wait(self, request_id: str, expected: int) -> None: ... + async def wait(self, request_id: str, expected: int) -> None: + pass - async def discard(self, request_id: str) -> None: ... + async def discard(self, request_id: str) -> None: + pass