Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 47 additions & 6 deletions docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,13 +45,54 @@ for latency (smaller batch, faster per-token).

### Proxy (`xpyd/proxy.py`)

The central routing and scheduling component. It:
`Proxy` owns scheduling, tokenizer state, backend forwarding, and instance
membership. `ProxyServer` constructs these runtime components and starts uvicorn.
The existing `xpyd.proxy:main` CLI entry point remains supported.

### CLI and Application Assembly

- `xpyd/cli.py` parses arguments, loads and overrides configuration, and runs
configuration-only commands. `load_config()` returns a `ProxyConfig` when a
server should start, or `None` when a configuration command completes without
starting one. It does not import or construct the proxy runtime.
- `xpyd/app.py` exposes `create_app(config, proxy, health_monitor=None)`. It
assembles FastAPI around an existing proxy, mounts its router, and owns CORS,
readiness checks, and the discovery/health-monitor/ZMQ-listener lifecycle.
Constructing the app does not start a network listener or background services.

Backends may be offline at startup. Discovery starts in the background, inference
returns 503 until ready, and health/status/metrics/admin endpoints remain
accessible.

### Endpoint Modules (`xpyd/routes/`)

Each logical endpoint has a module with `register(router, server)`.
`routes/__init__.py` registers them all. URL aliases share their logical module:
`score.py` handles `/score` and `/v1/score`, `rerank.py` handles all three rerank
paths, and `health.py` handles `/health` and `/ping`. `status.py` serves both
`/status` and `/status/instances`; `instances.py` handles addition and removal.
Existing OPTIONS handlers stay beside their endpoints.

`completions.py` and `chat_completions.py` are separate HTTP entry points. Both
delegate to `xpyd/completion_handler.py`, which contains shared validation,
request preparation, aggregated/P-D execution, response conversion, first-token
generators, metrics, and reservation cleanup. Passthrough endpoints reuse
`Proxy.post_to_instance()`.

Runtime imports are one-way:

```text
proxy -> cli -> config
proxy -> app -> discovery / health_monitor
proxy -> routes -> completion_handler
proxy -> completion_handler
```

- Accepts OpenAI-compatible API requests from clients
- Routes requests through the prefill → decode pipeline
- Manages prefill and decode instance pools
- Supports dynamic instance addition/removal via admin API
- Provides health checking and status endpoints
App assembly, routes, and completion helpers receive the proxy object rather
than importing its implementation at runtime. References to `Proxy` for type
annotations are guarded by `TYPE_CHECKING`. Completion helpers do not import
routes or app assembly. Resource ownership during streaming is unchanged:
request completion and reservation release remain tied to response consumption.

### Scheduling Policies

Expand Down
8 changes: 7 additions & 1 deletion tests/unit/routes/test_admin_routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,16 @@
from fastapi.testclient import TestClient

# First Party
from xpyd.routes.admin import register
from xpyd.routes import instances, status

from ._helpers import build_app


def register(router, server):
instances.register(router, server)
status.register(router, server)


API_KEY = "s3cr3t"


Expand Down
81 changes: 81 additions & 0 deletions tests/unit/routes/test_completion_routes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
# SPDX-License-Identifier: Apache-2.0
"""Completion entry points share execution without coupling route modules."""

from types import SimpleNamespace
from unittest.mock import AsyncMock, patch

import pytest
from fastapi import Request
from fastapi.responses import JSONResponse
from fastapi.testclient import TestClient

from xpyd.routes import chat_completions, completions

from ._helpers import build_app


@pytest.fixture(
params=[
(completions, "/v1/completions", False, "custom_create_completion"),
(
chat_completions,
"/v1/chat/completions",
True,
"custom_create_chat_completion",
),
],
ids=["completions", "chat-completions"],
)
def endpoint(request):
return request.param


def test_default_entry_delegates_to_shared_handler(endpoint):
module, path, is_chat, _ = endpoint
server = SimpleNamespace(
custom_create_completion=None,
custom_create_chat_completion=None,
)
handler = AsyncMock(return_value=JSONResponse({"shared": True}, status_code=202))
with patch.object(module, "handle_completion", handler):
client = TestClient(build_app(module.register, server))
response = client.post(path, json={"model": "example"})

assert response.status_code == 202
assert response.json() == {"shared": True}
handler.assert_awaited_once()
assert handler.call_args.args[0] == path
assert isinstance(handler.call_args.args[1], Request)
assert handler.call_args.args[2] is server
assert handler.call_args.kwargs == {"is_chat": is_chat}
assert client.options(path).status_code == 200
other_path = "/v1/completions" if is_chat else "/v1/chat/completions"
assert client.post(other_path, json={}).status_code == 404


def test_custom_handler_keeps_content_type_validation(endpoint):
module, path, _, custom_attribute = endpoint
calls = []

async def custom(request: Request):
calls.append(await request.json())
return JSONResponse({"custom": True})

server = SimpleNamespace(
custom_create_completion=None,
custom_create_chat_completion=None,
)
setattr(server, custom_attribute, custom)
client = TestClient(build_app(module.register, server))

assert (
client.post(
path, content="{}", headers={"content-type": "text/plain"}
).status_code
== 415
)
assert calls == []
response = client.post(path, json={"model": "example"})
assert response.status_code == 200
assert response.json() == {"custom": True}
assert calls == [{"model": "example"}]
24 changes: 23 additions & 1 deletion tests/unit/routes/test_forward_routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,32 @@
from fastapi.testclient import TestClient

# First Party
from xpyd.routes.forward import register
from xpyd.routes import (
detokenize,
embeddings,
invocations,
pooling,
rerank,
score,
tokenize,
)

from ._helpers import build_app


def register(router, server):
for module in (
tokenize,
detokenize,
embeddings,
pooling,
score,
rerank,
invocations,
):
module.register(router, server)


# (route, forwarded path, expected json template, a valid request body)
ENDPOINTS = [
(
Expand Down
7 changes: 6 additions & 1 deletion tests/unit/routes/test_health_routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,16 @@

# First Party
from xpyd.proxy import Proxy
from xpyd.routes.health import register
from xpyd.routes import health, metrics, models, version

from ._helpers import FakeResponse, build_app, fake_aiohttp, make_proxy


def register(router, server):
for module in (health, models, version, metrics):
module.register(router, server)


class _HealthServer:
"""Records how the routes call into the proxy fetch helper."""

Expand Down
4 changes: 2 additions & 2 deletions tests/unit/test_aggregated_completion.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@

import pytest

from xpyd.completion_handler import handle_completion
from xpyd.proxy import Proxy
from xpyd.registry import InstanceRegistry
from xpyd.routes.completions import handle_completion
from xpyd.scheduler import SchedulingContext, default_registry

MODEL = "qwen-2"
Expand Down Expand Up @@ -174,7 +174,7 @@ async def forward(*args, **kwargs):
yield b"" # pragma: no cover

proxy.forward_request = forward
with patch("xpyd.routes.completions.track_request_end") as ended:
with patch("xpyd.completion_handler.track_request_end") as ended:
with pytest.raises(asyncio.CancelledError):
await handle_completion("/v1/completions", request, proxy, False)
ended.assert_called_once()
Expand Down
Loading
Loading