From 98f3ccd646192655031f643f5b75db331d9f4d9c Mon Sep 17 00:00:00 2001 From: Tony Lin Date: Thu, 29 Oct 2026 05:00:37 +0000 Subject: [PATCH 1/3] refactor: decouple scheduling policies from registry and backend I/O Signed-off-by: Tony Lin --- docs/architecture.md | 6 ++ docs/scheduling.md | 35 +++++++- tests/unit/routes/test_instance_removal.py | 16 ++-- tests/unit/scheduler/test_cache_aware.py | 2 +- tests/unit/scheduler/test_consistent_hash.py | 2 +- .../unit/scheduler/test_load_balanced_deep.py | 41 ++++++++- tests/unit/scheduler/test_policy_registry.py | 21 ++--- tests/unit/scheduler/test_scheduler.py | 32 +++---- tests/unit/scheduler/test_uniform_contract.py | 84 +++++++++++++++---- tests/unit/test_aggregated_completion.py | 2 +- tests/unit/test_app.py | 22 ++++- tests/unit/test_multi_model.py | 44 ++++------ tests/unit/test_tokenizer_loading.py | 2 +- xpyd/proxy.py | 18 ++-- xpyd/scheduler/cache_aware.py | 6 +- xpyd/scheduler/load_balanced.py | 33 ++++++-- xpyd/scheduler/policy_registry.py | 10 ++- xpyd/scheduler/power_of_two.py | 4 +- xpyd/scheduler/round_robin.py | 8 +- xpyd/scheduler/scheduler_base.py | 27 ++---- 20 files changed, 267 insertions(+), 148 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index dcf1f52..550a3c6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -137,6 +137,12 @@ reservations. Policy-specific token/KV accounting stays inside each subclass. Aggregated and P/D request paths share this boundary; see [Scheduling Policies](scheduling.md#common-policy-contract). +Policies do not hold or import the instance registry. Capacity-aware policies +declare their data requirement; the proxy composition layer queries node model +lengths and injects a mapping into their factory. Strategy constructors perform +no backend I/O. Runtime filtering remains the only scheduling-layer consumer +of registry health and active-request state. + ## Request Flow ``` diff --git a/docs/scheduling.md b/docs/scheduling.md index 8146822..3fbd14d 100644 --- a/docs/scheduling.md +++ b/docs/scheduling.md @@ -60,6 +60,12 @@ minimizes active requests, breaking busy-node ties using in-flight prompt tokens P/D candidates must also satisfy the model-length limit. Reservations release these counts when a request completes, fails, or is cancelled. +The policy does not query backends. At startup the proxy composition layer +queries P/D node capacities and passes an address-to-`max_model_len` mapping +into the policy. Unreachable nodes retain the existing logged fallback of +131072, so proxy-first startup remains supported. Dynamically added nodes +receive their validated capacity through `on_instance_added()`. + ### Characteristics - **Load-aware** — naturally adapts to heterogeneous instance performance. @@ -215,6 +221,32 @@ policy and context, so changing a model's tokenizer fallback cannot release load against the wrong policy. Releasing a reservation does not record success or failure; the request executor handles outcomes separately. +Only the runtime holds an `InstanceRegistry`; policies and their base class +neither accept nor store one. `from_config()` receives topology data, +`model_lengths`, tokenizer input and strategy options, not live registry state. +Policies declare `requires_model_lengths = True` when they need startup +capacities. The composition layer inspects that declaration via +`PolicyRegistry.get_policy_class()`, queries capacities only for those policies, +then uses the common factory. Aliases and custom subclasses follow the same +path without a branch on a concrete policy name or type. + +For direct Python construction, capacity-aware P/D policies require supplied +capacities; missing entries are errors rather than implicit network queries: + +```python +from xpyd.scheduler import LoadBalancedScheduler + +policy = LoadBalancedScheduler( + ["prefill:8100"], + ["decode:8200"], + model_lengths={"prefill:8100": 8192, "decode:8200": 16384}, +) +``` + +The mapping is read by address, not by iteration order. Node membership lists +retain their shared lifecycle semantics; capacity values are copied into policy +state. Aggregated-only load balancing needs no P/D capacity data. + The proxy uses this same contract for aggregated and P/D requests. It does not branch on concrete strategy types. Request code calls `Proxy.reserve(context)` and retains the returned reservation until it can call `release()`. @@ -232,7 +264,8 @@ power-of-two counters. Implement `SchedulingPolicy.select_node()` and any required lifecycle hooks. Override `from_config()` if the constructor needs special arguments; otherwise -the default factory accepts `workers`, `registry`, and the configured options. +the default factory passes `workers` and the configured options to the constructor. +Override it to consume additional common inputs such as `model_lengths`. Register the class before loading YAML: ```python diff --git a/tests/unit/routes/test_instance_removal.py b/tests/unit/routes/test_instance_removal.py index 1428234..62282ad 100644 --- a/tests/unit/routes/test_instance_removal.py +++ b/tests/unit/routes/test_instance_removal.py @@ -24,7 +24,7 @@ def _proxy(registry: InstanceRegistry) -> Proxy: decode_instances=["127.0.0.1:8200", "127.0.0.1:8201"], model="test/model", registry=registry, - scheduling_policy=RoundRobinSchedulingPolicy(registry=registry), + scheduling_policy=RoundRobinSchedulingPolicy(), ) proxy.health_monitor = MagicMock() proxy.discovery = MagicMock() @@ -170,15 +170,11 @@ async def test_load_balanced_scheduler_can_select_runtime_instance() -> None: registry.add("decode", "127.0.0.1:8200", model="test/model") registry.mark_healthy("127.0.0.1:8100") registry.mark_healthy("127.0.0.1:8200") - with patch( - "xpyd.scheduler.load_balanced.query_instance_model_len", - side_effect=lambda instances: [4096] * len(instances), - ): - policy = LoadBalancedScheduler( - ["127.0.0.1:8100"], - ["127.0.0.1:8200"], - registry=registry, - ) + policy = LoadBalancedScheduler( + ["127.0.0.1:8100"], + ["127.0.0.1:8200"], + model_lengths={"127.0.0.1:8100": 4096, "127.0.0.1:8200": 4096}, + ) proxy = _proxy(registry) proxy.scheduling_policy = policy proxy.prefill_instances = policy.prefill_instances diff --git a/tests/unit/scheduler/test_cache_aware.py b/tests/unit/scheduler/test_cache_aware.py index 6090e5d..cd87a6f 100644 --- a/tests/unit/scheduler/test_cache_aware.py +++ b/tests/unit/scheduler/test_cache_aware.py @@ -260,7 +260,7 @@ def test_schedule_does_not_fall_back_to_a_draining_worker(self): registry.add("decode", "w1") registry.mark_healthy("w1") registry.begin_draining("decode", "w1") - policy = CacheAwarePolicy(workers=["w1"], registry=registry) + policy = CacheAwarePolicy(workers=["w1"]) selected = Scheduler(registry).reserve( policy, diff --git a/tests/unit/scheduler/test_consistent_hash.py b/tests/unit/scheduler/test_consistent_hash.py index b470721..4e234ff 100644 --- a/tests/unit/scheduler/test_consistent_hash.py +++ b/tests/unit/scheduler/test_consistent_hash.py @@ -154,7 +154,7 @@ def test_schedule_does_not_fall_back_to_a_draining_worker(self): registry.add("decode", "w1") registry.mark_healthy("w1") registry.begin_draining("decode", "w1") - policy = ConsistentHashPolicy(workers=["w1"], registry=registry) + policy = ConsistentHashPolicy(workers=["w1"]) selected = Scheduler(registry).reserve( policy, diff --git a/tests/unit/scheduler/test_load_balanced_deep.py b/tests/unit/scheduler/test_load_balanced_deep.py index 9dc782a..bf3ff42 100644 --- a/tests/unit/scheduler/test_load_balanced_deep.py +++ b/tests/unit/scheduler/test_load_balanced_deep.py @@ -11,11 +11,44 @@ @pytest.fixture def policy(): + return LoadBalancedScheduler( + ["p1", "p2"], + ["d1", "d2"], + model_lengths={address: 4096 for address in ("p1", "p2", "d1", "d2")}, + ) + + +@pytest.mark.parametrize("role", ["prefill", "decode"]) +def test_injected_capacities_are_matched_by_address_without_backend_io(role): + capacities = {"large": 8192, "small": 1024} + nodes = ["small", "large"] with patch( - "xpyd.scheduler.load_balanced.query_instance_model_len", - side_effect=lambda nodes: [4096] * len(nodes), - ): - return LoadBalancedScheduler(["p1", "p2"], ["d1", "d2"]) + "requests.get", side_effect=AssertionError("Policy performed HTTP I/O") + ) as query: + policy = LoadBalancedScheduler( + nodes if role == "prefill" else [], + nodes if role == "decode" else [], + model_lengths=capacities, + ) + capacities["small"] = 8192 + lease = Scheduler().reserve( + policy, + SchedulingContext(role=role, request_len=1024, max_tokens=1), + nodes, + ) + query.assert_not_called() + assert lease.address == "large" + lease.release() + + +@pytest.mark.parametrize("role", ["prefill", "decode"]) +def test_missing_capacity_is_rejected(role): + with pytest.raises(ValueError, match="Missing model lengths.*missing"): + LoadBalancedScheduler( + ["missing"] if role == "prefill" else [], + ["missing"] if role == "decode" else [], + model_lengths={}, + ) @pytest.mark.parametrize("role", ["prefill", "decode"]) diff --git a/tests/unit/scheduler/test_policy_registry.py b/tests/unit/scheduler/test_policy_registry.py index cdd144e..e0c13bf 100644 --- a/tests/unit/scheduler/test_policy_registry.py +++ b/tests/unit/scheduler/test_policy_registry.py @@ -10,8 +10,8 @@ class _DummyPolicy(SchedulingPolicy): """Minimal concrete policy for testing.""" - def __init__(self, workers=None, registry=None, **kwargs): - super().__init__(registry=registry, workers=workers) + def __init__(self, workers=None, **kwargs): + super().__init__(workers=workers) self.kwargs = kwargs def select_node(self, context, candidates): @@ -78,18 +78,13 @@ def test_duplicate_register_warns(self, caplog): def test_build_returns_correct_type(self): """build() returns an instance of the registered class.""" - from unittest.mock import patch - rr = default_registry.build("roundrobin") assert isinstance(rr, SchedulingPolicy) - with patch( - "xpyd.scheduler.load_balanced.query_instance_model_len", - return_value=[131072], - ): - lb = default_registry.build( - "loadbalanced", - prefill_instances=["p1"], - decode_instances=["d1"], - ) + lb = default_registry.build( + "loadbalanced", + prefill_instances=["p1"], + decode_instances=["d1"], + model_lengths={"p1": 131072, "d1": 131072}, + ) assert isinstance(lb, SchedulingPolicy) diff --git a/tests/unit/scheduler/test_scheduler.py b/tests/unit/scheduler/test_scheduler.py index 2272b7e..0084d65 100644 --- a/tests/unit/scheduler/test_scheduler.py +++ b/tests/unit/scheduler/test_scheduler.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 """Policy selection and runtime eligibility are separate contracts.""" -from unittest.mock import patch - import pytest from xpyd.registry import InstanceRegistry @@ -76,15 +74,12 @@ def test_runtime_filters_unhealthy(strategy, role, healthy): registry.add(role, node) for node in healthy: registry.mark_healthy(node) - with patch( - "xpyd.scheduler.load_balanced.query_instance_model_len", - side_effect=lambda instances: [4096] * len(instances), - ): - policy = default_registry.build( - strategy, - prefill_instances=nodes if role == "prefill" else [], - decode_instances=nodes if role == "decode" else [], - ) + policy = default_registry.build( + strategy, + prefill_instances=nodes if role == "prefill" else [], + decode_instances=nodes if role == "decode" else [], + model_lengths={address: 4096 for address in nodes}, + ) runtime = Scheduler(registry) context = SchedulingContext(role=role, request_len=100, max_tokens=10) leases = [runtime.reserve(policy, context, nodes) for _ in range(6)] @@ -101,15 +96,12 @@ def test_runtime_filters_unhealthy(strategy, role, healthy): @pytest.mark.parametrize("role", ["prefill", "decode"]) def test_runtime_without_registry_uses_explicit_candidates(strategy, role): nodes = ["a", "b", "c"] - with patch( - "xpyd.scheduler.load_balanced.query_instance_model_len", - side_effect=lambda instances: [4096] * len(instances), - ): - policy = default_registry.build( - strategy, - prefill_instances=nodes if role == "prefill" else [], - decode_instances=nodes if role == "decode" else [], - ) + policy = default_registry.build( + strategy, + prefill_instances=nodes if role == "prefill" else [], + decode_instances=nodes if role == "decode" else [], + model_lengths={address: 4096 for address in nodes}, + ) runtime = Scheduler() context = SchedulingContext(role=role, request_len=100, max_tokens=1) leases = [runtime.reserve(policy, context, nodes) for _ in nodes] diff --git a/tests/unit/scheduler/test_uniform_contract.py b/tests/unit/scheduler/test_uniform_contract.py index c46b29e..272ec64 100644 --- a/tests/unit/scheduler/test_uniform_contract.py +++ b/tests/unit/scheduler/test_uniform_contract.py @@ -26,13 +26,67 @@ ROLES = ["aggregated", "prefill", "decode"] -@pytest.fixture -def model_lengths(): +@pytest.mark.parametrize("strategy", STRATEGIES) +def test_policies_do_not_accept_instance_registry(strategy): + with pytest.raises(TypeError, match="registry"): + default_registry.build(strategy, registry=InstanceRegistry()) + + +@pytest.mark.parametrize( + "strategy", ["loadbalanced", "load_balanced", "custom_capacity"] +) +def test_composition_queries_and_injects_capacities(strategy, monkeypatch): + from xpyd.proxy import _create_scheduling_policy + from xpyd.scheduler import LoadBalancedScheduler + + class CustomCapacityPolicy(LoadBalancedScheduler): + pass + + monkeypatch.setitem( + default_registry._policies, "custom_capacity", CustomCapacityPolicy + ) + config = ProxyConfig( + model="demo", + prefill=["127.0.0.1:18101"], + decode=["127.0.0.1:18102"], + scheduling=strategy, + ) with patch( - "xpyd.scheduler.load_balanced.query_instance_model_len", - side_effect=lambda nodes: [4096] * len(nodes), - ): - yield + "xpyd.proxy.query_instance_model_len", return_value=[1024, 8192] + ) as query: + policy = _create_scheduling_policy(config) + query.assert_called_once_with(config.prefill + config.decode) + assert policy.prefill_model_len == [1024] + assert policy.decode_model_len == [8192] + + +@pytest.mark.parametrize( + "strategy,has_pd_nodes", + [ + ("roundrobin", True), + ("consistent_hash", True), + ("power_of_two", True), + ("cache_aware", True), + ("loadbalanced", False), + ], +) +def test_composition_skips_unneeded_capacity_queries(strategy, has_pd_nodes): + from xpyd.proxy import _create_scheduling_policy + + config = ProxyConfig( + model="demo", + prefill=["127.0.0.1:18101"] if has_pd_nodes else [], + decode=["127.0.0.1:18102"] if has_pd_nodes else [], + instances=( + None + if has_pd_nodes + else [{"role": "aggregated", "address": "127.0.0.1:18103"}] + ), + scheduling=strategy, + ) + with patch("xpyd.proxy.query_instance_model_len") as query: + _create_scheduling_policy(config) + query.assert_not_called() def make_proxy(strategy, role): @@ -48,7 +102,7 @@ def make_proxy(strategy, role): prefill_instances=prefill, decode_instances=decode, workers=nodes, - registry=registry, + model_lengths={address: 4096 for address in nodes}, ) return ( Proxy( @@ -65,7 +119,7 @@ def make_proxy(strategy, role): @pytest.mark.parametrize("strategy", STRATEGIES) @pytest.mark.parametrize("role", ROLES) -def test_filtering_and_idempotent_release(model_lengths, strategy, role): +def test_filtering_and_idempotent_release(strategy, role): proxy, nodes = make_proxy(strategy, role) context = SchedulingContext( role=role, model="demo", request_len=5, max_tokens=1, prompt="prefix" @@ -84,7 +138,7 @@ def test_filtering_and_idempotent_release(model_lengths, strategy, role): @pytest.mark.parametrize("strategy", STRATEGIES) @pytest.mark.parametrize("role", ROLES) @pytest.mark.asyncio -async def test_membership_changes_through_proxy(model_lengths, strategy, role): +async def test_membership_changes_through_proxy(strategy, role): proxy, nodes = make_proxy(strategy, role) context = SchedulingContext(role=role, model="demo", request_len=5, prompt="prefix") # Create any lazily configured per-model policy before changing membership. @@ -104,7 +158,7 @@ async def test_membership_changes_through_proxy(model_lengths, strategy, role): @pytest.mark.parametrize("strategy", ["loadbalanced", "power_of_two"]) @pytest.mark.parametrize("role", ROLES) -def test_concurrent_reservations_preserve_load(model_lengths, strategy, role): +def test_concurrent_reservations_preserve_load(strategy, role): proxy, nodes = make_proxy(strategy, role) # Initialize per-model policy before calling from multiple worker threads. proxy.reserve(SchedulingContext(role=role, model="demo", request_len=5)).release() @@ -116,7 +170,7 @@ def test_concurrent_reservations_preserve_load(model_lengths, strategy, role): assert [proxy.registry.get_active_requests(node) for node in nodes] == [0, 0] -def test_release_uses_original_policy_after_fallback_changes(model_lengths): +def test_release_uses_original_policy_after_fallback_changes(): proxy, nodes = make_proxy("loadbalanced", "prefill") lease = proxy.reserve( SchedulingContext(role="prefill", model="demo", request_len=30) @@ -141,8 +195,8 @@ def select_node(self, context, candidates): @pytest.mark.parametrize("role", ROLES) def test_custom_policy_from_yaml_needs_no_proxy_branch(tmp_path, monkeypatch, role): class LastPolicy(SchedulingPolicy): - def __init__(self, workers=(), registry=None, offset=0): - super().__init__(registry) + def __init__(self, workers=(), offset=0): + super().__init__() self.offset = offset self.finished = [] @@ -202,7 +256,7 @@ def test_configuration_selects_policy_through_common_factory(): @pytest.mark.parametrize("strategy", STRATEGIES) -def test_only_unified_policy_entry_points_are_exposed(model_lengths, strategy): +def test_only_unified_policy_entry_points_are_exposed(strategy): policy = default_registry.build(strategy) for name in ( "schedule", @@ -267,7 +321,7 @@ def test_unknown_policy_option_is_not_silently_ignored(strategy): @pytest.mark.parametrize("strategy", STRATEGIES) @pytest.mark.asyncio -async def test_failed_membership_update_rolls_back(model_lengths, strategy): +async def test_failed_membership_update_rolls_back(strategy): from unittest.mock import Mock proxy, nodes = make_proxy(strategy, "decode") diff --git a/tests/unit/test_aggregated_completion.py b/tests/unit/test_aggregated_completion.py index ba43993..d64cedd 100644 --- a/tests/unit/test_aggregated_completion.py +++ b/tests/unit/test_aggregated_completion.py @@ -25,7 +25,7 @@ def make_proxy(strategy="roundrobin", override=None, registry=True): [], [], MODEL, - default_registry.build(strategy, registry=reg), + default_registry.build(strategy), registry=reg, aggregated_instances={MODEL: list(NODES)}, model_schedulers={MODEL: override} if override else {}, diff --git a/tests/unit/test_app.py b/tests/unit/test_app.py index d81bb9b..818817f 100644 --- a/tests/unit/test_app.py +++ b/tests/unit/test_app.py @@ -53,7 +53,8 @@ def test_support_modules_do_not_import_proxy_runtime(): @pytest.mark.parametrize("topology", ["disaggregated", "aggregated", "mixed"]) -def test_proxy_first_discovery_loss_and_reconnection(topology): +@pytest.mark.parametrize("strategy", ["roundrobin", "loadbalanced"]) +def test_proxy_first_discovery_loss_and_reconnection(topology, strategy): roles = [] if topology != "aggregated": roles.extend(["prefill", "decode", "decode"]) @@ -73,13 +74,28 @@ def test_proxy_first_discovery_loss_and_reconnection(topology): } for role, address in zip(roles, addresses) ], - scheduling="roundrobin", + scheduling=strategy, probe_interval_seconds=3600, heartbeat_interval_seconds=0.001, health_check={"enabled": True, "interval_seconds": 3600}, ) - server = ProxyServer(config) + with patch( + "requests.get", side_effect=ConnectionError("Backends are offline") + ) as capacity_query: + server = ProxyServer(config) proxy = server.proxy_instance + if strategy == "loadbalanced": + assert capacity_query.call_count == roles.count("prefill") + roles.count( + "decode" + ) + assert proxy.scheduling_policy.prefill_model_len == [131072] * roles.count( + "prefill" + ) + assert proxy.scheduling_policy.decode_model_len == [131072] * roles.count( + "decode" + ) + else: + capacity_query.assert_not_called() proxy.ensure_tokenizer = Mock(return_value=True) forwarded = [] diff --git a/tests/unit/test_multi_model.py b/tests/unit/test_multi_model.py index 10c17b2..a53aea2 100644 --- a/tests/unit/test_multi_model.py +++ b/tests/unit/test_multi_model.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 """Unit tests for multi-model routing support.""" -from unittest.mock import patch - from xpyd.registry import InstanceRegistry from xpyd.scheduler import Scheduler, SchedulingContext @@ -119,15 +117,11 @@ def test_load_balanced_routes_by_model(self): all_prefill = ["10.0.0.1:8000", "10.0.0.2:8000"] all_decode = ["10.0.0.1:9000", "10.0.0.2:9000"] - with patch( - "xpyd.scheduler.load_balanced.query_instance_model_len", - return_value=[131072] * 2, - ): - sched = LoadBalancedScheduler( - all_prefill, - all_decode, - registry=reg, - ) + sched = LoadBalancedScheduler( + all_prefill, + all_decode, + model_lengths={address: 131072 for address in all_prefill + all_decode}, + ) runtime = Scheduler(reg) result = runtime.reserve( @@ -163,15 +157,11 @@ def test_returns_none_for_unknown_model(self): reg.mark_healthy("10.0.0.1:8000") reg.mark_healthy("10.0.0.1:9000") - with patch( - "xpyd.scheduler.load_balanced.query_instance_model_len", - return_value=[131072], - ): - sched = LoadBalancedScheduler( - ["10.0.0.1:8000"], - ["10.0.0.1:9000"], - registry=reg, - ) + sched = LoadBalancedScheduler( + ["10.0.0.1:8000"], + ["10.0.0.1:9000"], + model_lengths={"10.0.0.1:8000": 131072, "10.0.0.1:9000": 131072}, + ) result = Scheduler(reg).reserve( sched, @@ -202,15 +192,11 @@ def test_routes_only_to_correct_model_instances(self): all_decode = ["10.0.0.1:9001", "10.0.0.2:9002", "10.0.0.3:9003"] - with patch( - "xpyd.scheduler.load_balanced.query_instance_model_len", - return_value=[131072] * 3, - ): - sched = LoadBalancedScheduler( - [], - all_decode, - registry=reg, - ) + sched = LoadBalancedScheduler( + [], + all_decode, + model_lengths={address: 131072 for address in all_decode}, + ) # All requests with model="llama-3" must only go to llama-3 instances runtime = Scheduler(reg) diff --git a/tests/unit/test_tokenizer_loading.py b/tests/unit/test_tokenizer_loading.py index f4a2ed0..daeb6bd 100644 --- a/tests/unit/test_tokenizer_loading.py +++ b/tests/unit/test_tokenizer_loading.py @@ -108,7 +108,7 @@ def test_pd_model_with_failed_tokenizer_uses_round_robin(): proxy.decode_instances = ["10.0.0.2:8000"] proxy._scheduler = Scheduler(registry) proxy._round_robin_models.add("org/model") - proxy._round_robin_policy = RoundRobinSchedulingPolicy(registry=registry) + proxy._round_robin_policy = RoundRobinSchedulingPolicy() proxy.scheduling_policy = MagicMock() lease = proxy.reserve( diff --git a/xpyd/proxy.py b/xpyd/proxy.py index b75ea6b..2183895 100644 --- a/xpyd/proxy.py +++ b/xpyd/proxy.py @@ -49,6 +49,7 @@ SchedulingPolicy, default_registry, ) +from xpyd.utils import query_instance_model_len from xpyd.zmq_notifications import ZmqNotificationListener @@ -121,9 +122,7 @@ def __init__( self.tokenizer_path = tokenizer_path self._tokenizers: dict[str, Any] = {} self._round_robin_models: set[str] = set() - self._round_robin_policy = default_registry.build( - "roundrobin", registry=registry - ) + self._round_robin_policy = default_registry.build("roundrobin") self.disaggregated_mode = disaggregated_mode self.first_token_source = first_token_source self.zmq_config = zmq_config @@ -242,7 +241,6 @@ def _get_aggregated_policy( strategy = self.model_schedulers.get(model) options: dict[str, Any] = { "workers": instances, - "registry": self.registry, "tokenizer": self.get_tokenizer(model), } if strategy: @@ -783,7 +781,6 @@ async def validate_instance(self, instance: str) -> bool: def _create_scheduling_policy( config: ProxyConfig, - registry: Optional[InstanceRegistry] = None, all_prefill: Optional[list[str]] = None, all_decode: Optional[list[str]] = None, ) -> SchedulingPolicy: @@ -794,12 +791,20 @@ def _create_scheduling_policy( strategy = config.scheduling strategy_opts = config.scheduling_config.get(strategy, {}) + policy_class = default_registry.get_policy_class(strategy) + model_lengths: dict[str, int] = {} + if policy_class.requires_model_lengths and (prefill or decode): + addresses = list(prefill) + list(decode) + model_lengths = dict( + zip(addresses, query_instance_model_len(addresses), strict=True) + ) + return default_registry.build( strategy, prefill_instances=prefill, decode_instances=decode, workers=list(prefill) + list(decode), - registry=registry, + model_lengths=model_lengths, **strategy_opts, ) @@ -940,7 +945,6 @@ def __init__( global_policy = _create_scheduling_policy( config, - self.registry, all_prefill=all_prefill, all_decode=all_decode, ) diff --git a/xpyd/scheduler/cache_aware.py b/xpyd/scheduler/cache_aware.py index 10b3b0b..5491213 100644 --- a/xpyd/scheduler/cache_aware.py +++ b/xpyd/scheduler/cache_aware.py @@ -179,13 +179,11 @@ def from_config( prefill_instances=(), decode_instances=(), workers=(), - registry=None, + model_lengths=None, tokenizer=None, **options, ): - return cls( - workers=list(workers), registry=registry, tokenizer=tokenizer, **options - ) + return cls(workers=list(workers), tokenizer=tokenizer, **options) def select_node( self, context: SchedulingContext, candidates: Sequence[Candidate] diff --git a/xpyd/scheduler/load_balanced.py b/xpyd/scheduler/load_balanced.py index 3449456..1d43e42 100644 --- a/xpyd/scheduler/load_balanced.py +++ b/xpyd/scheduler/load_balanced.py @@ -2,10 +2,9 @@ """Load-balanced scheduling policy.""" import logging -from typing import Optional, Sequence +from typing import Mapping, Optional, Sequence from xpyd.scheduler.scheduler_base import Candidate, SchedulingContext, SchedulingPolicy -from xpyd.utils import query_instance_model_len logger = logging.getLogger("xpyd.proxy") @@ -13,9 +12,20 @@ class LoadBalancedScheduler(SchedulingPolicy): """Select the least-loaded instance, respecting model-length limits.""" + requires_model_lengths = True + def __init__( - self, prefill_instances: list[str], decode_instances: list[str], registry=None + self, + prefill_instances: Sequence[str], + decode_instances: Sequence[str], + *, + model_lengths: Mapping[str, int], ): + missing = ( + set(prefill_instances) | set(decode_instances) + ) - model_lengths.keys() + if missing: + raise ValueError(f"Missing model lengths for instances: {sorted(missing)}") self.prefill_utils_counter = [0] * len(prefill_instances) self.prefill_bs_counter = [0] * len(prefill_instances) self.decode_kv_utils_counter = [0] * len(decode_instances) @@ -42,12 +52,14 @@ def __init__( self.decode_schedule_index = 0 self.decode_schedule_completion_index = 0 - self.prefill_model_len = query_instance_model_len(prefill_instances) - self.decode_model_len = query_instance_model_len(decode_instances) + self.prefill_model_len = [ + model_lengths[address] for address in prefill_instances + ] + self.decode_model_len = [model_lengths[address] for address in decode_instances] logger.info("Prefill instance model lens: %s", self.prefill_model_len) logger.info("Decode instance model lens: %s", self.decode_model_len) - super().__init__(registry=registry) + super().__init__() @classmethod def from_config( @@ -56,11 +68,16 @@ def from_config( prefill_instances=(), decode_instances=(), workers=(), - registry=None, + model_lengths=None, tokenizer=None, **options, ): - return cls(prefill_instances, decode_instances, registry=registry, **options) + return cls( + prefill_instances, + decode_instances, + model_lengths=model_lengths if model_lengths is not None else {}, + **options, + ) def select_node( self, context: SchedulingContext, candidates: Sequence[Candidate] diff --git a/xpyd/scheduler/policy_registry.py b/xpyd/scheduler/policy_registry.py index 994db51..92d1be5 100644 --- a/xpyd/scheduler/policy_registry.py +++ b/xpyd/scheduler/policy_registry.py @@ -50,14 +50,18 @@ def has(self, name: str) -> bool: """Return whether *name* is a registered policy.""" return name in self._policies - def build(self, name: str, **kwargs: Any) -> SchedulingPolicy: - """Construct through the topology-independent policy factory.""" + def get_policy_class(self, name: str) -> type[SchedulingPolicy]: + """Resolve a policy and its declared data requirements without constructing it.""" if name not in self._policies: raise ValueError( f"Unknown scheduling policy: {name!r}. " f"Available: {self.list_policies()}" ) - return self._policies[name].from_config(**kwargs) + return self._policies[name] + + def build(self, name: str, **kwargs: Any) -> SchedulingPolicy: + """Construct through the topology-independent policy factory.""" + return self.get_policy_class(name).from_config(**kwargs) def list_policies(self) -> list[str]: """Return a sorted list of registered policy names.""" diff --git a/xpyd/scheduler/power_of_two.py b/xpyd/scheduler/power_of_two.py index 9b61a6a..a22e1ce 100644 --- a/xpyd/scheduler/power_of_two.py +++ b/xpyd/scheduler/power_of_two.py @@ -10,8 +10,8 @@ class PowerOfTwoPolicy(SchedulingPolicy): """Choose the less-loaded of two eligible candidates.""" - def __init__(self, workers=None, registry=None): - super().__init__(registry=registry) + def __init__(self, workers=None): + super().__init__() self._last_pair: tuple[str, ...] = () @property diff --git a/xpyd/scheduler/round_robin.py b/xpyd/scheduler/round_robin.py index 2d5f407..aac8192 100644 --- a/xpyd/scheduler/round_robin.py +++ b/xpyd/scheduler/round_robin.py @@ -12,8 +12,8 @@ class RoundRobinSchedulingPolicy(SchedulingPolicy): """Cycle through instances in order, ignoring load.""" - def __init__(self, registry=None): - super().__init__(registry=registry) + def __init__(self): + super().__init__() self._positions: dict[tuple[str, str], int] = {} logger.info("RoundRobinSchedulingPolicy initialized") @@ -24,11 +24,11 @@ def from_config( prefill_instances=(), decode_instances=(), workers=(), - registry=None, + model_lengths=None, tokenizer=None, **options, ): - return cls(registry=registry, **options) + return cls(**options) def select_node( self, context: SchedulingContext, candidates: Sequence[Candidate] diff --git a/xpyd/scheduler/scheduler_base.py b/xpyd/scheduler/scheduler_base.py index 2816a64..8136e08 100644 --- a/xpyd/scheduler/scheduler_base.py +++ b/xpyd/scheduler/scheduler_base.py @@ -4,11 +4,7 @@ import threading from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Literal, Optional, Sequence - -if TYPE_CHECKING: - from xpyd.registry import InstanceRegistry - +from typing import Any, ClassVar, Literal, Mapping, Optional, Sequence InstanceRole = Literal["prefill", "decode", "aggregated"] @@ -35,21 +31,10 @@ class Candidate: class SchedulingPolicy(ABC): """Base class for all scheduling policies.""" - def __init__( - self, - registry: Optional["InstanceRegistry"] = None, - workers: Optional[Sequence[str]] = None, - ): - self.lock = threading.Lock() - self._registry: Optional["InstanceRegistry"] = registry + requires_model_lengths: ClassVar[bool] = False - @property - def registry(self) -> Optional["InstanceRegistry"]: - return self._registry - - @registry.setter - def registry(self, value: Optional["InstanceRegistry"]) -> None: - self._registry = value + def __init__(self, workers: Optional[Sequence[str]] = None): + self.lock = threading.Lock() @abstractmethod def select_node( @@ -65,12 +50,12 @@ def from_config( prefill_instances: Sequence[str] = (), decode_instances: Sequence[str] = (), workers: Sequence[str] = (), - registry: Optional["InstanceRegistry"] = None, + model_lengths: Mapping[str, int] | None = None, tokenizer: Any = None, **options: Any, ) -> "SchedulingPolicy": """Construct a policy without requiring callers to know its constructor.""" - return cls(workers=list(workers), registry=registry, **options) + return cls(workers=list(workers), **options) def on_instance_added( # noqa: B027 - optional lifecycle hook self, role: InstanceRole, address: str, max_model_len: int From bcb64240138e7b945d2488c97c310c0bb41685d2 Mon Sep 17 00:00:00 2001 From: Tony Lin Date: Thu, 29 Oct 2026 05:09:57 +0000 Subject: [PATCH 2/3] style: shorten scheduler registry docstring Signed-off-by: Tony Lin --- xpyd/scheduler/policy_registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xpyd/scheduler/policy_registry.py b/xpyd/scheduler/policy_registry.py index 92d1be5..3b40e18 100644 --- a/xpyd/scheduler/policy_registry.py +++ b/xpyd/scheduler/policy_registry.py @@ -51,7 +51,7 @@ def has(self, name: str) -> bool: return name in self._policies def get_policy_class(self, name: str) -> type[SchedulingPolicy]: - """Resolve a policy and its declared data requirements without constructing it.""" + """Resolve a policy class and its requirements without constructing it.""" if name not in self._policies: raise ValueError( f"Unknown scheduling policy: {name!r}. " From 3e9c1a46d28e749a2f99445a274688b29d59da50 Mon Sep 17 00:00:00 2001 From: Tony Lin Date: Thu, 29 Oct 2026 05:41:18 +0000 Subject: [PATCH 3/3] ci: retrigger checks for scheduler refactor Signed-off-by: Tony Lin