From d6237d1504243cd58788627f4450417f31f35fe2 Mon Sep 17 00:00:00 2001 From: youngrok-XCENA Date: Wed, 21 Oct 2026 12:08:12 +0900 Subject: [PATCH 1/8] feat(remote_cxl): access policies and region extensions for tiered memory Some memory answers every address but not at one speed: a device keeps a small fast cache in front of slower media, so data is brought in before it is read and written back after it is written. Maru stays device-neutral and lets a separately installed package control only when its transfers run: - An access policy (entry-point group maru.access_policies), selected by name in MaruConfig.access_policy, runs the READs of a load in order by calling the transfer Maru hands it, prepares and finishes WRITEs, and says when a page given back may be reused. - A region extension (maru.region_extensions), selected with the remote CXL agent's --region-extension, sees the regions the agent maps, may refuse one, and may keep one out of use after the NIC lost it until its own work on it ended; the sweep and close ask it again. - Policies reach their extension over the connection that mapped a region. The agent checks every range against what that connection holds, the handle, the region bounds and the epoch, and refuses a write prepared with a read handle, before the extension sees the message. A policy that fails gives an outcome Maru already has: a miss, a key not stored, a page or region kept out of use. Without a policy nothing changes. The vLLM connector passes maru_access_policy(_config) to the worker only. --- maru_common/config.py | 10 + maru_handler/storage/access_policy.py | 184 ++++++++++++ maru_handler/storage/remote_cxl/client.py | 257 ++++++++++++++--- maru_handler/storage/remote_cxl/datapath.py | 38 +++ maru_handler/storage/remote_cxl/lifecycle.py | 15 + maru_handler/storage/remote_cxl/region.py | 38 +++ maru_remote/region_map.py | 19 ++ maru_remote_cxl_agent/cli.py | 23 ++ maru_remote_cxl_agent/extension.py | 99 +++++++ maru_remote_cxl_agent/server.py | 2 + maru_remote_cxl_agent/service.py | 170 ++++++++++-- maru_vllm/connector.py | 10 + tests/unit/remote_fakes.py | 9 +- tests/unit/test_remote_cxl_access_policy.py | 278 +++++++++++++++++++ tests/unit/test_remote_cxl_storage.py | 7 +- tests/unit/test_vllm_remote_cxl_config.py | 31 +++ 16 files changed, 1120 insertions(+), 70 deletions(-) create mode 100644 maru_handler/storage/access_policy.py create mode 100644 maru_remote_cxl_agent/extension.py create mode 100644 tests/unit/test_remote_cxl_access_policy.py diff --git a/maru_common/config.py b/maru_common/config.py index ec477e7e..a16dcb59 100644 --- a/maru_common/config.py +++ b/maru_common/config.py @@ -80,6 +80,12 @@ class MaruConfig: # Key the remote CXL agents of the pool require (a copy of the memory node's # /etc/maru/remote-cxl-agent.key, readable by the worker's user only) remote_cxl_agent_key_file: str = "~/.maru/remote-cxl-agent.key" + # Out-of-tree access policy (maru_handler.storage.access_policy): the name + # an installed package registers under ``maru.access_policies``, and an + # opaque string handed to it (typically its own configuration file). + # Empty: no policy; the backend moves bytes as soon as it can. + access_policy: str = "" + access_policy_config: str = "" @property def server_urls(self) -> list[str]: @@ -144,6 +150,10 @@ def __post_init__(self): and self.pool_size // self.chunk_size_bytes > 1_000_000 ): raise ValueError("CPU pool supports at most 1000000 pages") + if self.access_policy and self.storage_backend != "remote_cxl": + raise ValueError("access_policy is supported by the remote_cxl backend") + if self.access_policy_config and not self.access_policy: + raise ValueError("access_policy_config needs access_policy") if "," in self.server_url and self.storage_backend != "remote_cxl": raise ValueError("Several MaruServer URLs require the remote_cxl backend") if self.storage_backend == "remote_cxl": diff --git a/maru_handler/storage/access_policy.py b/maru_handler/storage/access_policy.py new file mode 100644 index 00000000..fb680405 --- /dev/null +++ b/maru_handler/storage/access_policy.py @@ -0,0 +1,184 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 XCENA Inc. +"""Access policies: out-of-tree control over when a storage backend touches memory. + +Some memory answers every address but not always at the same speed: a device +may keep a small fast cache in front of slower media, so data has to be +brought in before it is read and written back after it is written. Maru keeps +no knowledge of such devices. A separately installed package registers an +access policy under the ``maru.access_policies`` entry-point group, and a +worker selects it by name in ``MaruConfig.access_policy``. Installing a +package selects nothing; only the configured name is loaded. + +What Maru owns and what the policy owns: + +- Maru decides which keys hit, which pages hold them, which bytes move and + whether a key is published. The policy never sees keys and never moves + bytes. +- The policy decides *when* the bytes of a read move (it calls the transfer + Maru hands it, in order), whether a page may be written now, and whether a + page given back may be reused yet. +- The policy talks to its own counterpart next to the memory through + :class:`AccessChannel`. Maru carries those messages over the connection it + already authenticated, and the memory side checks that every range in a + message belongs to a region the sending connection holds before the + counterpart sees it. + +Every hook may block; the storage backend calls it without its lock held. A +hook that raises makes the operation take an outcome Maru already has: + +===================== =================================================== +hook raises outcome +===================== =================================================== +``read`` the hits load as misses (the engine recomputes) +``write_batch_limit`` the pages are not written; the keys are not stored +``prepare_write`` the pages are not written; the keys are not stored +``written`` the keys are not stored +``completed_write`` the late WRITE's pages stay out of use; retried +``reusable`` the page stays out of use; asked again later +===================== =================================================== +""" + +from __future__ import annotations + +import importlib.metadata +from collections.abc import Callable, Iterable, Sequence +from dataclasses import asdict, dataclass +from typing import Any, Protocol, runtime_checkable + +from maru_common.storage_types import StorageLocation + +#: Entry-point group scanned for access policies. +ACCESS_POLICY_GROUP = "maru.access_policies" + + +@dataclass(frozen=True) +class AccessRange: + """One page of a region that an access touches. + + Attributes: + handle: The region capability the worker maps the region with: its + owner handle, or the read handle MaruServer gave with a lease. + location: Where the page lies in the region. + """ + + handle: dict + location: StorageLocation + + def to_dict(self) -> dict: + """Wire form: the handle and the location's fields.""" + return {"handle": dict(self.handle), **asdict(self.location)} + + +class AccessChannel(Protocol): + """Messages from a policy to its counterpart next to the memory.""" + + def request( + self, + action: str, + ranges: Sequence[AccessRange] = (), + *, + write: bool = False, + **payload: Any, + ) -> dict: + """Send one message and return the counterpart's reply. + + Args: + action: Policy-defined action name; Maru does not interpret it. + ranges: Pages the message is about. The memory side checks each + before the counterpart sees it. + write: True if the message prepares a write; a read handle may + not prepare one. + **payload: Further policy-defined fields (msgpack-encodable). + + Returns: + The counterpart's reply. + + Raises: + StorageUnavailableError: if the connection that mapped the region + changed, so the counterpart may not know the region. + Exception: if the memory side refused the message. + + A message may be delivered more than once when a connection breaks; + actions must be idempotent. + """ + ... + + +@runtime_checkable +class AccessPolicy(Protocol): + """Hooks a storage backend calls around its data transfers.""" + + def read( + self, + channel: AccessChannel, + ranges: Sequence[AccessRange], + transfer: Callable[[int, int], None], + ) -> None: + """Move the bytes of ``ranges`` by calling ``transfer``. + + ``transfer(start, end)`` reads ``ranges[start:end]``. The policy must + call it in order, with each ``start`` equal to the previous ``end``, + until every range is read; it decides only when each call happens. + An exception ``transfer`` raises must propagate. + """ + ... + + def write_batch_limit(self, channel: AccessChannel) -> int: + """How many pages of one region may be written in one batch.""" + ... + + def prepare_write( + self, channel: AccessChannel, ranges: Sequence[AccessRange] + ) -> Any: + """Make ``ranges`` ready to be written; return a token for :meth:`written`.""" + ... + + def written(self, channel: AccessChannel, token: Any) -> None: + """The WRITE prepared under ``token`` completed.""" + ... + + def completed_write(self, locations: Sequence[StorageLocation]) -> None: + """A WRITE whose completion was uncertain turned out to have completed.""" + ... + + def reusable(self, location: StorageLocation) -> bool: + """Whether a page given back may hold new data now.""" + ... + + +def load_access_policy( + name: str, + config: str = "", + *, + entry_points: Iterable[Any] | None = None, +) -> AccessPolicy: + """Load the access policy registered under ``name``. + + Args: + name: Entry-point name in :data:`ACCESS_POLICY_GROUP`. + config: Opaque string passed to the policy factory (typically the + path of the policy's own configuration file). + entry_points: Entry points to search instead of the installed ones + (tests). + + Returns: + The policy the factory built. + + Raises: + ValueError: if no installed package registers ``name``, or the + object built lacks a hook. + """ + found = entry_points + if found is None: + found = importlib.metadata.entry_points(group=ACCESS_POLICY_GROUP) + matches = [ep for ep in found if ep.name == name] + if not matches: + raise ValueError( + f"no package registers access policy {name!r} " + f"(entry-point group {ACCESS_POLICY_GROUP})" + ) + policy = matches[0].load()(config) + if not isinstance(policy, AccessPolicy): + raise ValueError(f"access policy {name!r} does not implement every hook") + return policy diff --git a/maru_handler/storage/remote_cxl/client.py b/maru_handler/storage/remote_cxl/client.py index 63dfc149..f91cdc24 100644 --- a/maru_handler/storage/remote_cxl/client.py +++ b/maru_handler/storage/remote_cxl/client.py @@ -9,7 +9,7 @@ import threading import time import uuid -from collections.abc import Callable +from collections.abc import Callable, Sequence from dataclasses import asdict from typing import Any @@ -22,6 +22,7 @@ StorageUnavailableError, ) +from ..access_policy import AccessRange, load_access_policy from ..cpu import CpuReadLease from ..staging import StagingBuffer from .datapath import RdmaDataPath @@ -81,6 +82,32 @@ def wait_for_late_transfers(clients: list[RemoteCxlStorageClient]) -> None: time.sleep(LATE_TRANSFER_POLL_S) +class _RegionChannel: + """``AccessChannel`` of one mapped region: messages go to its remote CXL + agent on the connection that mapped it.""" + + def __init__(self, data: RdmaDataPath, access: RegionAccess) -> None: + self._data = data + self._access = access + + def request( + self, + action: str, + ranges: Sequence[AccessRange] = (), + *, + write: bool = False, + **payload: Any, + ) -> dict: + """Send one policy message; see ``AccessChannel.request``.""" + message = { + **payload, + "action": action, + "ranges": [r.to_dict() for r in ranges], + "write": bool(write), + } + return self._data.extension_request(self._access, message) + + class RemoteCxlStorageClient(RegionLifecycle): """Storage-session client whose regions are reached over RDMA.""" @@ -147,6 +174,13 @@ def __init__( # Orders free and store of an allocation that has no page yet self._unplaced_lock = threading.Lock() self._unplaced_warned_at = float("-inf") + # Out-of-tree access policy the config names (access_policy.py); it + # decides when transfers run and when pages given back are reused. + self._policy = ( + load_access_policy(config.access_policy, config.access_policy_config) + if getattr(config, "access_policy", "") + else None + ) # ------------------------------------------------------------------ setup @@ -786,8 +820,6 @@ def _map_for_write( def _write_group(self, group: list[RemoteCxlAllocation]) -> bool: """WRITE the pages of one region; on failure they are given back or isolated, and False is returned.""" - from maru_remote.transport import TransferTimeout, TransferUncertainError - region = group[0].region with self._lock: if region.retired: @@ -805,9 +837,44 @@ def _write_group(self, group: list[RemoteCxlAllocation]) -> bool: access = self._map_for_write(region, group) if access is None: return False + if self._policy is None: + return self._write_batch(region, access, group, None) + channel = _RegionChannel(self._data, access) + try: + limit = self._policy.write_batch_limit(channel) + if type(limit) is not int or limit <= 0: + raise ValueError(f"invalid write batch limit {limit!r}") + except Exception: + logger.warning("access policy gave no write batch limit", exc_info=True) + with self._lock: + self._give_back(group) + return False + batches = [group[i : i + limit] for i in range(0, len(group), limit)] + for i, batch in enumerate(batches): + if not self._write_batch(region, access, batch, channel): + # Nothing of the group is committed: the rest goes back too. + with self._lock: + for other in batches[:i] + batches[i + 1 :]: + self._give_back(other) + return False + return True + + def _write_batch( + self, + region: RemoteCxlRegion, + access: RegionAccess, + batch: list[RemoteCxlAllocation], + channel: _RegionChannel | None, + ) -> bool: + """WRITE one batch of a region's pages; on failure they are given back + or isolated, and False is returned. With an access policy + (``channel`` set) the policy prepares the pages first and learns of + the WRITE's completion after.""" + from maru_remote.transport import TransferTimeout, TransferUncertainError + with self._lock: if region.retired: - self._give_back(group) + self._give_back(batch) return False pairs = [ ( @@ -815,15 +882,31 @@ def _write_group(self, group: list[RemoteCxlAllocation]) -> bool: access.base_addr + h.location.offset, h.location.length, ) - for h in group + for h in batch ] + token = None + if channel is not None: + ranges = [AccessRange(region.handle, h.location) for h in batch] + try: + token = self._policy.prepare_write(channel, ranges) + except Exception: + # The pages go back; the policy is asked before they are reused. + logger.warning("access policy did not prepare a WRITE", exc_info=True) + with self._lock: + self._give_back(batch) + return False + with self._lock: + if region.retired: + # It went back while the policy prepared the pages. + self._give_back(batch) + return False try: self._transport.write( access.peer, pairs, timeout_s=self.config.rdma_timeout_s ) except (TransferTimeout, TransferUncertainError) as exc: # The NIC may still write: keep pages and slots until it ends. - self._isolate_write(region, group, exc.pending) + self._isolate_write(region, batch, exc.pending) logger.warning("RDMA WRITE not confirmed: %s", exc) return False except Exception as exc: @@ -835,12 +918,12 @@ def _write_group(self, group: list[RemoteCxlAllocation]) -> bool: # region; the release before the region goes back ends any late # WRITE. The slots are safe once this end's connection is torn # down, so the connection goes before the slots. - for h in group: + for h in batch: region.untrack(h) region.quarantine(h.location) h.state = "freed" self._data.disconnect({access.peer}) - for h in group: + for h in batch: self._give_slot(h) # A node that cannot be reached any more has the region set aside # now; its keys stop being reported as hits if it stays so. @@ -864,13 +947,23 @@ def _write_group(self, group: list[RemoteCxlAllocation]) -> bool: if pending is None: # Interrupted before NIXL had a transfer: nothing was sent. with self._lock: - self._give_back(group) + self._give_back(batch) else: # Interrupted while the WRITE may run: keep pages and slots # out of use until it ends (the caller gives back only what # is still being sent). - self._isolate_write(region, group, pending) + self._isolate_write(region, batch, pending) raise + if channel is not None: + try: + self._policy.written(channel, token) + except Exception: + logger.warning( + "access policy did not take a WRITE's end", exc_info=True + ) + with self._lock: + self._give_back(batch) + return False return True def _isolate_write( @@ -1133,6 +1226,10 @@ def _read( by_peer: dict[str, list[tuple[int, int, int]]] = {} slots_by_peer: dict[str, list[int]] = {} + # With an access policy: per region, its access, ranges, pairs and slots + groups: dict[ + str, tuple[RegionAccess, list[AccessRange], list[tuple], list[int]] + ] = {} try: for (_, location), slot in zip(hits, slots, strict=False): # An own region is mapped with its handle; another session's @@ -1159,14 +1256,18 @@ def _read( raise StorageUnavailableError( "A leased location ends past its region" ) - by_peer.setdefault(access.peer, []).append( - ( - self._staging.addr(slot), - access.base_addr + location.offset, - location.length, - ) + pair = ( + self._staging.addr(slot), + access.base_addr + location.offset, + location.length, ) + by_peer.setdefault(access.peer, []).append(pair) slots_by_peer.setdefault(access.peer, []).append(slot) + if self._policy is not None: + group = groups.setdefault(location.pool_id, (access, [], [], [])) + group[1].append(AccessRange(handle, location)) + group[2].append(pair) + group[3].append(slot) except Exception as exc: # A region that cannot be mapped (node down, or its owner # released it) loads as a miss. @@ -1174,38 +1275,63 @@ def _read( for slot in slots: self._staging.give(slot) return False - for peer, pairs in by_peer.items(): + if self._policy is None: + work = [(peer, pairs, None) for peer, pairs in by_peer.items()] + else: + work = [(g[0].peer, g[2], g) for g in groups.values()] + for peer, pairs, group in work: + failure: list[BaseException] = [] # what the transport raised try: - self._transport.read(peer, pairs, timeout_s=self.config.rdma_timeout_s) - except (TransferTimeout, TransferUncertainError) as exc: - # A late READ only lands in these slots: isolate them. - peer_slots = slots_by_peer[peer] - with self._lock: - self._isolated.append( - IsolatedTransfer( - exc.pending, "READ", exc.pending.peer, None, [], peer_slots - ) + if group is None: + self._transport.read( + peer, pairs, timeout_s=self.config.rdma_timeout_s ) - # Earlier peers completed; later peers have not been posted. - held = set(peer_slots) - for slot in slots: - if slot not in held: + else: + self._policy_read(group[0], group[1], pairs, failure) + except BaseException as raised: + if group is not None and not failure: + # The policy failed with no READ under way: a miss. + for slot in slots: self._staging.give(slot) - logger.warning("RDMA READ not confirmed: %s", exc) - return False - except Exception as exc: - # NIXL released the failed transfer: nothing lands any more. - logger.warning("RDMA READ failed: %s", exc) - for slot in slots: - self._staging.give(slot) - try: - self._data.disconnect({peer}) - except Exception: - logger.warning( - "closing the connection to %s failed", peer, exc_info=True - ) - return False - except BaseException as exc: + if not isinstance(raised, Exception): + raise + logger.warning("access policy did not complete a READ: %s", raised) + return False + exc = failure[0] if failure else raised + # Only this unit's slots can still receive a late READ. + peer_slots = group[3] if group is not None else slots_by_peer[peer] + if isinstance(exc, (TransferTimeout, TransferUncertainError)): + # A late READ only lands in these slots: isolate them. + with self._lock: + self._isolated.append( + IsolatedTransfer( + exc.pending, + "READ", + exc.pending.peer, + None, + [], + peer_slots, + ) + ) + # Earlier units completed; later ones have not been posted. + held = set(peer_slots) + for slot in slots: + if slot not in held: + self._staging.give(slot) + logger.warning("RDMA READ not confirmed: %s", exc) + return False + if isinstance(exc, Exception): + # NIXL released the failed transfer: nothing lands any more. + logger.warning("RDMA READ failed: %s", exc) + for slot in slots: + self._staging.give(slot) + try: + self._data.disconnect({peer}) + except Exception: + logger.warning( + "closing the connection to %s failed", peer, exc_info=True + ) + return False pending = getattr(exc, "pending_transfer", None) if pending is None: # Interrupted before NIXL had a transfer: nothing lands. @@ -1214,7 +1340,6 @@ def _read( else: # Interrupted while the READ may run: its slots stay taken # until NIXL releases the transfer. - peer_slots = slots_by_peer[peer] with self._lock: self._isolated.append( IsolatedTransfer( @@ -1228,6 +1353,46 @@ def _read( raise return True + def _policy_read( + self, + access: RegionAccess, + ranges: list[AccessRange], + pairs: list[tuple[int, int, int]], + failure: list[BaseException], + ) -> None: + """Let the access policy run the READs of one region. + + The policy calls the transfer in order until every range is read; an + exception the transport raised is recorded in ``failure`` and raised + again even if the policy swallowed it. + + Raises: + ValueError: if the policy read out of order or left ranges unread. + BaseException: what the policy or the transport raised. + """ + cursor = 0 + + def transfer(start: int, end: int) -> None: + nonlocal cursor + if start != cursor or not start < end <= len(pairs): + raise ValueError( + f"access policy read ranges[{start}:{end}] after {cursor}" + ) + try: + self._transport.read( + access.peer, pairs[start:end], timeout_s=self.config.rdma_timeout_s + ) + except BaseException as exc: + failure.append(exc) + raise + cursor = end + + self._policy.read(_RegionChannel(self._data, access), ranges, transfer) + if failure: + raise failure[0] + if cursor != len(pairs): + raise ValueError(f"access policy read {cursor} of {len(pairs)} ranges") + def _release_lease(self, lease_id: str) -> bool: """Release a server lease; True only if it held for the whole read. diff --git a/maru_handler/storage/remote_cxl/datapath.py b/maru_handler/storage/remote_cxl/datapath.py index 058fbb39..ddbc3127 100644 --- a/maru_handler/storage/remote_cxl/datapath.py +++ b/maru_handler/storage/remote_cxl/datapath.py @@ -187,6 +187,44 @@ def start(self) -> None: self._probe, self._probe_addr = probe, probe_addr self._started = True + def extension_request(self, access: RegionAccess, message: dict) -> dict: + """Send an access policy's message over the connection that mapped a region. + + The message goes to the remote CXL agent that mapped the region, on + the connection the mapping belongs to: the agent checks the message's + ranges against what that connection holds. + + Args: + access: The region's access, from :meth:`region`. + message: ``action``, ``ranges``, ``write`` and the policy's fields. + + Returns: + The extension's reply. + + Raises: + StorageUnavailableError: if the connection that mapped the region + was replaced, before or during the request. + StorageError: if the agent refused the message. + """ + from maru_remote.region_map import RegionMapError, RegionMapUnavailable + + client = self._client(access.agent) + if client.generation != access.generation: + raise StorageUnavailableError( + "the connection that mapped the region changed" + ) + try: + reply = client.extension({**message, "epoch": client.epoch}) + except RegionMapUnavailable as exc: + raise StorageUnavailableError(str(exc)) from exc + except RegionMapError as exc: + raise StorageError(str(exc)) from exc + if client.generation != access.generation: + raise StorageUnavailableError( + "the connection that mapped the region changed" + ) + return reply + def region( self, pool_id: str, handle: dict, agents: list[str], own: bool = False ) -> RegionAccess: diff --git a/maru_handler/storage/remote_cxl/lifecycle.py b/maru_handler/storage/remote_cxl/lifecycle.py index a642c660..f9a1ba6c 100644 --- a/maru_handler/storage/remote_cxl/lifecycle.py +++ b/maru_handler/storage/remote_cxl/lifecycle.py @@ -75,6 +75,9 @@ def _add_region(self, info: dict) -> RemoteCxlRegion: self.config.chunk_size_bytes, info["handle"], ) + policy = getattr(self, "_policy", None) + if policy is not None: + region.reuse_guard = policy.reusable self.regions[region.pool_id] = region try: self._map_own(region) @@ -562,6 +565,18 @@ def _reap_isolated(self) -> None: # Pages go back to the region they came from; a retired # region ignores them. if entry.region is not None and entry.pending.completed: + policy = getattr(self, "_policy", None) + if policy is not None: + # The policy prepared these pages for the WRITE that + # turned out complete; it finishes them first. + try: + policy.completed_write(entry.locations) + except Exception: + logger.warning( + "access policy did not take a late WRITE", exc_info=True + ) + kept.append(entry) + continue for location in entry.locations: entry.region.unquarantine(location) entry.region.give_page(location) diff --git a/maru_handler/storage/remote_cxl/region.py b/maru_handler/storage/remote_cxl/region.py index 80613313..a49464ae 100644 --- a/maru_handler/storage/remote_cxl/region.py +++ b/maru_handler/storage/remote_cxl/region.py @@ -3,14 +3,18 @@ from __future__ import annotations +import logging import threading import uuid from collections import deque +from collections.abc import Callable from dataclasses import dataclass, field from typing import Any from maru_common.storage_types import StorageLocation, StorageUnavailableError +logger = logging.getLogger(__name__) + REMOTE_CXL = "remote_cxl" @@ -82,17 +86,23 @@ def __init__(self, pool_id: str, capacity: int, page_size: int, handle: dict): self.mapped = False # Boot of the memory node at the last map self.boot = "" + # Access policy hook asked before a page given back is reused + # (``AccessPolicy.reusable``); pages it holds wait in _waiting_reuse. + self.reuse_guard: Callable[[StorageLocation], bool] | None = None + self._waiting_reuse: list[StorageLocation] = [] def free_count(self) -> int: """Free pages a store can take now (none while the region is unusable).""" with self._lock: if self.retired or not self.reachable: return 0 + self._reap_reusable() return len(self._free) def has_free(self) -> bool: """Whether a page can be taken now.""" with self._lock: + self._reap_reusable() return not self.retired and self.reachable and bool(self._free) def take_page(self, size: int) -> StorageLocation: @@ -110,6 +120,7 @@ def take_page(self, size: int) -> StorageLocation: ) if type(size) is not int or not 0 < size <= self.page_size: raise ValueError("Remote CXL allocation must fit in one page") + self._reap_reusable() if not self._free: raise MemoryError("Remote CXL region is full") page = self._free.popleft() @@ -150,8 +161,34 @@ def give_page(self, location: StorageLocation) -> None: with self._lock: if location.pool_id != self.pool_id or self.retired: return + if self.reuse_guard is not None: + self._waiting_reuse.append(location) + self._reap_reusable() + return self._free.append(location.offset // self.page_size) + def _reap_reusable(self) -> None: + """Free the pages given back that the access policy lets go now. + + A page the policy does not answer for (it raised) stays waiting. + """ + if not self._waiting_reuse: + return + waiting = [] + for location in self._waiting_reuse: + try: + ready = bool(self.reuse_guard(location)) + except Exception: + logger.warning( + "access policy could not answer for a page", exc_info=True + ) + ready = False + if ready: + self._free.append(location.offset // self.page_size) + else: + waiting.append(location) + self._waiting_reuse = waiting + def quarantine(self, location: StorageLocation) -> None: """Count a page a late transfer may still touch.""" with self._lock: @@ -176,6 +213,7 @@ def retire(self) -> None: with self._lock: self.retired = True self._free.clear() + self._waiting_reuse.clear() @dataclass(frozen=True) diff --git a/maru_remote/region_map.py b/maru_remote/region_map.py index 6454dd75..c1e04344 100644 --- a/maru_remote/region_map.py +++ b/maru_remote/region_map.py @@ -455,6 +455,25 @@ def release(self, handle: dict) -> None: with self._lock: self._send({"op": "release", "handle": dict(handle)}) + def extension(self, message: dict) -> dict: + """Send an access policy's message to the remote CXL agent's extension. + + Args: + message: ``action``, ``ranges``, ``write``, ``epoch`` and the + policy's own fields. + + Returns: + The extension's reply (without ``ok``). + + Raises: + RegionMapError: if the agent refused it (no extension, a range + this connection does not hold, or the extension raised). + RegionMapUnavailable: if no connection got a reply. + """ + with self._lock: + reply = self._send({**message, "op": "extension"}) + return {k: v for k, v in reply.items() if k != "ok"} + def close(self) -> None: """Close the connection, which releases every mapping it holds.""" with self._lock: diff --git a/maru_remote_cxl_agent/cli.py b/maru_remote_cxl_agent/cli.py index e3a649aa..bcf0763b 100644 --- a/maru_remote_cxl_agent/cli.py +++ b/maru_remote_cxl_agent/cli.py @@ -293,6 +293,19 @@ def build_parser() -> argparse.ArgumentParser: "one host. A late RDMA WRITE can then land in a region after its owner " "released it", ) + parser.add_argument( + "--region-extension", + default="", + help="Name of a region extension an installed package registers under " + "the maru.region_extensions entry point (e.g. a device's access " + "policy counterpart). Default: none", + ) + parser.add_argument( + "--region-extension-config", + default="", + help="Opaque string passed to the region extension, typically the path " + "of its own configuration file", + ) parser.add_argument( "--log-level", default="INFO", @@ -375,6 +388,15 @@ def main(argv: list[str] | None = None) -> int: from maru_shm import MaruShmClient epoch = uuid.uuid4().hex + extension = None + if args.region_extension: + from .extension import load_region_extension + + extension = load_region_extension( + args.region_extension, epoch, args.region_extension_config + ) + elif args.region_extension_config: + build_parser().error("--region-extension-config needs --region-extension") transport = NixlTransport( f"maru-remote-cxl-agent-{args.node_id}-{epoch[:8]}", ucx_device=args.ucx_device ) @@ -386,6 +408,7 @@ def main(argv: list[str] | None = None) -> int: node_id=args.node_id, epoch=epoch, boot_id=boot_id, + extension=extension, ) server = RegionMapServer(service, host, args.port, key) diff --git a/maru_remote_cxl_agent/extension.py b/maru_remote_cxl_agent/extension.py new file mode 100644 index 00000000..a4fa6274 --- /dev/null +++ b/maru_remote_cxl_agent/extension.py @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 XCENA Inc. +"""Region extensions: an out-of-tree counterpart of an access policy on the memory node. + +A remote CXL agent can load one region extension, registered by a separately +installed package under the ``maru.region_extensions`` entry-point group and +named on the command line (``--region-extension``). It is the memory-side +half of a worker's access policy (``maru_handler.storage.access_policy``): + +- It sees the regions this agent maps: :meth:`RegionExtension.validate_access` + may refuse one before it is mapped, :meth:`RegionExtension.mapped` learns + one once the NIC has it, and :meth:`RegionExtension.retire` is asked before + one goes back. While it answers False the region stays here, out of use and + no longer registered with the NIC, and the agent asks again on its sweep. +- It receives the messages policies send through their channel, after the + agent checked that every range in a message belongs to a region the + sending connection holds, matches its handle, fits in it, and that a read + handle does not prepare a write. + +The agent owns mapping, NIC registration and the checks above; the extension +owns whatever it does to the memory of those regions. Its hooks run under the +agent's lock and must return promptly: work that waits on hardware belongs on +the extension's own threads. +""" + +from __future__ import annotations + +import importlib.metadata +from collections.abc import Iterable +from typing import Any, Protocol, runtime_checkable + +#: Entry-point group scanned for region extensions. +REGION_EXTENSION_GROUP = "maru.region_extensions" + + +@runtime_checkable +class RegionExtension(Protocol): + """Hooks a remote CXL agent calls around the regions it maps.""" + + def validate_access(self, handle: dict, access: Any) -> None: + """Raise to refuse mapping a region (``access`` is the resource + manager's answer for it: device path and UUID).""" + ... + + def mapped(self, handle: dict, access: Any) -> None: + """A region is mapped and registered with the NIC.""" + ... + + def request(self, conn: int, message: dict, ranges: list[dict]) -> dict: + """Answer a policy message whose ``ranges`` the agent checked.""" + ... + + def retire(self, handle: dict) -> bool: + """Whether a region the NIC no longer reaches may go back now.""" + ... + + def close(self) -> None: + """The agent is ending.""" + ... + + +def load_region_extension( + name: str, + epoch: str, + config: str = "", + *, + entry_points: Iterable[Any] | None = None, +) -> RegionExtension: + """Load the region extension registered under ``name``. + + Args: + name: Entry-point name in :data:`REGION_EXTENSION_GROUP`. + epoch: Identity of this agent process; messages from a worker carry + the epoch the worker saw. + config: Opaque string passed to the factory (typically the path of + the extension's own configuration file). + entry_points: Entry points to search instead of the installed ones + (tests). + + Returns: + The extension the factory built. + + Raises: + ValueError: if no installed package registers ``name``, or the + object built lacks a hook. + """ + found = entry_points + if found is None: + found = importlib.metadata.entry_points(group=REGION_EXTENSION_GROUP) + matches = [ep for ep in found if ep.name == name] + if not matches: + raise ValueError( + f"no package registers region extension {name!r} " + f"(entry-point group {REGION_EXTENSION_GROUP})" + ) + extension = matches[0].load()(epoch, config) + if not isinstance(extension, RegionExtension): + raise ValueError(f"region extension {name!r} does not implement every hook") + return extension diff --git a/maru_remote_cxl_agent/server.py b/maru_remote_cxl_agent/server.py index 8cc6b0c1..872981e3 100644 --- a/maru_remote_cxl_agent/server.py +++ b/maru_remote_cxl_agent/server.py @@ -154,6 +154,8 @@ def handle_request( if op == "release": service.release(message.get("handle")) return {"ok": True, "released": True} + if op == "extension": + return {"ok": True, **service.extension_request(conn, message)} return {"ok": False, "error": f"unknown op {op!r}", "code": "invalid"} except MapDenied as exc: return {"ok": False, "error": str(exc), "code": "denied"} diff --git a/maru_remote_cxl_agent/service.py b/maru_remote_cxl_agent/service.py index b1661ef1..8758a345 100644 --- a/maru_remote_cxl_agent/service.py +++ b/maru_remote_cxl_agent/service.py @@ -64,6 +64,7 @@ import itertools import logging import threading +import time from collections import OrderedDict from dataclasses import dataclass, field from typing import Any @@ -85,6 +86,11 @@ # second (same node) to 30 s (another node); this bounds how much longer an # old registration lives. SWEEP_INTERVAL_S = 1.0 +# Ranges one region extension message may name +_MAX_EXTENSION_RANGES = 4096 +# How long close waits for a region extension to let its regions go +_EXTENSION_DRAIN_S = 30.0 +_EXTENSION_DRAIN_POLL_S = 0.01 # How long close() waits for a sweep in progress (it may wait on the RM). _SWEEP_JOIN_S = 5.0 @@ -136,6 +142,10 @@ class _Mapping: registration: Any = None metadata: bytes = b"" holders: set[int] = field(default_factory=set) + # Set once the NIC no longer holds the registration; a region extension + # may keep the mapping a while longer (``retiring``) before it goes back. + fenced: bool = False + retiring: bool = False def parse_handle(value: Any) -> MaruHandle | ReadHandle: @@ -201,6 +211,7 @@ def __init__( node_id: str, epoch: str, boot_id: str = "", + extension: Any = None, ) -> None: """Create the service. @@ -222,6 +233,8 @@ def __init__( boot_id: Identity of this boot of the node. A new boot lost what the node's CPU caches held, including bytes its NIC wrote there that had not reached the CXL memory yet. + extension: Region extension (``extension.RegionExtension``) or + None. """ self._rm = rm self._mapper = mapper @@ -246,19 +259,84 @@ def __init__( # Regions deregistered here whose unhold has not reached the RM yet self._unheld: dict[int, MaruHandle] = {} self._unhold_failed = False + self._extension = extension def hello(self) -> dict: """Identity of this remote CXL agent process and of the node's boot.""" self._discard_notifs() return {"node": self.node_id, "epoch": self.epoch, "boot": self.boot_id} + def extension_request(self, conn: int, message: dict) -> dict: + """Hand an access policy's message to the region extension. + + Every range must lie in a region this connection holds, mapped and + in use, under a handle that matches it; a read handle may not + prepare a write. The check and the extension's answer happen under + the lock, so a release cannot come between them. + + Args: + conn: Connection id. + message: ``action``, ``ranges``, ``write``, ``epoch`` and the + policy's own fields. + + Returns: + The extension's reply. + + Raises: + ValueError: if no extension is loaded or the message is malformed. + NotMapped: if a range's region is not held on this connection. + MapDenied: if a handle does not match, or a read handle prepares + a write, or the message is for another epoch. + """ + if self._extension is None: + raise ValueError("this remote CXL agent has no region extension") + if message.get("epoch") != self.epoch: + raise MapDenied("the message is for another remote CXL agent process") + ranges = message.get("ranges", []) + if not isinstance(ranges, list) or len(ranges) > _MAX_EXTENSION_RANGES: + raise ValueError("ranges must be a list of at most 4096 maps") + write = message.get("write") is True + with self._lock: + for r in ranges: + if not isinstance(r, dict): + raise ValueError("a range must be a map") + h = parse_handle(r.get("handle")) + m = self._maps.get(h.region_id) + if ( + m is None + or not m.ready.is_set() + or m.error is not None + or m.retiring + or conn not in m.holders + ): + raise NotMapped(f"region {h.region_id} is not held here") + if isinstance(h, ReadHandle): + if write: + raise MapDenied("a read handle cannot prepare a write") + if not h.matches(m.handle): + raise MapDenied( + f"read token does not match region {h.region_id}" + ) + elif h != m.handle: + raise MapDenied(f"handle does not match region {h.region_id}") + offset, length = r.get("offset"), r.get("length") + if ( + type(offset) is not int + or type(length) is not int + or offset < 0 + or length <= 0 + or offset + length > h.length + ): + raise ValueError("a range must lie in its region") + return self._extension.request(conn, message, ranges) + def holders(self) -> dict[int, set[int]]: """Mapped regions and the connections holding each (for tests, status).""" with self._lock: return { rid: set(m.holders) for rid, m in self._maps.items() - if m.ready.is_set() and m.error is None + if m.ready.is_set() and m.error is None and not m.retiring } def map(self, conn: int, handle: Any, nixl_agent: str | None = None) -> dict: @@ -312,9 +390,11 @@ def _map(self, conn: int, handle: Any) -> dict: step = "establish" elif not m.ready.is_set(): step = "wait" - elif m.error is None and m.handle == h: + elif m.error is None and m.handle == h and not m.retiring: m.holders.add(conn) return self._reply(m) + elif m.retiring: + raise NotMapped(f"region {h.region_id} is being released") else: step = "replace" if step == "establish": @@ -335,6 +415,8 @@ def _map_for_reader(self, conn: int, h: ReadHandle) -> dict: if m is None or (m.ready.is_set() and m.error is not None): raise NotMapped(f"region {h.region_id} is not mapped here") if m.ready.is_set(): + if m.retiring: + raise NotMapped(f"region {h.region_id} is being released") if not h.matches(m.handle): raise MapDenied( f"read token does not match region {h.region_id}" @@ -393,7 +475,10 @@ def release(self, handle: Any) -> None: if m.handle != h: raise MapDenied(f"handle does not match region {h.region_id}") if not self._release_locked(h.region_id, m): - raise RuntimeError(f"deregistering region {h.region_id} failed") + raise RuntimeError( + f"region {h.region_id} is still held: deregistering " + "failed or the region extension is still working on it" + ) self._remember_released(h) break try: @@ -449,10 +534,11 @@ def _check_maps(self) -> None: if m.ready.is_set() and m.registration is not None ] for m in held: - if m.error is not None or not m.holders: + if m.error is not None or not m.holders or m.retiring: # Out of use (failed, released or its last holder left) - # but deregistering failed: retry without asking the - # resource manager, which may be away. + # but deregistering failed, or the region extension still + # holds it: retry without asking the resource manager, + # which may be away. self._release_locked(m.handle.region_id, m) held = [ m @@ -514,17 +600,35 @@ def _discard_notifs(self) -> None: else: self._drain_failed = False - def close(self) -> None: - """Stop the sweep and release every region (the process is ending).""" + def close(self, extension_drain_s: float = _EXTENSION_DRAIN_S) -> None: + """Stop the sweep and release every region (the process is ending). + + Regions a region extension still works on get up to + ``extension_drain_s`` seconds to be let go, then the extension is + closed; what it still holds is its own to report. + """ self._stop.set() self._progress.close() if self._sweeper is not None: self._sweeper.join(timeout=_SWEEP_JOIN_S) - with self._lock: - for rid, m in list(self._maps.items()): - if m.ready.is_set() and m.registration is not None: - self._release_locked(rid, m) + deadline = time.monotonic() + extension_drain_s + while True: + with self._lock: + for rid, m in list(self._maps.items()): + if m.ready.is_set() and m.registration is not None: + self._release_locked(rid, m) + waiting = any(m.retiring for m in self._maps.values()) + if not waiting or time.monotonic() >= deadline: + break + time.sleep(_EXTENSION_DRAIN_POLL_S) self._flush_unholds() + if self._extension is not None: + if waiting: + logger.error("the region extension still holds regions at exit") + try: + self._extension.close() + except Exception: + logger.error("closing the region extension failed", exc_info=True) def _sweep_loop(self, interval_s: float) -> None: while not self._stop.wait(interval_s): @@ -688,6 +792,8 @@ def _establish(self, m: _Mapping, conn: int) -> dict: # The resource manager handed the id out again. self._released.pop(h.region_id, None) m.device = _device_of(access) + if self._extension is not None: + self._extension.validate_access(h.to_dict(), access) self._fence(m) addr = self._mapper.map( access.dax_path, access.device_uuid, h.offset, h.length @@ -702,6 +808,8 @@ def _establish(self, m: _Mapping, conn: int) -> dict: raise try: metadata = self._transport.partial_metadata(registration) + if self._extension is not None: + self._extension.mapped(h.to_dict(), access) except BaseException: # The NIC holds a key for these addresses: they are # deregistered and kept out of use as on a release. @@ -853,15 +961,35 @@ def _release_locked(self, region_id: int, m: _Mapping) -> bool: Returns: True if the region is deregistered (and no longer mapped here). """ - try: - self._transport.deregister(m.registration) - except Exception: - # Keep the mapping: unmapping a range the NIC still holds would - # leave a registration without its mapping. A mapping in use is - # shared by later requests; one out of use is deregistered again - # by the next release, sweep, map of its range or close. - logger.error("deregistering region %d failed", region_id, exc_info=True) - return False + if not m.fenced: + try: + self._transport.deregister(m.registration) + except Exception: + # Keep the mapping: unmapping a range the NIC still holds would + # leave a registration without its mapping. A mapping in use is + # shared by later requests; one out of use is deregistered again + # by the next release, sweep, map of its range or close. + logger.error("deregistering region %d failed", region_id, exc_info=True) + return False + m.fenced = True + if self._extension is not None: + # The NIC no longer reaches the region; the extension may still + # be working on its memory. Until it lets go the mapping stays, + # out of use, and the sweep, a release or close ask again. + try: + done = self._extension.retire(m.handle.to_dict()) + except Exception: + logger.error( + "region extension did not answer for region %d", + region_id, + exc_info=True, + ) + done = False + if not done: + # Out of use from now on: no holder keeps it, no request uses it. + m.retiring = True + m.holders.clear() + return False del self._maps[region_id] self._unheld[region_id] = m.handle try: diff --git a/maru_vllm/connector.py b/maru_vllm/connector.py index f8139b49..9cf67637 100644 --- a/maru_vllm/connector.py +++ b/maru_vllm/connector.py @@ -375,6 +375,8 @@ def _chunk_keys(token_ids: list[int], chunk_tokens: int) -> list[str]: "maru_rdma_staging_size", "maru_rdma_timeout_s", "maru_remote_cxl_agent_key_file", + "maru_access_policy", + "maru_access_policy_config", ) # Smallest default RDMA staging buffer (bytes), as in maru_handler.storage.remote_cxl. _MIN_RDMA_STAGING = 1024**3 @@ -734,6 +736,14 @@ def _create_maru_handler( remote_cxl_agent_key_file=extra_config.get( "maru_remote_cxl_agent_key_file", "~/.maru/remote-cxl-agent.key" ), + # A device package's access policy runs where the bytes move: the + # worker, not the metadata-only scheduler. + access_policy=( + "" if metadata_only else extra_config.get("maru_access_policy", "") + ), + access_policy_config=( + "" if metadata_only else extra_config.get("maru_access_policy_config", "") + ), rdma_load_reserve=( _RDMA_ASYNC_LOAD_RESERVE if _remote_cxl_async(extra_config) and not metadata_only diff --git a/tests/unit/remote_fakes.py b/tests/unit/remote_fakes.py index 5523d906..2cf6a16b 100644 --- a/tests/unit/remote_fakes.py +++ b/tests/unit/remote_fakes.py @@ -448,9 +448,15 @@ class FakeMemoryNode: """ def __init__( - self, cxl: FakeCxl, name: str = "mem1", port: int = 0, key: object = "test" + self, + cxl: FakeCxl, + name: str = "mem1", + port: int = 0, + key: object = "test", + extension: object = None, ) -> None: self.cxl = cxl + self.extension = extension self.name = name self._port = port self.key = shared_key() if key == "test" else key @@ -477,6 +483,7 @@ def _start(self) -> None: node_id=self.name, epoch=epoch, boot_id=f"{self.name}-boot{self.boots}", + extension=self.extension, ) self.server = RegionMapServer(self.service, "127.0.0.1", self._port, self.key) self._port = self.server.address[1] diff --git a/tests/unit/test_remote_cxl_access_policy.py b/tests/unit/test_remote_cxl_access_policy.py new file mode 100644 index 00000000..d1d29ca5 --- /dev/null +++ b/tests/unit/test_remote_cxl_access_policy.py @@ -0,0 +1,278 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Access policies and region extensions: what a device-specific package may +control in the remote CXL path, and what Maru keeps for itself. + +A fake policy on the worker and a fake extension on the remote CXL agent stand +in for a device package; the directory, the agent and the client are real. +""" + +import pytest + +import maru_handler.storage.remote_cxl.client as client_module +from maru_common.config import MaruConfig +from maru_handler.storage.access_policy import ( + AccessPolicy, + load_access_policy, +) +from maru_remote.region_map import read_handle +from maru_remote_cxl_agent.extension import RegionExtension, load_region_extension +from maru_remote_cxl_agent.service import MapDenied, NotMapped +from tests.unit.test_remote_cxl_storage import PAGE, World, put, read + + +class FakeExtension: + """Region extension that records what it sees and can hold regions.""" + + def __init__(self): + self.requests = [] + self.mapped_regions = [] + self.refuse_map = False + self.hold = False + self.closed = False + + def validate_access(self, handle, access): + if self.refuse_map: + raise ValueError("not this device") + + def mapped(self, handle, access): + self.mapped_regions.append(handle["region_id"]) + + def request(self, conn, message, ranges): + self.requests.append((message["action"], message["write"], len(ranges))) + return {"seen": len(ranges)} + + def retire(self, handle): + return not self.hold + + def close(self): + self.closed = True + + +class FakePolicy: + """Access policy that reads in steps and records its calls.""" + + def __init__(self): + self.step = 1 + self.limit = 8 + self.fail = set() + self.held = set() + self.calls = [] + + def read(self, channel, ranges, transfer): + if "read" in self.fail: + raise RuntimeError("policy refused") + assert channel.request("plan", ranges)["seen"] == len(ranges) + if "disorder" in self.fail: + transfer(1, len(ranges)) + return + stop = len(ranges) - (1 if "short" in self.fail else 0) + for i in range(0, stop, self.step): + transfer(i, min(i + self.step, stop)) + self.calls.append(("transfer", i)) + + def write_batch_limit(self, channel): + return self.limit + + def prepare_write(self, channel, ranges): + if "prepare" in self.fail: + raise RuntimeError("no room to prepare") + channel.request("prepare", ranges, write=True) + self.calls.append(("prepare", len(ranges))) + return len(ranges) + + def written(self, channel, token): + self.calls.append(("written", token)) + + def completed_write(self, locations): + self.calls.append(("late", len(locations))) + + def reusable(self, location): + return location.offset not in self.held + + +class _EntryPoint: + def __init__(self, name, target): + self.name = name + self._target = target + + def load(self): + return self._target + + +@pytest.fixture +def setup(monkeypatch): + ext = FakeExtension() + w = World(extension=ext) + policy = FakePolicy() + monkeypatch.setattr(client_module, "load_access_policy", lambda n, c: policy) + yield w, ext, policy + ext.hold = False + w.close() + + +def test_policy_and_extension_are_loaded_by_name_only(): + policy = FakePolicy() + found = [ + _EntryPoint("other", lambda c: object()), + _EntryPoint("fake", lambda c: policy), + ] + assert load_access_policy("fake", "", entry_points=found) is policy + assert isinstance(policy, AccessPolicy) + with pytest.raises(ValueError, match="no package registers"): + load_access_policy("missing", "", entry_points=found) + with pytest.raises(ValueError, match="does not implement"): + load_access_policy("other", "", entry_points=found) + ext = FakeExtension() + eps = [_EntryPoint("fake", lambda epoch, config: ext)] + assert load_region_extension("fake", "e1", "", entry_points=eps) is ext + assert isinstance(ext, RegionExtension) + with pytest.raises(ValueError, match="access_policy"): + MaruConfig( + storage_backend="cpu", + engine_id="e", + cache_namespace="ns", + access_policy="fake", + ) + + +def test_store_and_load_run_through_the_policy_and_its_extension(setup): + w, ext, policy = setup + policy.limit = 2 + c = w.client("a", access_policy="fake") + handles = [] + for i in range(3): + h = c.alloc(PAGE) + h.buf[:PAGE] = bytes([i + 1]) * PAGE + handles.append(h) + assert c.batch_store(["k0", "k1", "k2"], handles) == [True] * 3 + assert [x for x in policy.calls if x[0] == "prepare"] == [ + ("prepare", 2), + ("prepare", 1), + ] + assert [x for x in policy.calls if x[0] == "written"] == [ + ("written", 2), + ("written", 1), + ] + assert ("prepare", True, 2) in ext.requests + assert read(c, ["k0", "k1", "k2"]) == [bytes([i + 1]) * PAGE for i in range(3)] + assert ("plan", False, 3) in ext.requests + assert [x for x in policy.calls if x[0] == "transfer"] == [ + ("transfer", 0), + ("transfer", 1), + ("transfer", 2), + ] + assert ext.mapped_regions + + +@pytest.mark.parametrize("failure", ["read", "disorder", "short"]) +def test_a_read_the_policy_does_not_complete_loads_as_a_miss(setup, failure): + w, ext, policy = setup + c = w.client("a", access_policy="fake") + assert put(c, "k", b"x" * PAGE)[0] + policy.fail.add(failure) + assert read(c, ["k"]) == [None] + policy.fail.clear() + assert read(c, ["k"]) == [b"x" * PAGE] # the connection and slots are intact + + +def test_a_write_the_policy_does_not_prepare_stores_nothing(setup): + w, ext, policy = setup + c = w.client("a", access_policy="fake") + policy.fail.add("prepare") + assert put(c, "k", b"x" * PAGE)[0] is False + policy.fail.clear() + assert read(c, ["k"]) == [None] + assert put(c, "k", b"y" * PAGE)[0] + assert read(c, ["k"]) == [b"y" * PAGE] + + +def test_a_page_given_back_waits_until_the_policy_lets_it_go(setup): + w, ext, policy = setup + c = w.client("a", access_policy="fake") + region = next(iter(c.regions.values())) + before = region.free_count() + handle = c.alloc(PAGE) + policy.held.add(handle.location.offset) + policy.fail.add("prepare") + handle.buf[:PAGE] = b"x" * PAGE + assert c.batch_store(["k"], [handle]) == [False] + assert region.free_count() == before - 1 # given back, not reusable yet + policy.held.clear() + assert region.free_count() == before + + +def test_the_agent_checks_every_range_before_the_extension_sees_it(setup): + w, ext, policy = setup + c = w.client("a", access_policy="fake") + assert put(c, "k", b"x" * PAGE)[0] + service = w.node.service + region = next(iter(c.regions.values())) + rid = region.handle["region_id"] + conn = next(iter(service.holders()[rid])) + + def message(handle, write=False, offset=0, epoch=None): + return { + "action": "probe", + "write": write, + "epoch": service.epoch if epoch is None else epoch, + "ranges": [{"handle": handle, "offset": offset, "length": PAGE}], + } + + seen = len(ext.requests) + with pytest.raises(MapDenied): + service.extension_request(conn, message(region.handle, epoch="old")) + with pytest.raises(NotMapped): + service.extension_request(conn + 1000, message(region.handle)) + with pytest.raises(MapDenied): + service.extension_request(conn, message(read_handle(region.handle), write=True)) + with pytest.raises(ValueError): + service.extension_request(conn, message(region.handle, offset=region.capacity)) + assert len(ext.requests) == seen # none of these reached the extension + assert service.extension_request(conn, message(read_handle(region.handle))) + assert ext.requests[-1] == ("probe", False, 1) + + +def test_a_region_stays_until_the_extension_lets_it_go(setup): + w, ext, policy = setup + c = w.client("a", access_policy="fake") + assert put(c, "k", b"x" * PAGE)[0] + service = w.node.service + region = next(iter(c.regions.values())) + rid = region.handle["region_id"] + conn = next(iter(service.holders()[rid])) + ext.hold = True + with pytest.raises(RuntimeError, match="still held"): + service.release(region.handle) + assert rid not in service.holders() # out of use + with pytest.raises(NotMapped): + service.map(conn, region.handle) + service.sweep() + assert rid in service._maps # noqa: SLF001 (the mapping is still there) + ext.hold = False + service.sweep() + assert rid not in service._maps # noqa: SLF001 + service.release(region.handle) # already released: succeeds + + +def test_a_region_the_extension_refuses_is_not_mapped(monkeypatch): + ext = FakeExtension() + ext.refuse_map = True + w = World(extension=ext) + monkeypatch.setattr(client_module, "load_access_policy", lambda n, c: FakePolicy()) + try: + c = w.client("a", access_policy="fake") + # The agent refused every region: no page can be taken for a store. + with pytest.raises(MemoryError, match="No remote CXL region is reachable"): + c.alloc(PAGE) + assert not ext.mapped_regions + finally: + w.close() + + +def test_closing_the_agent_closes_the_extension(setup): + w, ext, policy = setup + c = w.client("a", access_policy="fake") + assert put(c, "k", b"x" * PAGE)[0] + ext.hold = True + w.node.service.close(extension_drain_s=0.05) + assert ext.closed diff --git a/tests/unit/test_remote_cxl_storage.py b/tests/unit/test_remote_cxl_storage.py index 2de862c0..60792b8b 100644 --- a/tests/unit/test_remote_cxl_storage.py +++ b/tests/unit/test_remote_cxl_storage.py @@ -70,7 +70,7 @@ def storage(self, action, payload): class World: - def __init__(self, nodes=1): + def __init__(self, nodes=1, extension=None): reset_fake_agents() self.clock = FakeClock() self.cxl = FakeCxl() @@ -80,7 +80,10 @@ def __init__(self, nodes=1): clock=self.clock, session_ttl=30, ) - self.nodes = [FakeMemoryNode(self.cxl, f"mem{i + 1}") for i in range(nodes)] + self.nodes = [ + FakeMemoryNode(self.cxl, f"mem{i + 1}", extension=extension) + for i in range(nodes) + ] self.clients: list[RemoteCxlStorageClient] = [] # MaruServer's side of the regions whose owner left; tests run it # one pass at a time. diff --git a/tests/unit/test_vllm_remote_cxl_config.py b/tests/unit/test_vllm_remote_cxl_config.py index 2e2299c4..4abace15 100644 --- a/tests/unit/test_vllm_remote_cxl_config.py +++ b/tests/unit/test_vllm_remote_cxl_config.py @@ -309,3 +309,34 @@ def refuse(config): worker._handler_retry_after = 0.0 worker._ensure_handler() assert len(errors) == 1 and errors[0][0] == connector._HANDLER_FAILURES_BEFORE_ERROR + + +def test_access_policy_settings_need_the_remote_cxl_backend(): + from tests.unit.test_vllm_cpu_config import extra as cpu_extra + + with pytest.raises(ValueError, match="remote_cxl"): + _validate_storage_config({**cpu_extra(), "maru_access_policy": "device"}) + + +@pytest.mark.parametrize("metadata_only", [False, True]) +def test_the_access_policy_goes_to_the_worker_only(monkeypatch, metadata_only): + seen = [] + + class Handler: + def __init__(self, config): + seen.append(config) + + def connect(self): + return True + + monkeypatch.setattr(maru, "MaruHandler", Handler) + _create_maru_handler( + extra( + maru_remote_cxl_agent_key_file="/k", + maru_access_policy="device", + maru_access_policy_config="/etc/device.toml", + ), + metadata_only=metadata_only, + ) + expected = ("", "") if metadata_only else ("device", "/etc/device.toml") + assert (seen[0].access_policy, seen[0].access_policy_config) == expected From 7de5abda2a794195e6e0b695c953a625c09354e1 Mon Sep 17 00:00:00 2001 From: youngrok-XCENA Date: Wed, 21 Oct 2026 12:08:44 +0900 Subject: [PATCH 2/8] feat(remote_cxl_agent): say in hello when a region extension is loaded --- maru_remote_cxl_agent/service.py | 8 ++++++-- tests/unit/test_remote_cxl_access_policy.py | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/maru_remote_cxl_agent/service.py b/maru_remote_cxl_agent/service.py index 8758a345..3b8907eb 100644 --- a/maru_remote_cxl_agent/service.py +++ b/maru_remote_cxl_agent/service.py @@ -262,9 +262,13 @@ def __init__( self._extension = extension def hello(self) -> dict: - """Identity of this remote CXL agent process and of the node's boot.""" + """Identity of this remote CXL agent process and of the node's boot; + ``region_extension`` is added when one is loaded.""" self._discard_notifs() - return {"node": self.node_id, "epoch": self.epoch, "boot": self.boot_id} + identity = {"node": self.node_id, "epoch": self.epoch, "boot": self.boot_id} + if self._extension is not None: + identity["region_extension"] = True + return identity def extension_request(self, conn: int, message: dict) -> dict: """Hand an access policy's message to the region extension. diff --git a/tests/unit/test_remote_cxl_access_policy.py b/tests/unit/test_remote_cxl_access_policy.py index d1d29ca5..163e212f 100644 --- a/tests/unit/test_remote_cxl_access_policy.py +++ b/tests/unit/test_remote_cxl_access_policy.py @@ -162,6 +162,7 @@ def test_store_and_load_run_through_the_policy_and_its_extension(setup): ("transfer", 2), ] assert ext.mapped_regions + assert w.node.service.hello()["region_extension"] is True @pytest.mark.parametrize("failure", ["read", "disorder", "short"]) From 20ca6ae375745faee83fa9a2af38839d4880c6ac Mon Sep 17 00:00:00 2001 From: youngrok-XCENA Date: Wed, 21 Oct 2026 12:23:48 +0900 Subject: [PATCH 3/8] fix(remote_cxl): tighten the access policy and region extension contracts - A late WRITE's staging slots go back once NIXL released it; only its pages wait for the policy (_late_pages), so a policy that cannot take them yet no longer keeps the slots or the unconfirmed-transfer state. - After a transfer raised, the policy cannot transfer into the same slots again, nor after read returned. - A worker with a policy refuses to message an agent without a region extension (hello says whether it has one). - Pages given back are offered to the policy at most every 10 ms, unless a store found no free page; a policy that keeps raising is logged once. - The agent refuses a non-boolean write flag. - The contracts say which hooks run under locks, that extension hooks must be thread-safe, that retire may name a region mapped never saw, and how a prepared but unwritten page ends. --- maru_handler/storage/access_policy.py | 21 ++++-- maru_handler/storage/remote_cxl/client.py | 12 +++- maru_handler/storage/remote_cxl/datapath.py | 7 ++ maru_handler/storage/remote_cxl/lifecycle.py | 48 +++++++++---- maru_handler/storage/remote_cxl/region.py | 24 +++++-- maru_remote/region_map.py | 3 + maru_remote_cxl_agent/extension.py | 13 +++- maru_remote_cxl_agent/service.py | 4 +- tests/unit/test_remote_cxl_access_policy.py | 76 +++++++++++++++++++- 9 files changed, 176 insertions(+), 32 deletions(-) diff --git a/maru_handler/storage/access_policy.py b/maru_handler/storage/access_policy.py index fb680405..c8fc34ce 100644 --- a/maru_handler/storage/access_policy.py +++ b/maru_handler/storage/access_policy.py @@ -24,8 +24,12 @@ message belongs to a region the sending connection holds before the counterpart sees it. -Every hook may block; the storage backend calls it without its lock held. A -hook that raises makes the operation take an outcome Maru already has: +Locks: ``read``, ``write_batch_limit``, ``prepare_write`` and ``written`` run +without the backend's locks and may block (a policy bounds its own waits). +``reusable`` and ``completed_write`` run while the backend holds its locks: +they must answer promptly, and a policy that cannot tell yet answers False +or raises and is asked again later. A hook that raises makes the operation +take an outcome Maru already has: ===================== =================================================== hook raises outcome @@ -119,8 +123,10 @@ def read( ``transfer(start, end)`` reads ``ranges[start:end]``. The policy must call it in order, with each ``start`` equal to the previous ``end``, - until every range is read; it decides only when each call happens. - An exception ``transfer`` raises must propagate. + until every range is read, and only on the calling thread before + ``read`` returns; it decides only when each call happens. Once a + transfer raised, every later call raises too, and the exception + should propagate. """ ... @@ -131,7 +137,12 @@ def write_batch_limit(self, channel: AccessChannel) -> int: def prepare_write( self, channel: AccessChannel, ranges: Sequence[AccessRange] ) -> Any: - """Make ``ranges`` ready to be written; return a token for :meth:`written`.""" + """Make ``ranges`` ready to be written; return a token for :meth:`written`. + + A prepared page whose WRITE never reports :meth:`written` (it failed, + or its region went back) is next seen through :meth:`reusable`; the + policy ends the preparation there. + """ ... def written(self, channel: AccessChannel, token: Any) -> None: diff --git a/maru_handler/storage/remote_cxl/client.py b/maru_handler/storage/remote_cxl/client.py index f91cdc24..cf0ba6e8 100644 --- a/maru_handler/storage/remote_cxl/client.py +++ b/maru_handler/storage/remote_cxl/client.py @@ -163,6 +163,7 @@ def __init__( # lease while the session lives, and a leased key cannot be evicted self._unreleased: list[str] = [] self._isolated: list[IsolatedTransfer] = [] + self._late_pages: list[tuple[RemoteCxlRegion, list[StorageLocation]]] = [] self._staging_warned = False # RDMA transfers run without the lock; close waits for them to end. self._transfers = 0 @@ -1371,9 +1372,15 @@ def _policy_read( BaseException: what the policy or the transport raised. """ cursor = 0 + done = False def transfer(start: int, end: int) -> None: nonlocal cursor + if failure: + # A READ may still land in the slots: no further READ into them. + raise failure[0] + if done: + raise RuntimeError("the read already returned; its slots are gone") if start != cursor or not start < end <= len(pairs): raise ValueError( f"access policy read ranges[{start}:{end}] after {cursor}" @@ -1387,7 +1394,10 @@ def transfer(start: int, end: int) -> None: raise cursor = end - self._policy.read(_RegionChannel(self._data, access), ranges, transfer) + try: + self._policy.read(_RegionChannel(self._data, access), ranges, transfer) + finally: + done = True if failure: raise failure[0] if cursor != len(pairs): diff --git a/maru_handler/storage/remote_cxl/datapath.py b/maru_handler/storage/remote_cxl/datapath.py index ddbc3127..1cdf24d2 100644 --- a/maru_handler/storage/remote_cxl/datapath.py +++ b/maru_handler/storage/remote_cxl/datapath.py @@ -213,6 +213,13 @@ def extension_request(self, access: RegionAccess, message: dict) -> dict: raise StorageUnavailableError( "the connection that mapped the region changed" ) + if not client.region_extension: + # The worker has an access policy and the agent nothing to answer + # it: a configuration error, not a passing outage. + raise StorageError( + f"remote CXL agent {access.agent} has no region extension for " + "this worker's access policy" + ) try: reply = client.extension({**message, "epoch": client.epoch}) except RegionMapUnavailable as exc: diff --git a/maru_handler/storage/remote_cxl/lifecycle.py b/maru_handler/storage/remote_cxl/lifecycle.py index f9a1ba6c..00aa2ab5 100644 --- a/maru_handler/storage/remote_cxl/lifecycle.py +++ b/maru_handler/storage/remote_cxl/lifecycle.py @@ -51,6 +51,8 @@ class RegionLifecycle: _removing: list[dict] _dropping: list[dict] _isolated: list[IsolatedTransfer] + # Pages of late-completed WRITEs waiting for the access policy + _late_pages: list[tuple[RemoteCxlRegion, list[StorageLocation]]] _expand_op: str | None _credentials: dict _lock: threading.RLock @@ -559,33 +561,49 @@ def _reap_isolated(self) -> None: still be in the network, so its pages stay out of use while this worker owns the region, as after a WRITE that fails at once. """ + policy = getattr(self, "_policy", None) kept = [] for entry in self._isolated: if entry.pending.finished or entry.pending.poll(): # Pages go back to the region they came from; a retired # region ignores them. if entry.region is not None and entry.pending.completed: - policy = getattr(self, "_policy", None) - if policy is not None: - # The policy prepared these pages for the WRITE that - # turned out complete; it finishes them first. - try: - policy.completed_write(entry.locations) - except Exception: - logger.warning( - "access policy did not take a late WRITE", exc_info=True - ) - kept.append(entry) - continue - for location in entry.locations: - entry.region.unquarantine(location) - entry.region.give_page(location) + if policy is not None and not entry.region.retired: + # The access policy prepared these pages for the WRITE + # that turned out complete: they wait for it to take + # them (_settle_late_pages). The slots do not. + self._late_pages.append((entry.region, list(entry.locations))) + else: + for location in entry.locations: + entry.region.unquarantine(location) + entry.region.give_page(location) if self._data.staging is not None: for slot in entry.slots: self._data.staging.give(slot) else: kept.append(entry) self._isolated = kept + self._settle_late_pages() + + def _settle_late_pages(self) -> None: + """Hand the pages of late-completed WRITEs to the access policy, then + back to their region; a page the policy did not take waits for the + next housekeeping pass.""" + policy = getattr(self, "_policy", None) + waiting = [] + for region, locations in self._late_pages: + if region.retired: + continue # its pages went back with it + try: + policy.completed_write(locations) + except Exception as exc: + logger.warning("access policy did not take a late WRITE: %s", exc) + waiting.append((region, locations)) + continue + for location in locations: + region.unquarantine(location) + region.give_page(location) + self._late_pages = waiting def _settle_writes(self, timeout_s: float) -> bool: """Wait (bounded) for the unconfirmed WRITEs to end. diff --git a/maru_handler/storage/remote_cxl/region.py b/maru_handler/storage/remote_cxl/region.py index a49464ae..9f2764d8 100644 --- a/maru_handler/storage/remote_cxl/region.py +++ b/maru_handler/storage/remote_cxl/region.py @@ -5,6 +5,7 @@ import logging import threading +import time import uuid from collections import deque from collections.abc import Callable @@ -16,6 +17,8 @@ logger = logging.getLogger(__name__) REMOTE_CXL = "remote_cxl" +# How often the access policy is asked about pages given back +REUSE_POLL_S = 0.01 @dataclass(eq=False) @@ -90,6 +93,8 @@ def __init__(self, pool_id: str, capacity: int, page_size: int, handle: dict): # (``AccessPolicy.reusable``); pages it holds wait in _waiting_reuse. self.reuse_guard: Callable[[StorageLocation], bool] | None = None self._waiting_reuse: list[StorageLocation] = [] + self._reuse_asked_at = float("-inf") + self._reuse_warned = False def free_count(self) -> int: """Free pages a store can take now (none while the region is unusable).""" @@ -120,7 +125,8 @@ def take_page(self, size: int) -> StorageLocation: ) if type(size) is not int or not 0 < size <= self.page_size: raise ValueError("Remote CXL allocation must fit in one page") - self._reap_reusable() + # Out of pages: ask now rather than wait for the next interval. + self._reap_reusable(force=not self._free) if not self._free: raise MemoryError("Remote CXL region is full") page = self._free.popleft() @@ -167,21 +173,27 @@ def give_page(self, location: StorageLocation) -> None: return self._free.append(location.offset // self.page_size) - def _reap_reusable(self) -> None: + def _reap_reusable(self, force: bool = False) -> None: """Free the pages given back that the access policy lets go now. + The policy is asked at most every ``REUSE_POLL_S`` unless ``force``. A page the policy does not answer for (it raised) stays waiting. """ if not self._waiting_reuse: return + now = time.monotonic() + if not force and now - self._reuse_asked_at < REUSE_POLL_S: + return + self._reuse_asked_at = now waiting = [] for location in self._waiting_reuse: try: ready = bool(self.reuse_guard(location)) - except Exception: - logger.warning( - "access policy could not answer for a page", exc_info=True - ) + self._reuse_warned = False + except Exception as exc: + if not self._reuse_warned: + logger.warning("access policy could not answer for a page: %s", exc) + self._reuse_warned = True ready = False if ready: self._free.append(location.offset // self.page_size) diff --git a/maru_remote/region_map.py b/maru_remote/region_map.py index c1e04344..0ec98e93 100644 --- a/maru_remote/region_map.py +++ b/maru_remote/region_map.py @@ -373,6 +373,8 @@ def __init__( self.node = "" self.boot = "" self.generation = 0 + # Whether the agent has a region extension (from its hello) + self.region_extension = False def hello(self) -> dict: """Connect if needed and return the remote CXL agent's identity. @@ -526,6 +528,7 @@ def _ensure(self) -> None: self.epoch = str(reply["epoch"]) self.node = str(reply["node"]) self.boot = boot + self.region_extension = reply.get("region_extension") is True def _authenticate(self, nonce: Any) -> None: """Prove the key to the remote CXL agent and check its proof. diff --git a/maru_remote_cxl_agent/extension.py b/maru_remote_cxl_agent/extension.py index a4fa6274..8632b8d9 100644 --- a/maru_remote_cxl_agent/extension.py +++ b/maru_remote_cxl_agent/extension.py @@ -18,9 +18,16 @@ handle does not prepare a write. The agent owns mapping, NIC registration and the checks above; the extension -owns whatever it does to the memory of those regions. Its hooks run under the -agent's lock and must return promptly: work that waits on hardware belongs on -the extension's own threads. +owns whatever it does to the memory of those regions. + +Threads and locks: ``request`` and ``retire`` run under the agent's lock, so +every other request of every worker waits for them; they must return +promptly, and work that waits on hardware belongs on the extension's own +threads. ``validate_access`` and ``mapped`` run on the threads that map +regions, possibly several at once. The extension must be thread-safe. + +``retire`` may be asked about a region ``mapped`` never saw (its mapping +failed half way); the extension answers True for a region it does not know. """ from __future__ import annotations diff --git a/maru_remote_cxl_agent/service.py b/maru_remote_cxl_agent/service.py index 3b8907eb..01ec2386 100644 --- a/maru_remote_cxl_agent/service.py +++ b/maru_remote_cxl_agent/service.py @@ -299,7 +299,9 @@ def extension_request(self, conn: int, message: dict) -> dict: ranges = message.get("ranges", []) if not isinstance(ranges, list) or len(ranges) > _MAX_EXTENSION_RANGES: raise ValueError("ranges must be a list of at most 4096 maps") - write = message.get("write") is True + write = message.get("write", False) + if type(write) is not bool: + raise ValueError("write must be true or false") with self._lock: for r in ranges: if not isinstance(r, dict): diff --git a/tests/unit/test_remote_cxl_access_policy.py b/tests/unit/test_remote_cxl_access_policy.py index 163e212f..765559e8 100644 --- a/tests/unit/test_remote_cxl_access_policy.py +++ b/tests/unit/test_remote_cxl_access_policy.py @@ -17,7 +17,14 @@ from maru_remote.region_map import read_handle from maru_remote_cxl_agent.extension import RegionExtension, load_region_extension from maru_remote_cxl_agent.service import MapDenied, NotMapped -from tests.unit.test_remote_cxl_storage import PAGE, World, put, read +from tests.unit.test_remote_cxl_storage import ( + PAGE, + World, + only_region, + put, + read, + worker_agent, +) class FakeExtension: @@ -65,6 +72,12 @@ def read(self, channel, ranges, transfer): if "disorder" in self.fail: transfer(1, len(ranges)) return + if "retry" in self.fail: + try: + transfer(0, 1) + except Exception: + transfer(0, 1) # must raise: a READ may still land + return stop = len(ranges) - (1 if "short" in self.fail else 0) for i in range(0, stop, self.step): transfer(i, min(i + self.step, stop)) @@ -84,6 +97,8 @@ def written(self, channel, token): self.calls.append(("written", token)) def completed_write(self, locations): + if "late" in self.fail: + raise RuntimeError("cannot take it yet") self.calls.append(("late", len(locations))) def reusable(self, location): @@ -99,6 +114,13 @@ def load(self): return self._target +@pytest.fixture(autouse=True) +def _ask_reuse_every_time(monkeypatch): + import maru_handler.storage.remote_cxl.region as region_module + + monkeypatch.setattr(region_module, "REUSE_POLL_S", 0.0) + + @pytest.fixture def setup(monkeypatch): ext = FakeExtension() @@ -277,3 +299,55 @@ def test_closing_the_agent_closes_the_extension(setup): ext.hold = True w.node.service.close(extension_drain_s=0.05) assert ext.closed + + +def test_a_failed_transfer_cannot_be_retried_into_the_same_slots(setup): + w, ext, policy = setup + c = w.client("a", access_policy="fake") + assert put(c, "k", b"x" * PAGE)[0] + calls = [] + + def broken(peer, pairs, **kwargs): + calls.append(len(pairs)) + raise RuntimeError("NIC error") + + c._transport.read = broken + policy.fail.add("retry") + assert read(c, ["k"]) == [None] + assert calls == [1] # the second transfer never reached the transport + + +def test_a_policy_without_an_agent_extension_stores_and_loads_nothing(monkeypatch): + w = World() # the agent has no region extension + policy = FakePolicy() + monkeypatch.setattr(client_module, "load_access_policy", lambda n, c: policy) + try: + c = w.client("a", access_policy="fake") + assert put(c, "k", b"x" * PAGE)[0] is False + assert not any(x[0] == "prepare" for x in policy.calls) + finally: + w.close() + + +def test_a_late_write_returns_its_slot_and_its_page_waits_for_the_policy(setup): + w, ext, policy = setup + c = w.client("a", pages=2, rdma_timeout_s=0.01, access_policy="fake") + agent = worker_agent("a") + agent.stall = True + handle = c.alloc(4) + handle.buf[:4] = b"late" + assert c.batch_store(["k"], [handle]) == [False] + region = only_region(c) + free_slots = c._staging.free_count() + policy.fail.add("late") + agent.stall = False + agent.finish_stalled() + c._reap_isolated() + # The NIXL transfer ended: the slot is back, the page waits for the policy. + assert c._staging.free_count() == free_slots + 1 + assert not c.has_unconfirmed_transfers() + assert region.usage()["quarantined_bytes"] == PAGE + policy.fail.discard("late") + c._reap_isolated() + assert ("late", 1) in policy.calls + assert region.usage() == {"allocated_bytes": 0, "quarantined_bytes": 0} From 32175fddb644810556c9da433248070260205e7f Mon Sep 17 00:00:00 2001 From: youngrok-XCENA Date: Wed, 21 Oct 2026 20:18:15 +0900 Subject: [PATCH 4/8] fix(remote_cxl_agent): end cleanly when the region extension cannot load Report the extension and the reason, give the node lock back and exit 1 instead of leaving a traceback with the lock held. --- maru_remote_cxl_agent/cli.py | 14 +++++++++++--- tests/unit/test_remote_cxl_agent.py | 20 ++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/maru_remote_cxl_agent/cli.py b/maru_remote_cxl_agent/cli.py index bcf0763b..b91e597e 100644 --- a/maru_remote_cxl_agent/cli.py +++ b/maru_remote_cxl_agent/cli.py @@ -392,10 +392,18 @@ def main(argv: list[str] | None = None) -> int: if args.region_extension: from .extension import load_region_extension - extension = load_region_extension( - args.region_extension, epoch, args.region_extension_config - ) + try: + extension = load_region_extension( + args.region_extension, epoch, args.region_extension_config + ) + except Exception as exc: + logger.error( + "cannot load region extension %s: %s", args.region_extension, exc + ) + os.close(lock_fd) + return 1 elif args.region_extension_config: + os.close(lock_fd) build_parser().error("--region-extension-config needs --region-extension") transport = NixlTransport( f"maru-remote-cxl-agent-{args.node_id}-{epoch[:8]}", ucx_device=args.ucx_device diff --git a/tests/unit/test_remote_cxl_agent.py b/tests/unit/test_remote_cxl_agent.py index 571bad7d..8d3d2ae9 100644 --- a/tests/unit/test_remote_cxl_agent.py +++ b/tests/unit/test_remote_cxl_agent.py @@ -1021,3 +1021,23 @@ def test_a_worker_refuses_an_agent_without_a_boot_id(): client.map(cxl.reserve("owner", 4096).to_dict()) finally: mem.stop() + + +def test_main_refuses_to_start_when_its_region_extension_cannot_load( + monkeypatch, tmp_path +): + import maru_remote_cxl_agent.extension as extension + + def refuse(name, epoch, config): + raise ValueError(f"no package registers region extension {name!r}") + + monkeypatch.setattr(extension, "load_region_extension", refuse) + monkeypatch.setenv("UCX_TLS", "rc") + argv = ["--rm-address", "mem1:9850", "--lock-file", str(tmp_path / "lock")] + argv += ["--key-file", key_file(), "--host", "127.0.0.1"] + assert cli.main([*argv, "--region-extension", "device"]) == 1 + # The node lock went with the failed start: another agent may take it. + lock_fd = cli.hold_node_lock(str(tmp_path / "lock")) + import os + + os.close(lock_fd) From d739b0b67e3143cdff19e5199a5864c8929421e0 Mon Sep 17 00:00:00 2001 From: youngrok-XCENA Date: Fri, 23 Oct 2026 15:49:00 +0900 Subject: [PATCH 5/8] feat(remote_cxl): read hints, so an access policy can prepare a load's bytes early An access policy hears of a read only when the worker starts it: with memory that keeps a fast cache in front of slower media, the bytes are not ready and the READs follow the device's fill chunk by chunk. The engine knows which keys a request will load much earlier. - The scheduler connector gives the backend a read hint once per request: at the first lookup that finds an asynchronous load, or, with maru_read_hint_on_arrival, as soon as vLLM's engine receives the request (EngineCore.preprocess_add_request). Its end follows request_finished. - The backend queues the hint for one thread per client: a probe that also returns where each hit is (with a read handle, no lease), then the policy's optional read_hint / read_hint_end hooks with a channel to each region's remote CXL agent. Callers never wait. Without such a policy nothing runs. - The remote CXL agent accepts a hint for a region it maps from a connection that holds none (the scheduler has no data path), never as a write. - The worker names the request its reads load for (set_read_request), and the policy's read hook receives it. - The scheduler's handler gets the access policy too (it only hears hints), and a prompt's chunk keys are hashed once for the lookup, the hint and the load. --- docs/index.md | 1 + docs/source/api_reference/config.md | 5 + docs/source/design_doc/access_policies.md | 103 +++++++++ maru_handler/handler.py | 35 ++++ maru_handler/storage/access_policy.py | 37 ++++ maru_handler/storage/remote_cxl/client.py | 107 +++++++++- maru_handler/storage/remote_cxl/hints.py | 165 +++++++++++++++ maru_handler/storage/remote_cxl/router.py | 17 ++ maru_remote_cxl_agent/extension.py | 5 +- maru_remote_cxl_agent/service.py | 15 +- maru_server/replica_directory.py | 17 +- maru_vllm/connector.py | 220 +++++++++++++++++--- tests/unit/test_remote_cxl_access_policy.py | 3 +- tests/unit/test_remote_cxl_read_hints.py | 170 +++++++++++++++ tests/unit/test_vllm_read_hints.py | 139 +++++++++++++ tests/unit/test_vllm_remote_cxl_config.py | 7 +- 16 files changed, 1008 insertions(+), 38 deletions(-) create mode 100644 docs/source/design_doc/access_policies.md create mode 100644 maru_handler/storage/remote_cxl/hints.py create mode 100644 tests/unit/test_remote_cxl_read_hints.py create mode 100644 tests/unit/test_vllm_read_hints.py diff --git a/docs/index.md b/docs/index.md index bbb26417..a47ce990 100644 --- a/docs/index.md +++ b/docs/index.md @@ -125,6 +125,7 @@ Architecture Overview Memory Model KV Cache Management Consistency and Safety +Access Policies and Read Hints CPU/CXL L1 and SSD L2 (Proposal) ``` diff --git a/docs/source/api_reference/config.md b/docs/source/api_reference/config.md index 2b23167b..6fc84de6 100644 --- a/docs/source/api_reference/config.md +++ b/docs/source/api_reference/config.md @@ -168,6 +168,9 @@ apply to that backend only: | `maru_rdma_device` | `str` | UCX default | The engine's RDMA NIC, e.g. `mlx5_0:1` | | `maru_rdma_staging_size` | `str \| int` | two `max_model_len` requests, at least 1 GiB | Host staging buffer registered with the NIC and CUDA | | `maru_rdma_timeout_s` | `float` | `30` | Deadline of one RDMA batch | +| `maru_access_policy` | `str` | empty | Name of an installed access policy (entry point in `maru.access_policies`) that decides when the backend's transfers run; see [Access policies](../design_doc/access_policies.md) | +| `maru_access_policy_config` | `str` | empty | Opaque string given to that policy, typically its own configuration file | +| `maru_read_hint_on_arrival` | `bool` | `false` | Give the access policy its read hint when vLLM receives a request, not when the scheduler first looks it up; needs `maru_async_load` | ### Remote CXL agent (`maru-remote-cxl-agent`) @@ -184,6 +187,8 @@ it, and the service reads these variables from `/etc/maru/remote-cxl-agent.env`. | `--key-file` (`MARU_REMOTE_CXL_AGENT_KEY_FILE`) | `/etc/maru/remote-cxl-agent.key` | Key every connection must prove; the installer creates it | | `--node-id` | hostname | Name of the memory node; names the default lock file | | `--lock-file` (`MARU_REMOTE_CXL_AGENT_LOCK_FILE`) | `/run/maru/remote-cxl-agent-.lock` | Keeps one remote CXL agent per node | +| `--region-extension` | none | Name of an installed region extension (entry point in `maru.region_extensions`), the memory-node half of an access policy; see [Access policies](../design_doc/access_policies.md) | +| `--region-extension-config` | empty | Opaque string given to that extension | | `--allow-unchecked-transport` | off | Accept UCX transports that do not check remote keys (tests on one host only) | | `--log-level` | `INFO` | Logging level (DEBUG, INFO, WARNING, ERROR) | diff --git a/docs/source/design_doc/access_policies.md b/docs/source/design_doc/access_policies.md new file mode 100644 index 00000000..38b9fe6a --- /dev/null +++ b/docs/source/design_doc/access_policies.md @@ -0,0 +1,103 @@ +# Access Policies and Read Hints + +Some CXL memory answers every address but not at the same speed: a device may +keep a small fast cache in front of larger, slower media, so bytes have to be +brought into the cache before a read and written back after a write. Maru keeps +no knowledge of such devices. The `remote_cxl` backend instead offers two +extension points that a separately installed package fills in, and a read hint +that tells that package early which bytes a load will read. + +## Who owns what + +```mermaid +flowchart LR + subgraph gpu["GPU node"] + sched["vLLM scheduler
+ Maru connector"] + worker["vLLM worker
+ Maru connector"] + polS["access policy
(read hints)"] + polW["access policy
(read, write)"] + sched --- polS + worker --- polW + end + subgraph mem["memory node"] + server["maru-server
(keys, leases)"] + agent["remote CXL agent
(map, register, check)"] + ext["region extension"] + dev[("CXL memory")] + agent --- ext + ext -. "device control" .-> dev + end + sched -- "lookup" --> server + worker -- "lease" --> server + polS -- "read hint" --> agent + polW -- "policy messages" --> agent + worker == "RDMA READ / WRITE" ==> dev +``` + +- **Maru** decides which keys hit, which pages hold them, which bytes move and + whether a key is published. It maps and registers regions on the memory node + and checks every policy message against the regions it maps. +- **The access policy** (entry point group `maru.access_policies`, selected with + `maru_access_policy`) runs in the engine's processes. In the worker it decides + *when* each READ runs and whether a page may be written or reused. In the + scheduler it only hears read hints. +- **The region extension** (entry point group `maru.region_extensions`, selected + with `--region-extension` on the agent) is the policy's counterpart next to + the memory. It receives the policy's messages after the agent checked them, + and whatever it does to the device is its own business. + +Installing a package selects nothing; only the configured names are loaded. +The hook contracts are documented in `maru_handler/storage/access_policy.py`, +`maru_handler/storage/remote_cxl/hints.py` and +`maru_remote_cxl_agent/extension.py`. + +## Why read hints + +A policy that is told about a read only when the worker starts it has no time +to prepare: the first bytes are not in the fast cache yet, and if the device +fills the cache about as fast as the network reads it, the READs follow the +fill chunk by chunk for the whole load. + +The engine knows much earlier which keys a request will load. A request that +arrives while an engine step runs waits for that step to end before the +scheduler picks it, and the scheduler then admits the load before the worker +starts it. A read hint hands that knowledge to the policy at the earliest of +these points, so the device can fill while the request waits. + +```mermaid +sequenceDiagram + participant E as vLLM engine
(input thread) + participant S as Maru connector
(scheduler) + participant H as hint thread + participant A as remote CXL agent
+ region extension + participant W as Maru connector
(worker) + E->>S: request received (maru_read_hint_on_arrival) + S->>H: hint_read(request, keys) + Note over E: request waits for the running step + H->>H: probe with locations: leading hits per region + H->>A: read_hint(request, ranges) + Note over A: device prepares the bytes + S->>S: scheduler picks the request (first lookup: hint once) + W->>W: lease the keys + W->>A: policy.read(..., request): READs run as bytes are ready + S->>H: end_read_hint(request) when the request finishes + H->>A: read_hint_end(request) +``` + +- **Where hints come from.** By default the scheduler connector hints a request + the first time it finds an asynchronous load for it. With + `maru_read_hint_on_arrival` it hints as soon as vLLM's engine receives the + request, by wrapping `EngineCore.preprocess_add_request`; the first lookup + then finds the request hinted. A request is hinted at most once, never after + it finished, and its end follows its hint. +- **No waiting.** The caller only queues the hint. One thread per backend looks + the keys up at maru-server (a probe that also returns where each hit is and a + read handle of its region; it takes no lease) and calls the policy, in the + order hints were given. +- **Advice only.** The agent accepts a hint for a region it maps even from a + process that holds no region (the scheduler has no data path), but never as a + write. A hint that is late, lost or wrong costs only the time it would have + saved: the load still leases and reads every key, and the policy's `read` + hook learns the request id it serves, so it can match the load with its hint. +- **Off without a policy.** Without an access policy that defines `read_hint`, + no thread starts and no probe is sent. diff --git a/maru_handler/handler.py b/maru_handler/handler.py index 7c52b9d8..ad51bbda 100644 --- a/maru_handler/handler.py +++ b/maru_handler/handler.py @@ -1272,6 +1272,41 @@ def batch_exists(self, keys: list[str], route_key: str | None = None) -> list[bo ) return batch_resp.results + def hint_read( + self, request_id: str, keys: list[str], route_key: str | None = None + ) -> None: + """Tell storage that ``request_id`` will read ``keys`` soon, in order. + + Returns at once. Storage whose access policy takes read hints looks + the keys up and lets the policy prepare their bytes before the load + starts; other storage ignores it. The load still reads every key. + + Args: + request_id: The serving engine's request id. + keys: Chunk keys the request's load will read, in order. + route_key: The request's first chunk key (picks the pool, as in + :meth:`batch_exists`). + """ + hint = getattr(self._cpu, "hint_read", None) + if hint is not None: + hint(request_id, keys, route_key=route_key) + + def end_read_hint(self, request_id: str) -> None: + """``request_id`` ended; storage may drop what it prepared for it.""" + end = getattr(self._cpu, "end_read_hint", None) + if end is not None: + end(request_id) + + def set_read_request(self, request_id: str) -> None: + """Name the request this thread's next ``batch_retrieve`` calls load for. + + Storage whose access policy takes read hints matches those reads + with the request's hint; ``""`` clears the name. + """ + mark = getattr(self._cpu, "set_read_request", None) + if mark is not None: + mark(request_id) + def batch_pin(self, keys: list[str]) -> list[bool]: """Check existence and pin multiple keys in a single RPC call. diff --git a/maru_handler/storage/access_policy.py b/maru_handler/storage/access_policy.py index c8fc34ce..f3c3dcc2 100644 --- a/maru_handler/storage/access_policy.py +++ b/maru_handler/storage/access_policy.py @@ -18,6 +18,12 @@ - The policy decides *when* the bytes of a read move (it calls the transfer Maru hands it, in order), whether a page may be written now, and whether a page given back may be reused yet. +- A policy may also take read hints (optional hooks ``read_hint`` and + ``read_hint_end``, see :class:`ReadHintHooks`): the engine knows which + keys a request will load well before the load starts, and the backend + tells the policy where those bytes are, so slow media can be read ahead + while the request waits. Hints run in the process that gives them (a + serving engine's scheduler), off its threads. - The policy talks to its own counterpart next to the memory through :class:`AccessChannel`. Maru carries those messages over the connection it already authenticated, and the memory side checks that every range in a @@ -118,9 +124,13 @@ def read( channel: AccessChannel, ranges: Sequence[AccessRange], transfer: Callable[[int, int], None], + request_id: str = "", ) -> None: """Move the bytes of ``ranges`` by calling ``transfer``. + ``request_id`` names the request whose load reads them, as a read + hint named it ("" if the engine did not say). + ``transfer(start, end)`` reads ``ranges[start:end]``. The policy must call it in order, with each ``start`` equal to the previous ``end``, until every range is read, and only on the calling thread before @@ -158,6 +168,33 @@ def reusable(self, location: StorageLocation) -> bool: ... +class ReadHintHooks(Protocol): + """Optional hooks of an access policy that takes read hints. + + The backend calls them on its hint thread, in the order the hints were + given; a policy without them gets no hints. They may block (only the + hint thread waits). An exception is logged and the hint dropped: the + load reads every byte either way. + """ + + def read_hint( + self, channel: Any, request_id: str, ranges: Sequence[AccessRange] + ) -> None: + """``request_id`` will read ``ranges`` of one region, in this order. + + ``channel.request(action, ranges, **payload)`` reaches the region + extension of the agent that maps the region. The agent takes a hint + for a region this process does not hold, as long as it maps it, and + never as a write. The bytes are not leased: they may be gone or + replaced by the time the load leases them. + """ + ... + + def read_hint_end(self, channel: Any, request_id: str) -> None: + """``request_id`` ended; what was prepared for it may go.""" + ... + + def load_access_policy( name: str, config: str = "", diff --git a/maru_handler/storage/remote_cxl/client.py b/maru_handler/storage/remote_cxl/client.py index cf0ba6e8..01b99b5d 100644 --- a/maru_handler/storage/remote_cxl/client.py +++ b/maru_handler/storage/remote_cxl/client.py @@ -25,7 +25,8 @@ from ..access_policy import AccessRange, load_access_policy from ..cpu import CpuReadLease from ..staging import StagingBuffer -from .datapath import RdmaDataPath +from .datapath import REPLY_TIMEOUT_MIN_S, RdmaDataPath +from .hints import ReadHints from .lifecycle import RegionLifecycle from .region import ( REMOTE_CXL, @@ -182,6 +183,98 @@ def __init__( if getattr(config, "access_policy", "") else None ) + # Read hints (hints.py): the policy hears which bytes a load will + # read before it starts. Off without a policy that takes them. + self._hints = ( + ReadHints(self._policy, self._hint_locate, self._hint_send) + if self._policy is not None + else None + ) + self._hint_agents: dict[str, Any] = {} + self._hint_key: bytes | None = None + # The request whose load this thread reads (read_context) + self._reading = threading.local() + + # ------------------------------------------------------------ read hints + + def hint_read( + self, request_id: str, keys: list[str], route_key: str | None = None + ) -> None: + """Tell the access policy that ``request_id`` will read ``keys`` soon. + + Returns at once: the lookup of the leading hit keys and the policy's + ``read_hint`` run on the hint thread. Without a policy that takes + hints this does nothing. A hint takes no lease; the load still + leases and reads every key. + """ + if self._hints is not None and self._hints.enabled: + self._hints.hint(request_id, keys) + + def end_read_hint(self, request_id: str) -> None: + """``request_id`` ended: the policy may drop what it prepared for it.""" + if self._hints is not None and self._hints.enabled: + self._hints.end(request_id) + + def set_read_request(self, request_id: str) -> None: + """Name the request whose load this thread's next reads belong to. + + The access policy's ``read`` hook gets the name, so it can find what + a read hint prepared; ``""`` clears it. + """ + self._reading.request_id = request_id + + def _hint_locate(self, keys: list[str]) -> list[tuple[str, list[AccessRange]]]: + """The leading hit keys, grouped per region in key order, with the + remote CXL agent of each region (on the hint thread).""" + from maru_remote.region_map import pick_agent + + self._ensure(data=False) + groups: list[tuple[str, list[AccessRange]]] = [] + current = None + for i in range(0, len(keys), MAX_STORAGE_BATCH): + batch = keys[i : i + MAX_STORAGE_BATCH] + result = self._call( + "probe", + engine_id=self.config.engine_id, + namespace=self.config.cache_namespace, + keys=batch, + locations=True, + ) + entries = result.get("entries") or [] + regions = result.get("regions") or {} + for entry in entries[: len(batch)]: + if entry is None: + return groups # the load reads the leading hits only + location = StorageLocation(**entry) + handle = regions.get(location.pool_id) + if handle is None or not self._agents: + return groups + if current != location.pool_id: + agent = pick_agent(self._agents, int(handle["region_id"])) + groups.append((agent, [])) + current = location.pool_id + groups[-1][1].append(AccessRange(handle, location)) + if len(entries) < len(batch): + return groups + return groups + + def _hint_send(self, agent: str, message: dict) -> dict: + """Send a hint message on this process's own connection to ``agent``.""" + from maru_remote.region_map import RegionMapClient, load_key + + client = self._hint_agents.get(agent) + if client is None: + if self._hint_key is None: + self._hint_key = load_key(self.config.remote_cxl_agent_key_file) + client = self._hint_agents[agent] = RegionMapClient( + agent, + timeout_s=max(self.config.rdma_timeout_s, REPLY_TIMEOUT_MIN_S), + key=self._hint_key, + ) + client.hello() # connects if needed; the epoch below is its agent's + if not client.region_extension: + raise StorageError(f"remote CXL agent {agent} has no region extension") + return client.extension({**message, "epoch": client.epoch}) # ------------------------------------------------------------------ setup @@ -1395,7 +1488,12 @@ def transfer(start: int, end: int) -> None: cursor = end try: - self._policy.read(_RegionChannel(self._data, access), ranges, transfer) + self._policy.read( + _RegionChannel(self._data, access), + ranges, + transfer, + getattr(self._reading, "request_id", ""), + ) finally: done = True if failure: @@ -1461,6 +1559,11 @@ def close(self) -> None: for thread in self._threads: thread.join() self._threads = [] + if self._hints is not None: + self._hints.close() + for agent in self._hint_agents.values(): + agent.close() + self._hint_agents.clear() with self._lock: # Transfers run without the lock: let those under way end (each # within its timeout) before the regions and the data path go. diff --git a/maru_handler/storage/remote_cxl/hints.py b/maru_handler/storage/remote_cxl/hints.py new file mode 100644 index 00000000..52556d30 --- /dev/null +++ b/maru_handler/storage/remote_cxl/hints.py @@ -0,0 +1,165 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 XCENA Inc. +"""Read hints: tell an access policy which bytes a load will read, before it starts. + +A serving engine knows which stored keys a request will load well before +its worker starts the load: the request waits for the running engine step +first, and the scheduler looks its prefix up when it picks it. Memory with a +slow tier behind a fast cache (see ``access_policy``) can use that time to +bring the bytes in. A read hint carries that knowledge to the access policy: +the backend finds where the hit keys are and calls the policy's optional +``read_hint`` hook with a :class:`HintChannel` to the remote CXL agent of +each region, and ``read_hint_end`` once the request is over. + +All of it runs on one background thread per client, in the order hints were +given, so the caller (an engine's input thread or its scheduler) never waits +for a lookup, a connection or a reply. A hint is advice: one that is late, +lost or wrong costs only the time it would have saved. +""" + +from __future__ import annotations + +import logging +import queue +import threading +from collections.abc import Callable, Sequence +from typing import Any + +from ..access_policy import AccessRange + +logger = logging.getLogger(__name__) + +#: Hints waiting for the background thread; more are dropped. +MAX_PENDING_HINTS = 1024 + + +class HintChannel: + """Messages from a policy's ``read_hint`` to the remote CXL agent of a region. + + A hint names regions this process does not hold, so the agent accepts it + only as advice: the regions must be mapped there and the message may not + prepare a write. + """ + + def __init__(self, agent: str, send: Callable[[str, dict], dict]) -> None: + self.agent = agent + self._send = send + + def request( + self, action: str, ranges: Sequence[AccessRange] = (), **payload: Any + ) -> dict: + """Send one hint message and return the region extension's reply. + + Runs on the hint thread; it waits for the reply, which keeps the + messages of one agent in order. + + Raises: + Exception: if the agent is unreachable or refused the message. + """ + message = { + **payload, + "action": action, + "ranges": [r.to_dict() for r in ranges], + "write": False, + "hint": True, + } + return self._send(self.agent, message) + + +class ReadHints: + """Runs a policy's read hints on one background thread. + + Args: + policy: The access policy; hints are skipped if it has no + ``read_hint`` hook. + locate: ``keys -> [(agent, ranges)]``: the leading hit keys grouped + per region in key order, with the agent that maps each region. + Called on the hint thread (it may ask MaruServer). + send: ``(agent, message) -> reply`` over a connection of this + process to that agent. + """ + + def __init__( + self, + policy: Any, + locate: Callable[[list[str]], list[tuple[str, list[AccessRange]]]], + send: Callable[[str, dict], dict], + ) -> None: + self._hint = getattr(policy, "read_hint", None) + self._end = getattr(policy, "read_hint_end", None) + self._locate = locate + self._send = send + self._queue: queue.Queue[tuple[str, str, list[str]] | None] = queue.Queue( + MAX_PENDING_HINTS + ) + self._thread: threading.Thread | None = None + self._lock = threading.Lock() + # Agents each hinted request reached, for its end + self._agents: dict[str, list[str]] = {} + + @property + def enabled(self) -> bool: + """Whether the policy takes read hints.""" + return self._hint is not None + + def hint(self, request_id: str, keys: list[str]) -> None: + """Queue a hint that ``request_id`` will read ``keys`` in this order.""" + if self._hint is not None and request_id and keys: + self._put(("hint", request_id, list(keys))) + + def end(self, request_id: str) -> None: + """Queue the end of ``request_id``: what was prepared for it may go.""" + if self._hint is not None and request_id: + self._put(("end", request_id, [])) + + def close(self) -> None: + """Stop the thread after the hints already queued.""" + with self._lock: + thread, self._thread = self._thread, None + if thread is not None: + self._queue.put(None) + thread.join(timeout=5.0) + + def _put(self, item: tuple[str, str, list[str]]) -> None: + with self._lock: + if self._thread is None: + self._thread = threading.Thread( + target=self._run, name="maru-read-hints", daemon=True + ) + self._thread.start() + try: + self._queue.put_nowait(item) + except queue.Full: + logger.debug("read hint dropped: %d hints are waiting", MAX_PENDING_HINTS) + + def _run(self) -> None: + while True: + item = self._queue.get() + if item is None: + return + kind, request_id, keys = item + try: + if kind == "hint": + self._give(request_id, keys) + else: + self._finish(request_id) + except Exception: + # Advice only: the load reads the bytes either way. + logger.debug( + "read hint %s for %s failed", kind, request_id, exc_info=True + ) + + def _give(self, request_id: str, keys: list[str]) -> None: + if request_id in self._agents: + return # hinted already + groups = self._locate(keys) + self._agents[request_id] = [agent for agent, _ in groups] + for agent, ranges in groups: + self._hint(HintChannel(agent, self._send), request_id, ranges) + + def _finish(self, request_id: str) -> None: + agents = self._agents.pop(request_id, None) + if not agents or self._end is None: + return + for agent in dict.fromkeys(agents): + self._end(HintChannel(agent, self._send), request_id) diff --git a/maru_handler/storage/remote_cxl/router.py b/maru_handler/storage/remote_cxl/router.py index 2da61f3e..f597ee53 100644 --- a/maru_handler/storage/remote_cxl/router.py +++ b/maru_handler/storage/remote_cxl/router.py @@ -229,6 +229,23 @@ def batch_retrieve( c = self._pick(route_key or keys[0]) return c.batch_retrieve(keys) if c.connected else [None] * len(keys) + def hint_read( + self, request_id: str, keys: list[str], route_key: str | None = None + ) -> None: + """Hint the pool the route key (or the first key) picks.""" + if keys: + self._pick(route_key or keys[0]).hint_read(request_id, keys) + + def end_read_hint(self, request_id: str) -> None: + """End the hint wherever it went (a pool that got none ignores it).""" + for c in self.clients: + c.end_read_hint(request_id) + + def set_read_request(self, request_id: str) -> None: + """Name the request this thread loads for, in every pool.""" + for c in self.clients: + c.set_read_request(request_id) + def _close_clients(self) -> None: # A pool disabled after an expiry or a MaruServer restart holds no # region but still has its threads and MaruServer connection. diff --git a/maru_remote_cxl_agent/extension.py b/maru_remote_cxl_agent/extension.py index 8632b8d9..aacce132 100644 --- a/maru_remote_cxl_agent/extension.py +++ b/maru_remote_cxl_agent/extension.py @@ -15,7 +15,10 @@ - It receives the messages policies send through their channel, after the agent checked that every range in a message belongs to a region the sending connection holds, matches its handle, fits in it, and that a read - handle does not prepare a write. + handle does not prepare a write. A read hint (``message["hint"]`` true) + is advice that reads of its ranges will follow; its regions only need to + be mapped here, since the scheduler that sends it holds none, and it + never prepares a write. The extension may ignore any hint. The agent owns mapping, NIC registration and the checks above; the extension owns whatever it does to the memory of those regions. diff --git a/maru_remote_cxl_agent/service.py b/maru_remote_cxl_agent/service.py index 01ec2386..73d3b229 100644 --- a/maru_remote_cxl_agent/service.py +++ b/maru_remote_cxl_agent/service.py @@ -275,13 +275,17 @@ def extension_request(self, conn: int, message: dict) -> dict: Every range must lie in a region this connection holds, mapped and in use, under a handle that matches it; a read handle may not + prepare a write. A read hint (``hint`` true) is advice about reads + that have not started: its regions need to be mapped and in use + here, but not held by this connection, so a process without a data + path (a serving engine's scheduler) may send one, and it may not prepare a write. The check and the extension's answer happen under the lock, so a release cannot come between them. Args: conn: Connection id. - message: ``action``, ``ranges``, ``write``, ``epoch`` and the - policy's own fields. + message: ``action``, ``ranges``, ``write``, ``hint``, ``epoch`` + and the policy's own fields. Returns: The extension's reply. @@ -302,6 +306,11 @@ def extension_request(self, conn: int, message: dict) -> dict: write = message.get("write", False) if type(write) is not bool: raise ValueError("write must be true or false") + hint = message.get("hint", False) + if type(hint) is not bool: + raise ValueError("hint must be true or false") + if hint and write: + raise MapDenied("a read hint cannot prepare a write") with self._lock: for r in ranges: if not isinstance(r, dict): @@ -313,7 +322,7 @@ def extension_request(self, conn: int, message: dict) -> dict: or not m.ready.is_set() or m.error is not None or m.retiring - or conn not in m.holders + or (conn not in m.holders and not hint) ): raise NotMapped(f"region {h.region_id} is not held here") if isinstance(h, ReadHandle): diff --git a/maru_server/replica_directory.py b/maru_server/replica_directory.py index aa3c5d02..b4597cbb 100644 --- a/maru_server/replica_directory.py +++ b/maru_server/replica_directory.py @@ -667,19 +667,34 @@ def _probe(self, p: dict) -> dict: # cannot load, and an engine still waiting for its first region has # no session to load them with. remote_cxl = valid and bool(session.region_size) + # With ``locations``, a remote CXL hit also says where it is, with a + # read handle of its region, so the caller can tell a storage policy + # which bytes a load will read before it starts. A probe takes no + # lease: the bytes may be gone by the time the load leases them. + locations = p.get("locations") is True + entries: list[dict | None] = [] + regions: dict[str, dict] = {} for key in keys: hit = bool(valid and self._select(session, key)) + entry = None if not hit and remote_cxl: pool = self._remote_cxl_lookup(namespace, key) if pool is not None: pool.last_use[key] = now hit = True + if locations: + entry = asdict(pool.replicas[key]) + regions[pool.pool_id] = read_handle(pool.handle.to_dict()) results.append(hit) + entries.append(entry) first = next(iter(self._all_pools(session)), None) if valid else None - return { + reply = { "results": results, "pool_id": first.pool_id if first is not None else None, } + if locations: + reply.update(entries=entries, regions=regions) + return reply # ------------------------------------------------------------------- read diff --git a/maru_vllm/connector.py b/maru_vllm/connector.py index 9cf67637..6eaf7a22 100644 --- a/maru_vllm/connector.py +++ b/maru_vllm/connector.py @@ -19,13 +19,15 @@ from __future__ import annotations +import contextlib +import functools import hashlib import json import re import threading import time from collections import deque -from collections.abc import Iterable +from collections.abc import Iterable, Iterator from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field, replace from typing import TYPE_CHECKING, Any @@ -361,13 +363,25 @@ def _chunk_keys(token_ids: list[int], chunk_tokens: int) -> list[str]: num_full_chunks = len(token_ids) // chunk_tokens if num_full_chunks == 0: return [] - arr = torch.tensor(token_ids[: num_full_chunks * chunk_tokens]).numpy() + # The scheduler's lookup, its read hint and the worker's load key the + # same prompt: hash it once. + return list( + _hashed_chunk_keys( + tuple(token_ids[: num_full_chunks * chunk_tokens]), chunk_tokens + ) + ) + + +@functools.lru_cache(maxsize=256) +def _hashed_chunk_keys(tokens: tuple[int, ...], chunk_tokens: int) -> tuple[str, ...]: + """Chunk keys of a prompt of whole chunks (see :func:`_chunk_keys`).""" + arr = torch.tensor(tokens).numpy() hasher = hashlib.blake2b(digest_size=8) # 8 bytes → 16 hex chars keys = [] - for i in range(num_full_chunks): + for i in range(len(tokens) // chunk_tokens): hasher.update(arr[i * chunk_tokens : (i + 1) * chunk_tokens].tobytes()) keys.append(f"kv_{hasher.hexdigest()}") - return keys + return tuple(keys) _RDMA_KNOBS = ( @@ -377,6 +391,7 @@ def _chunk_keys(token_ids: list[int], chunk_tokens: int) -> list[str]: "maru_remote_cxl_agent_key_file", "maru_access_policy", "maru_access_policy_config", + "maru_read_hint_on_arrival", ) # Smallest default RDMA staging buffer (bytes), as in maru_handler.storage.remote_cxl. _MIN_RDMA_STAGING = 1024**3 @@ -736,14 +751,10 @@ def _create_maru_handler( remote_cxl_agent_key_file=extra_config.get( "maru_remote_cxl_agent_key_file", "~/.maru/remote-cxl-agent.key" ), - # A device package's access policy runs where the bytes move: the - # worker, not the metadata-only scheduler. - access_policy=( - "" if metadata_only else extra_config.get("maru_access_policy", "") - ), - access_policy_config=( - "" if metadata_only else extra_config.get("maru_access_policy_config", "") - ), + # A device package's access policy: the worker's moves the bytes; + # the scheduler's only hears read hints (which loads come next). + access_policy=extra_config.get("maru_access_policy", ""), + access_policy_config=extra_config.get("maru_access_policy_config", ""), rdma_load_reserve=( _RDMA_ASYNC_LOAD_RESERVE if _remote_cxl_async(extra_config) and not metadata_only @@ -1169,6 +1180,56 @@ def shutdown(self): # ============================================================================ +_ARRIVAL_HINTER: Any = None + + +def _install_arrival_hint(connector: Any) -> None: + """Have vLLM's engine hand each request to ``connector`` when it arrives. + + EngineCore turns a received request into a Request on its input thread + (``preprocess_add_request``), but its scheduler looks the request up only + once the running engine step ends: under load, tens of milliseconds + later. The wrapper passes the new Request to + ``connector.request_arrived`` right away, so storage can prepare the + bytes the request will load while it waits. Installed once per process; + an engine without a Maru scheduler connector uses the latest one. + """ + global _ARRIVAL_HINTER + _ARRIVAL_HINTER = connector + try: + from vllm.v1.engine import core as engine_core + except Exception: + logger.warning("Maru: no vLLM engine core here; read hints on arrival are off") + return + cls = engine_core.EngineCore + original = getattr(cls, "preprocess_add_request", None) + if original is None: + logger.warning( + "Maru: this vLLM has no EngineCore.preprocess_add_request; " + "read hints on arrival are off" + ) + return + if getattr(original, "_maru_arrival_hint", False): + return + + @functools.wraps(original) + def preprocess_add_request(self: Any, request: Any) -> Any: + result = original(self, request) + scheduler = getattr(getattr(self, "scheduler", None), "connector", None) + target = getattr(scheduler, "_scheduler", None) + if not hasattr(target, "request_arrived"): + target = _ARRIVAL_HINTER + if target is not None: + try: + target.request_arrived(result[0]) + except Exception: + logger.debug("Maru: arrival hint not queued", exc_info=True) + return result + + preprocess_add_request._maru_arrival_hint = True # type: ignore[attr-defined] + cls.preprocess_add_request = preprocess_add_request + + class MaruSchedulerConnector: """Scheduler-side: checks chunk-by-chunk which prefix is cached in Maru.""" @@ -1194,6 +1255,17 @@ def __init__( # Lazy-init MaruHandler for exists checks self._handler = None + # Read hints (MaruHandler.hint_read): storage hears which keys a + # request will load once the scheduler finds them, or as soon as the + # request arrives with maru_read_hint_on_arrival. One hint per + # request, none after its end, and its end follows its hint. + self._hinted: set[str] = set() + self._arrived: set[str] = set() + self._hint_lock = threading.Lock() + self._handler_lock = threading.Lock() + self._arrival_pool: ThreadPoolExecutor | None = None + if bool(extra_config.get("maru_read_hint_on_arrival", False)): + _install_arrival_hint(self) # Requests that need KV loaded from maru self._requests_need_load: dict[str, tuple[Request, int]] = {} @@ -1289,18 +1361,84 @@ def _chunk_exists_key(self, base_key: str) -> str: def _ensure_handler(self): if self._handler is not None: return - if time.monotonic() < self._handler_retry_after: - return - # A scheduler never allocates or maps a data pool. - try: - self._handler = _create_maru_handler(self._extra_config, metadata_only=True) - except Exception as exc: - if self._cpu_mode and isinstance(exc, StorageError): - raise ValueError(f"CPU cache setup rejected: {exc}") from exc - self._handler_retry_after = time.monotonic() + 5.0 - logger.warning( - "Scheduler MaruHandler creation failed, backing off 5s: %s", exc + # The arrival-hint thread may get here too: create one handler. + with self._handler_lock: + if self._handler is not None: + return + if time.monotonic() < self._handler_retry_after: + return + # A scheduler never allocates or maps a data pool. + try: + self._handler = _create_maru_handler( + self._extra_config, metadata_only=True + ) + except Exception as exc: + if self._cpu_mode and isinstance(exc, StorageError): + raise ValueError(f"CPU cache setup rejected: {exc}") from exc + self._handler_retry_after = time.monotonic() + 5.0 + logger.warning( + "Scheduler MaruHandler creation failed, backing off 5s: %s", exc + ) + + def request_arrived(self, request: Request) -> None: + """vLLM received ``request``: hint its reads off the caller's thread. + + Called on the engine's input thread (see :func:`_install_arrival_hint`). + """ + with self._hint_lock: + self._arrived.add(request.request_id) + if self._arrival_pool is None: + self._arrival_pool = ThreadPoolExecutor( + max_workers=1, thread_name_prefix="maru-arrival-hint" ) + self._arrival_pool.submit(self._hint_on_arrival, request) + + def _hint_on_arrival(self, request: Request) -> None: + try: + if not self._deferred_loading or ( + self._cpu_mode and _cpu_bypass_request(request) + ): + return # the load would not wait for the request's turn + token_ids = list(request.prompt_token_ids or []) + if len(token_ids) < self._kv_chunk_tokens: + return + self._ensure_handler() + # Every chunk from the first: storage hints the leading hits. + keys = _chunk_keys(token_ids, self._kv_chunk_tokens) + self._hint(request.request_id, keys, arrival=True) + except Exception: + logger.debug("Maru: arrival read hint failed", exc_info=True) + + def _hint(self, request_id: str, keys: list[str], *, arrival: bool = False) -> None: + """Give storage the read hint of ``request_id`` once.""" + handler = self._handler + if handler is None or not keys: + return + with self._hint_lock: + if request_id in self._hinted: + return + if arrival and request_id not in self._arrived: + return # it ended before the hint was ready + self._hinted.add(request_id) + try: + handler.hint_read( + request_id, keys, **_session_route(self._cpu_mode, keys) + ) + except Exception: + logger.debug("Maru: read hint failed", exc_info=True) + + def _end_hint(self, request_id: str) -> None: + with self._hint_lock: + self._arrived.discard(request_id) + if request_id not in self._hinted: + return + self._hinted.discard(request_id) + handler = self._handler + if handler is not None: + try: + handler.end_read_hint(request_id) + except Exception: + logger.debug("Maru: read hint end failed", exc_info=True) def _count_matched_chunks(self, token_ids: list[int]) -> int: """Count how many consecutive prefix chunks are cached in Maru. @@ -1506,6 +1644,12 @@ def _match( self._last_match_result[request.request_id] = num_matched_chunks if self._cpu_mode: self._load_starts[request.request_id] = num_computed_tokens + if self._deferred_loading: + # The load waits for vLLM to admit it: tell storage what it reads. + ct = self._kv_chunk_tokens + keys = _chunk_keys(token_ids, ct) + first, last = num_computed_tokens // ct, -(-matched_tokens // ct) + self._hint(request.request_id, keys[first:last]) return new_matched, self._deferred_loading @@ -1765,6 +1909,7 @@ def request_finished( block_ids: list[int], ) -> tuple[bool, dict[str, Any] | None]: """Transfer block ownership to the worker for write-behind stores.""" + self._end_hint(request.request_id) self._async_load_issued.discard(request.request_id) if request.request_id in self._awaiting_deferred_recv: # vLLM already retains a parked request's blocks until its receive @@ -2300,7 +2445,8 @@ def start_load_kv( keys = _load_keys(chunk_keys, num_chunks, layers, self._use_layerwise) try: _t0 = time.monotonic() - infos = self._batch_retrieve_all(keys, **route) + with self._read_context(req_meta.req_id): + infos = self._batch_retrieve_all(keys, **route) if self._timing: _emit_timing( f"retrieve batch {len(keys)} keys = " @@ -2505,9 +2651,10 @@ def _deferred_load_job( keys = _load_keys(chunk_keys, num_chunks, layers, self._use_layerwise) _t0 = time.monotonic() - infos = self._batch_retrieve_all( - keys, **_session_route(self._cpu_mode, chunk_keys) - ) + with self._read_context(req_meta.req_id): + infos = self._batch_retrieve_all( + keys, **_session_route(self._cpu_mode, chunk_keys) + ) if self._timing: _emit_timing( f"deferred retrieve batch {len(keys)} keys = " @@ -3312,7 +3459,8 @@ def fail(rest: MaruReqMeta) -> None: batches += 1 try: _t0 = time.monotonic() - infos = self._batch_retrieve_all(keys, **route) + with self._read_context(req_meta.req_id): + infos = self._batch_retrieve_all(keys, **route) if self._timing: _emit_timing( f"retrieve batch {len(keys)} keys = " @@ -4143,6 +4291,22 @@ def _lease_buffers_gpu_accessible(self) -> bool: return False return bool(self._handler.storage_gpu_accessible()) + @contextlib.contextmanager + def _read_context(self, request_id: str) -> Iterator[None]: + """Tell storage which request the reads in this block load for. + + An access policy matches them with that request's read hint. + """ + mark = getattr(self._handler, "set_read_request", None) + if mark is None: + yield + return + mark(request_id) + try: + yield + finally: + mark("") + def _batch_retrieve_all( self, keys: list[str], batch_size: int = 1024, route_key: str | None = None ) -> list: diff --git a/tests/unit/test_remote_cxl_access_policy.py b/tests/unit/test_remote_cxl_access_policy.py index 765559e8..bfdf421d 100644 --- a/tests/unit/test_remote_cxl_access_policy.py +++ b/tests/unit/test_remote_cxl_access_policy.py @@ -65,7 +65,8 @@ def __init__(self): self.held = set() self.calls = [] - def read(self, channel, ranges, transfer): + def read(self, channel, ranges, transfer, request_id=""): + self.read_requests = [*getattr(self, "read_requests", []), request_id] if "read" in self.fail: raise RuntimeError("policy refused") assert channel.request("plan", ranges)["seen"] == len(ranges) diff --git a/tests/unit/test_remote_cxl_read_hints.py b/tests/unit/test_remote_cxl_read_hints.py new file mode 100644 index 00000000..537e3feb --- /dev/null +++ b/tests/unit/test_remote_cxl_read_hints.py @@ -0,0 +1,170 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Read hints: a scheduler tells the access policy which bytes a load will read. + +The directory, the remote CXL agent and both clients (a worker and its +metadata-only scheduler) are real; a fake policy and a fake extension stand in +for a device package. +""" + +import time + +import pytest + +import maru_handler.storage.remote_cxl.client as client_module +from maru_remote.region_map import read_handle +from maru_remote_cxl_agent.service import MapDenied, NotMapped +from tests.unit.test_remote_cxl_access_policy import FakeExtension, FakePolicy +from tests.unit.test_remote_cxl_storage import PAGE, World, put, read + + +class HintingPolicy(FakePolicy): + """A fake policy that also takes read hints.""" + + def __init__(self): + super().__init__() + self.hints = [] + self.ends = [] + + def read_hint(self, channel, request_id, ranges): + reply = channel.request("ahead", ranges, request=request_id) + self.hints.append((request_id, [r.location.offset for r in ranges], reply)) + + def read_hint_end(self, channel, request_id): + channel.request("forget", request=request_id) + self.ends.append(request_id) + + +class RecordingExtension(FakeExtension): + def request(self, conn, message, ranges): + self.messages = [*getattr(self, "messages", []), (conn, dict(message))] + return super().request(conn, message, ranges) + + +def _until(condition, timeout=5.0): + deadline = time.monotonic() + timeout + while not condition(): + assert time.monotonic() < deadline, "timed out" + time.sleep(0.01) + + +@pytest.fixture +def setup(monkeypatch): + ext = RecordingExtension() + w = World(extension=ext) + policy = HintingPolicy() + monkeypatch.setattr(client_module, "load_access_policy", lambda n, c: policy) + yield w, ext, policy + w.close() + + +def test_a_scheduler_hints_the_leading_hits_and_the_load_names_its_request(setup): + w, ext, policy = setup + worker = w.client("a", access_policy="fake") + for i in range(3): + assert put(worker, f"k{i}", bytes([i + 1]) * PAGE)[0] + scheduler = w.client("a", metadata_only=True, access_policy="fake") + # Keys are in prompt order: a miss ends what the load can read. + scheduler.hint_read("r1", ["k0", "k2", "missing", "k1"]) + _until(lambda: policy.hints) + request_id, offsets, reply = policy.hints[0] + assert request_id == "r1" and len(offsets) == 2 and reply == {"seen": 2} + conn, message = ext.messages[-1] + assert message["hint"] is True and message["write"] is False + assert message["action"] == "ahead" + # The scheduler holds no region: the agent took the hint as advice. + assert conn not in next(iter(w.node.service.holders().values())) + + worker.set_read_request("r1") + assert read(worker, ["k0", "k2"]) == [b"\x01" * PAGE, b"\x03" * PAGE] + worker.set_read_request("") + assert policy.read_requests[-1] == "r1" + assert read(worker, ["k1"]) == [b"\x02" * PAGE] + assert policy.read_requests[-1] == "" # the name was cleared + + scheduler.end_read_hint("r1") + _until(lambda: policy.ends) + assert policy.ends == ["r1"] + assert ext.messages[-1][1]["action"] == "forget" + + +def test_a_request_is_hinted_once_and_a_miss_hints_nothing(setup): + w, ext, policy = setup + worker = w.client("a", access_policy="fake") + assert put(worker, "k0", b"x" * PAGE)[0] + scheduler = w.client("a", metadata_only=True, access_policy="fake") + scheduler.hint_read("r1", ["k0"]) + scheduler.hint_read("r1", ["k0"]) + scheduler.hint_read("r2", ["missing", "k0"]) + scheduler.end_read_hint("r2") # never reached an agent: nothing to end + scheduler.end_read_hint("r1") + _until(lambda: policy.ends) + assert [h[0] for h in policy.hints] == ["r1"] + assert policy.ends == ["r1"] + + +def test_without_a_hint_hook_nothing_runs(monkeypatch): + w = World(extension=RecordingExtension()) + try: + policy = FakePolicy() # no read_hint + monkeypatch.setattr(client_module, "load_access_policy", lambda n, c: policy) + scheduler = w.client("a", metadata_only=True, access_policy="fake") + scheduler.hint_read("r1", ["k0"]) + assert scheduler._hints._thread is None + plain = w.client("b", metadata_only=True) + plain.hint_read("r1", ["k0"]) # no policy: ignored + assert plain._hints is None + finally: + w.close() + + +def test_the_agent_takes_a_hint_only_for_a_mapped_region_and_never_a_write(setup): + w, ext, policy = setup + worker = w.client("a", access_policy="fake") + assert put(worker, "k", b"x" * PAGE)[0] + service = w.node.service + region = next(iter(worker.regions.values())) + stranger = 10_000 # a connection that holds no region + + def message(handle, *, hint=True, write=False): + return { + "action": "ahead", + "hint": hint, + "write": write, + "epoch": service.epoch, + "ranges": [{"handle": handle, "offset": 0, "length": PAGE}], + } + + assert service.extension_request(stranger, message(read_handle(region.handle))) + with pytest.raises(NotMapped): + service.extension_request( + stranger, message(read_handle(region.handle), hint=False) + ) + with pytest.raises(MapDenied): + service.extension_request(stranger, message(region.handle, write=True)) + other = dict(region.handle, region_id=region.handle["region_id"] + 99) + with pytest.raises(NotMapped): + service.extension_request(stranger, message(read_handle(other))) + + +def test_a_probe_says_where_hits_are_only_when_asked(setup): + w, ext, policy = setup + worker = w.client("a", access_policy="fake") + assert put(worker, "k0", b"x" * PAGE)[0] + plain = w.directory.execute( + "probe", {"engine_id": "a", "namespace": "ns", "keys": ["k0"]} + )["result"] + assert "entries" not in plain + found = w.directory.execute( + "probe", + { + "engine_id": "a", + "namespace": "ns", + "keys": ["k0", "nope"], + "locations": True, + }, + )["result"] + entry, missing = found["entries"] + assert missing is None and entry["length"] == PAGE + (handle,) = found["regions"].values() + region = next(iter(worker.regions.values())) + assert handle == read_handle(region.handle) # never the owner's handle diff --git a/tests/unit/test_vllm_read_hints.py b/tests/unit/test_vllm_read_hints.py new file mode 100644 index 00000000..85dc420e --- /dev/null +++ b/tests/unit/test_vllm_read_hints.py @@ -0,0 +1,139 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The scheduler connector gives storage read hints: once per request, before +its load, and ends them when the request finishes.""" + +import time +from types import SimpleNamespace + +import maru +from maru_vllm.connector import _chunk_keys, _create_maru_handler +from tests.unit.test_vllm_remote_cxl_config import extra +from tests.unit.vllm_connector_helpers import make_scheduler + + +class Handler: + """Scheduler-side MaruHandler stand-in that records hints.""" + + def __init__(self, hits): + self.hits = hits + self.hints = [] + self.ends = [] + + def batch_exists(self, keys, route_key=None): + return [i < self.hits for i in range(len(keys))] + + def hint_read(self, request_id, keys, route_key=None): + self.hints.append((request_id, list(keys), route_key)) + + def end_read_hint(self, request_id): + self.ends.append(request_id) + + +def _request(rid, tokens=64): + return SimpleNamespace( + request_id=rid, + prompt_token_ids=list(range(tokens)), + num_tokens=tokens, + sampling_params=None, + ) + + +def _scheduler(hits, **config): + sched = make_scheduler(4, 8, extra(maru_async_load=True, **config)) + sched._handler = Handler(hits) + return sched + + +def test_the_first_hit_hints_the_keys_the_load_reads_once(): + sched = _scheduler(hits=5) + request = _request("r1") + keys = _chunk_keys(request.prompt_token_ids, 8) + matched, load_async = sched.get_num_new_matched_tokens(request, 8) + assert load_async and matched > 0 + sched.get_num_new_matched_tokens(request, 8) + (hint,) = sched._handler.hints + # Computed tokens 0-7 are chunk 0: the load reads chunks 1-4. + assert hint == ("r1", keys[1:5], keys[1]) + sched.request_finished(request, []) + assert sched._handler.ends == ["r1"] + + +def test_a_request_with_nothing_to_load_is_not_hinted(): + sched = _scheduler(hits=0) + request = _request("r1") + assert sched.get_num_new_matched_tokens(request, 0) == (0, False) + sched.request_finished(request, []) + assert sched._handler.hints == [] and sched._handler.ends == [] + + +def test_an_arrival_hint_is_sent_off_the_caller_and_never_after_the_end(): + sched = _scheduler(hits=5) + request = _request("r1") + sched.request_arrived(request) + deadline = time.monotonic() + 5 + while not sched._handler.hints: + assert time.monotonic() < deadline + time.sleep(0.01) + keys = _chunk_keys(request.prompt_token_ids, 8) + assert sched._handler.hints == [("r1", keys, keys[0])] + sched.get_num_new_matched_tokens(request, 0) + assert len(sched._handler.hints) == 1 # the first hit finds it hinted + + late = _request("r2") + sched._arrived.add("r2") + sched.request_finished(late, []) # ended before its hint was ready + sched._hint_on_arrival(late) + assert [h[0] for h in sched._handler.hints] == ["r1"] + + +def test_the_scheduler_gets_the_access_policy_too(monkeypatch): + seen = [] + + class Recorder: + def __init__(self, config): + seen.append(config) + + def connect(self): + return True + + monkeypatch.setattr(maru, "MaruHandler", Recorder) + config = extra( + maru_remote_cxl_agent_key_file="/k", + maru_access_policy="device", + maru_access_policy_config="/etc/device.toml", + ) + _create_maru_handler(config, metadata_only=True) + assert (seen[0].access_policy, seen[0].access_policy_config) == ( + "device", + "/etc/device.toml", + ) + + +def test_the_engine_hands_each_received_request_to_its_scheduler(monkeypatch): + import sys + import types + + import vllm.v1.engine as engine_pkg + + import maru_vllm.connector as connector + + class EngineCore: + def preprocess_add_request(self, request): + return (SimpleNamespace(request_id=request), 0) + + core = types.ModuleType("vllm.v1.engine.core") + core.EngineCore = EngineCore + monkeypatch.setitem(sys.modules, "vllm.v1.engine.core", core) + monkeypatch.setattr(engine_pkg, "core", core, raising=False) + monkeypatch.setattr(connector, "_ARRIVAL_HINTER", None) + sched = _scheduler(hits=5, maru_read_hint_on_arrival=True) + arrived = [] + sched.request_arrived = lambda request: arrived.append(request.request_id) + engine = EngineCore() + engine.scheduler = SimpleNamespace(connector=SimpleNamespace(_scheduler=sched)) + result = engine.preprocess_add_request("r1") + assert result[0].request_id == "r1" and arrived == ["r1"] + # Installing again does not wrap twice. + connector._install_arrival_hint(sched) + engine.preprocess_add_request("r2") + assert arrived == ["r1", "r2"] diff --git a/tests/unit/test_vllm_remote_cxl_config.py b/tests/unit/test_vllm_remote_cxl_config.py index 4abace15..afe24118 100644 --- a/tests/unit/test_vllm_remote_cxl_config.py +++ b/tests/unit/test_vllm_remote_cxl_config.py @@ -319,7 +319,9 @@ def test_access_policy_settings_need_the_remote_cxl_backend(): @pytest.mark.parametrize("metadata_only", [False, True]) -def test_the_access_policy_goes_to_the_worker_only(monkeypatch, metadata_only): +def test_the_access_policy_goes_to_the_worker_and_the_scheduler( + monkeypatch, metadata_only +): seen = [] class Handler: @@ -338,5 +340,6 @@ def connect(self): ), metadata_only=metadata_only, ) - expected = ("", "") if metadata_only else ("device", "/etc/device.toml") + # The worker's policy moves bytes; the scheduler's hears read hints. + expected = ("device", "/etc/device.toml") assert (seen[0].access_policy, seen[0].access_policy_config) == expected From f27ea29a44e7ad9db55906566f013f0c31e9799f Mon Sep 17 00:00:00 2001 From: youngrok-XCENA Date: Fri, 23 Oct 2026 16:11:03 +0900 Subject: [PATCH 6/8] fix(remote_cxl): keep read-hint ends, gate hint work on storage that takes hints - Never drop an end; only hints are capped (with a warning when dropped). - Ignore hints after close; close hint connections only after the thread ended. - Pass request_id to policy.read only for policies with read hints. - Refuse hints for regions no connection holds; document hints as read-only. - Scheduler: no hint state or thread unless storage takes hints; reuse an arrival hint's chunk keys instead of an LRU cache over whole prompts; turn maru_read_hint_on_arrival off without maru_async_load. --- docs/source/api_reference/config.md | 2 +- docs/source/design_doc/access_policies.md | 22 +++-- maru_handler/handler.py | 4 + maru_handler/storage/access_policy.py | 15 +++- maru_handler/storage/remote_cxl/client.py | 33 ++++--- maru_handler/storage/remote_cxl/hints.py | 74 ++++++++++++---- maru_handler/storage/remote_cxl/router.py | 5 ++ maru_remote_cxl_agent/extension.py | 9 +- maru_remote_cxl_agent/service.py | 5 +- maru_vllm/connector.py | 100 ++++++++++++++-------- tests/unit/test_remote_cxl_read_hints.py | 98 +++++++++++++++++++++ tests/unit/test_vllm_read_hints.py | 47 +++++++++- 12 files changed, 335 insertions(+), 79 deletions(-) diff --git a/docs/source/api_reference/config.md b/docs/source/api_reference/config.md index 6fc84de6..34df2e6e 100644 --- a/docs/source/api_reference/config.md +++ b/docs/source/api_reference/config.md @@ -170,7 +170,7 @@ apply to that backend only: | `maru_rdma_timeout_s` | `float` | `30` | Deadline of one RDMA batch | | `maru_access_policy` | `str` | empty | Name of an installed access policy (entry point in `maru.access_policies`) that decides when the backend's transfers run; see [Access policies](../design_doc/access_policies.md) | | `maru_access_policy_config` | `str` | empty | Opaque string given to that policy, typically its own configuration file | -| `maru_read_hint_on_arrival` | `bool` | `false` | Give the access policy its read hint when vLLM receives a request, not when the scheduler first looks it up; needs `maru_async_load` | +| `maru_read_hint_on_arrival` | `bool` | `false` | Give the access policy its read hint when vLLM receives a request, not when the scheduler first looks it up; needs `maru_async_load` (off with a warning otherwise) | ### Remote CXL agent (`maru-remote-cxl-agent`) diff --git a/docs/source/design_doc/access_policies.md b/docs/source/design_doc/access_policies.md index 38b9fe6a..d110c0c5 100644 --- a/docs/source/design_doc/access_policies.md +++ b/docs/source/design_doc/access_policies.md @@ -40,7 +40,9 @@ flowchart LR - **The access policy** (entry point group `maru.access_policies`, selected with `maru_access_policy`) runs in the engine's processes. In the worker it decides *when* each READ runs and whether a page may be written or reused. In the - scheduler it only hears read hints. + scheduler it only hears read hints. The scheduler and every worker build + their own instance, so the policy's constructor runs in each of those + processes. - **The region extension** (entry point group `maru.region_extensions`, selected with `--region-extension` on the agent) is the policy's counterpart next to the memory. It receives the policy's messages after the agent checked them, @@ -94,10 +96,16 @@ sequenceDiagram the keys up at maru-server (a probe that also returns where each hit is and a read handle of its region; it takes no lease) and calls the policy, in the order hints were given. -- **Advice only.** The agent accepts a hint for a region it maps even from a - process that holds no region (the scheduler has no data path), but never as a - write. A hint that is late, lost or wrong costs only the time it would have - saved: the load still leases and reads every key, and the policy's `read` - hook learns the request id it serves, so it can match the load with its hint. +- **Advice only.** The agent accepts a hint for a region it maps and some + connection holds, even from a process that holds no region (the scheduler + has no data path), but never as a write. A hint that is late, lost or wrong + costs only the time it would have saved: the load still leases and reads + every key, and the policy's `read` hook is passed `request_id=` the request + it serves, so it can match the load with its hint. +- **Inexact by design.** A hint on arrival covers the leading hits from the + first chunk, before vLLM's own prefix cache is consulted, so the load may + read fewer chunks than the hint named. While 1024 hints wait, further hints + are dropped with a warning; ends are never dropped. A process that dies + sends no end, so a policy should also expire what it prepared. - **Off without a policy.** Without an access policy that defines `read_hint`, - no thread starts and no probe is sent. + no thread starts, no probe is sent and the scheduler keeps no hint state. diff --git a/maru_handler/handler.py b/maru_handler/handler.py index ad51bbda..02fd92ad 100644 --- a/maru_handler/handler.py +++ b/maru_handler/handler.py @@ -1272,6 +1272,10 @@ def batch_exists(self, keys: list[str], route_key: str | None = None) -> list[bo ) return batch_resp.results + def read_hints_enabled(self) -> bool: + """Whether storage takes read hints (:meth:`hint_read` does anything).""" + return bool(getattr(self._cpu, "read_hints_enabled", False)) + def hint_read( self, request_id: str, keys: list[str], route_key: str | None = None ) -> None: diff --git a/maru_handler/storage/access_policy.py b/maru_handler/storage/access_policy.py index f3c3dcc2..7442e773 100644 --- a/maru_handler/storage/access_policy.py +++ b/maru_handler/storage/access_policy.py @@ -124,12 +124,12 @@ def read( channel: AccessChannel, ranges: Sequence[AccessRange], transfer: Callable[[int, int], None], - request_id: str = "", ) -> None: """Move the bytes of ``ranges`` by calling ``transfer``. - ``request_id`` names the request whose load reads them, as a read - hint named it ("" if the engine did not say). + A policy that takes read hints (:class:`ReadHintHooks`) is also + passed ``request_id=``: the request whose load reads the ranges, as + its read hint named it ("" if the engine did not say). ``transfer(start, end)`` reads ``ranges[start:end]``. The policy must call it in order, with each ``start`` equal to the previous ``end``, @@ -174,7 +174,14 @@ class ReadHintHooks(Protocol): The backend calls them on its hint thread, in the order the hints were given; a policy without them gets no hints. They may block (only the hint thread waits). An exception is logged and the hint dropped: the - load reads every byte either way. + load reads every byte either way. A policy with these hooks takes + ``request_id`` as a keyword argument of :meth:`AccessPolicy.read`. + + Hints are advice: a hint can be dropped when too many wait, and the + engine may hint chunks its own prefix cache later serves, so a load may + read less than its hint named. An end is never dropped once its hint + reached the policy, but the policy should still expire what it prepared + (a process that dies sends no end). """ def read_hint( diff --git a/maru_handler/storage/remote_cxl/client.py b/maru_handler/storage/remote_cxl/client.py index 01b99b5d..23d1abf9 100644 --- a/maru_handler/storage/remote_cxl/client.py +++ b/maru_handler/storage/remote_cxl/client.py @@ -197,6 +197,11 @@ def __init__( # ------------------------------------------------------------ read hints + @property + def read_hints_enabled(self) -> bool: + """Whether this backend's access policy takes read hints.""" + return self._hints is not None and self._hints.enabled + def hint_read( self, request_id: str, keys: list[str], route_key: str | None = None ) -> None: @@ -207,12 +212,12 @@ def hint_read( hints this does nothing. A hint takes no lease; the load still leases and reads every key. """ - if self._hints is not None and self._hints.enabled: + if self.read_hints_enabled: self._hints.hint(request_id, keys) def end_read_hint(self, request_id: str) -> None: """``request_id`` ended: the policy may drop what it prepared for it.""" - if self._hints is not None and self._hints.enabled: + if self.read_hints_enabled: self._hints.end(request_id) def set_read_request(self, request_id: str) -> None: @@ -271,7 +276,8 @@ def _hint_send(self, agent: str, message: dict) -> dict: timeout_s=max(self.config.rdma_timeout_s, REPLY_TIMEOUT_MIN_S), key=self._hint_key, ) - client.hello() # connects if needed; the epoch below is its agent's + if not client.epoch: + client.hello() # connects; a reconnect later renews the epoch if not client.region_extension: raise StorageError(f"remote CXL agent {agent} has no region extension") return client.extension({**message, "epoch": client.epoch}) @@ -1487,12 +1493,15 @@ def transfer(start: int, end: int) -> None: raise cursor = end + # Only a policy that takes read hints is told which request reads. + named = ( + {"request_id": getattr(self._reading, "request_id", "")} + if self.read_hints_enabled + else {} + ) try: self._policy.read( - _RegionChannel(self._data, access), - ranges, - transfer, - getattr(self._reading, "request_id", ""), + _RegionChannel(self._data, access), ranges, transfer, **named ) finally: done = True @@ -1559,11 +1568,11 @@ def close(self) -> None: for thread in self._threads: thread.join() self._threads = [] - if self._hints is not None: - self._hints.close() - for agent in self._hint_agents.values(): - agent.close() - self._hint_agents.clear() + # The hint thread owns its agent connections until it ended. + if self._hints is None or self._hints.close(): + for agent in self._hint_agents.values(): + agent.close() + self._hint_agents.clear() with self._lock: # Transfers run without the lock: let those under way end (each # within its timeout) before the regions and the data path go. diff --git a/maru_handler/storage/remote_cxl/hints.py b/maru_handler/storage/remote_cxl/hints.py index 52556d30..4a5a99b5 100644 --- a/maru_handler/storage/remote_cxl/hints.py +++ b/maru_handler/storage/remote_cxl/hints.py @@ -29,8 +29,10 @@ logger = logging.getLogger(__name__) -#: Hints waiting for the background thread; more are dropped. +#: Hints waiting for the background thread; more are dropped (ends are not). MAX_PENDING_HINTS = 1024 +#: A dropped hint is logged as a warning once per this many drops. +DROP_WARNING_EVERY = 1024 class HintChannel: @@ -89,12 +91,16 @@ def __init__( self._end = getattr(policy, "read_hint_end", None) self._locate = locate self._send = send - self._queue: queue.Queue[tuple[str, str, list[str]] | None] = queue.Queue( - MAX_PENDING_HINTS + # Unbounded so an end is never lost; _pending bounds the hints in it. + self._queue: queue.SimpleQueue[tuple[str, str, list[str]] | None] = ( + queue.SimpleQueue() ) self._thread: threading.Thread | None = None self._lock = threading.Lock() - # Agents each hinted request reached, for its end + self._pending = 0 + self._dropped = 0 + self._closed = False + # Agents each hinted request reached, for its end (hint thread only) self._agents: dict[str, list[str]] = {} @property @@ -103,34 +109,67 @@ def enabled(self) -> bool: return self._hint is not None def hint(self, request_id: str, keys: list[str]) -> None: - """Queue a hint that ``request_id`` will read ``keys`` in this order.""" - if self._hint is not None and request_id and keys: - self._put(("hint", request_id, list(keys))) + """Queue a hint that ``request_id`` will read ``keys`` in this order. + + Dropped, with a warning now and then, while + :data:`MAX_PENDING_HINTS` hints wait. + """ + if self._hint is None or not request_id or not keys: + return + with self._lock: + if self._pending >= MAX_PENDING_HINTS: + self._dropped += 1 + dropped = self._dropped + else: + self._pending += 1 + dropped = 0 + if dropped: + if dropped % DROP_WARNING_EVERY == 1: + logger.warning( + "read hints dropped (%d so far): %d are waiting for the policy", + dropped, + MAX_PENDING_HINTS, + ) + return + self._put(("hint", request_id, list(keys))) def end(self, request_id: str) -> None: """Queue the end of ``request_id``: what was prepared for it may go.""" if self._hint is not None and request_id: self._put(("end", request_id, [])) - def close(self) -> None: - """Stop the thread after the hints already queued.""" + def close(self, timeout_s: float = 5.0) -> bool: + """Stop the thread after the hints already queued; hints after this + are ignored. + + Returns: + True if the thread ended (or never started) within + ``timeout_s``; the caller may then close what it sends on. + """ with self._lock: + self._closed = True thread, self._thread = self._thread, None - if thread is not None: - self._queue.put(None) - thread.join(timeout=5.0) + if thread is None: + return True + self._queue.put(None) + thread.join(timeout=timeout_s) + if thread.is_alive(): + logger.warning("read hint thread still busy after %.0f s", timeout_s) + return False + return True def _put(self, item: tuple[str, str, list[str]]) -> None: with self._lock: + if self._closed: + if item[0] == "hint": + self._pending -= 1 + return if self._thread is None: self._thread = threading.Thread( target=self._run, name="maru-read-hints", daemon=True ) self._thread.start() - try: - self._queue.put_nowait(item) - except queue.Full: - logger.debug("read hint dropped: %d hints are waiting", MAX_PENDING_HINTS) + self._queue.put(item) def _run(self) -> None: while True: @@ -138,6 +177,9 @@ def _run(self) -> None: if item is None: return kind, request_id, keys = item + if kind == "hint": + with self._lock: + self._pending -= 1 try: if kind == "hint": self._give(request_id, keys) diff --git a/maru_handler/storage/remote_cxl/router.py b/maru_handler/storage/remote_cxl/router.py index f597ee53..661cb2ce 100644 --- a/maru_handler/storage/remote_cxl/router.py +++ b/maru_handler/storage/remote_cxl/router.py @@ -229,6 +229,11 @@ def batch_retrieve( c = self._pick(route_key or keys[0]) return c.batch_retrieve(keys) if c.connected else [None] * len(keys) + @property + def read_hints_enabled(self) -> bool: + """Whether any pool's access policy takes read hints.""" + return any(c.read_hints_enabled for c in self.clients) + def hint_read( self, request_id: str, keys: list[str], route_key: str | None = None ) -> None: diff --git a/maru_remote_cxl_agent/extension.py b/maru_remote_cxl_agent/extension.py index aacce132..261bd4dd 100644 --- a/maru_remote_cxl_agent/extension.py +++ b/maru_remote_cxl_agent/extension.py @@ -17,8 +17,13 @@ sending connection holds, matches its handle, fits in it, and that a read handle does not prepare a write. A read hint (``message["hint"]`` true) is advice that reads of its ranges will follow; its regions only need to - be mapped here, since the scheduler that sends it holds none, and it - never prepares a write. The extension may ignore any hint. + be mapped and held here, by any connection, since the scheduler that + sends it holds none, and it never prepares a write. The agent cannot + read ``action``, so the extension must treat every message with ``hint`` + true as read-only, whatever its action. A hint's ``conn`` holds no + region and may close without telling: state kept for a hint ends with + the hint's own end message or the extension's expiry, never with a + release of that connection. The extension may ignore any hint. The agent owns mapping, NIC registration and the checks above; the extension owns whatever it does to the memory of those regions. diff --git a/maru_remote_cxl_agent/service.py b/maru_remote_cxl_agent/service.py index 73d3b229..c81b9b3a 100644 --- a/maru_remote_cxl_agent/service.py +++ b/maru_remote_cxl_agent/service.py @@ -276,8 +276,8 @@ def extension_request(self, conn: int, message: dict) -> dict: Every range must lie in a region this connection holds, mapped and in use, under a handle that matches it; a read handle may not prepare a write. A read hint (``hint`` true) is advice about reads - that have not started: its regions need to be mapped and in use - here, but not held by this connection, so a process without a data + that have not started: its regions need to be mapped here and held + by some connection, not necessarily this one, so a process without a data path (a serving engine's scheduler) may send one, and it may not prepare a write. The check and the extension's answer happen under the lock, so a release cannot come between them. @@ -322,6 +322,7 @@ def extension_request(self, conn: int, message: dict) -> dict: or not m.ready.is_set() or m.error is not None or m.retiring + or not m.holders or (conn not in m.holders and not hint) ): raise NotMapped(f"region {h.region_id} is not held here") diff --git a/maru_vllm/connector.py b/maru_vllm/connector.py index 6eaf7a22..0e5ae154 100644 --- a/maru_vllm/connector.py +++ b/maru_vllm/connector.py @@ -363,25 +363,13 @@ def _chunk_keys(token_ids: list[int], chunk_tokens: int) -> list[str]: num_full_chunks = len(token_ids) // chunk_tokens if num_full_chunks == 0: return [] - # The scheduler's lookup, its read hint and the worker's load key the - # same prompt: hash it once. - return list( - _hashed_chunk_keys( - tuple(token_ids[: num_full_chunks * chunk_tokens]), chunk_tokens - ) - ) - - -@functools.lru_cache(maxsize=256) -def _hashed_chunk_keys(tokens: tuple[int, ...], chunk_tokens: int) -> tuple[str, ...]: - """Chunk keys of a prompt of whole chunks (see :func:`_chunk_keys`).""" - arr = torch.tensor(tokens).numpy() + arr = torch.tensor(token_ids[: num_full_chunks * chunk_tokens]).numpy() hasher = hashlib.blake2b(digest_size=8) # 8 bytes → 16 hex chars keys = [] - for i in range(len(tokens) // chunk_tokens): + for i in range(num_full_chunks): hasher.update(arr[i * chunk_tokens : (i + 1) * chunk_tokens].tobytes()) keys.append(f"kv_{hasher.hexdigest()}") - return tuple(keys) + return keys _RDMA_KNOBS = ( @@ -1258,14 +1246,17 @@ def __init__( # Read hints (MaruHandler.hint_read): storage hears which keys a # request will load once the scheduler finds them, or as soon as the # request arrives with maru_read_hint_on_arrival. One hint per - # request, none after its end, and its end follows its hint. + # request, none after its end, and its end follows its hint. All of + # it runs only for storage that takes hints (_read_hints_on). self._hinted: set[str] = set() self._arrived: set[str] = set() + # Chunk keys an arrival hint computed, for the request's lookup + self._arrival_keys: dict[str, list[str]] = {} self._hint_lock = threading.Lock() self._handler_lock = threading.Lock() self._arrival_pool: ThreadPoolExecutor | None = None - if bool(extra_config.get("maru_read_hint_on_arrival", False)): - _install_arrival_hint(self) + self._hints_handler: Any = None + self._hints_enabled = False # Requests that need KV loaded from maru self._requests_need_load: dict[str, tuple[Request, int]] = {} @@ -1277,6 +1268,14 @@ def __init__( # loading — get_num_new_matched_tokens returns async_load=True. self._deferred_loading = bool(_get_knob(extra_config, "maru_async_load")) _validate_deferred_loading(self._deferred_loading) + if bool(extra_config.get("maru_read_hint_on_arrival", False)): + if self._deferred_loading: + _install_arrival_hint(self) + else: + logger.warning( + "Maru: maru_read_hint_on_arrival requires " + "maru_async_load=true; disabling it" + ) self._write_behind = bool(_get_knob(extra_config, "maru_async_store")) self._use_layerwise = bool(extra_config.get("maru_use_layerwise", False)) overlap_requested = bool( @@ -1385,30 +1384,44 @@ def request_arrived(self, request: Request) -> None: Called on the engine's input thread (see :func:`_install_arrival_hint`). """ + if self._handler is not None and not self._read_hints_on(): + return + if self._cpu_mode and _cpu_bypass_request(request): + return # the load would not wait for the request's turn with self._hint_lock: self._arrived.add(request.request_id) - if self._arrival_pool is None: - self._arrival_pool = ThreadPoolExecutor( - max_workers=1, thread_name_prefix="maru-arrival-hint" - ) - self._arrival_pool.submit(self._hint_on_arrival, request) + if self._arrival_pool is None: + self._arrival_pool = ThreadPoolExecutor( + max_workers=1, thread_name_prefix="maru-arrival-hint" + ) + pool = self._arrival_pool + pool.submit(self._hint_on_arrival, request) def _hint_on_arrival(self, request: Request) -> None: try: - if not self._deferred_loading or ( - self._cpu_mode and _cpu_bypass_request(request) - ): - return # the load would not wait for the request's turn token_ids = list(request.prompt_token_ids or []) if len(token_ids) < self._kv_chunk_tokens: return self._ensure_handler() + if not self._read_hints_on(): + return # Every chunk from the first: storage hints the leading hits. keys = _chunk_keys(token_ids, self._kv_chunk_tokens) self._hint(request.request_id, keys, arrival=True) except Exception: logger.debug("Maru: arrival read hint failed", exc_info=True) + def _read_hints_on(self) -> bool: + """Whether storage takes read hints (asked once per handler).""" + handler = self._handler + if handler is None or not self._deferred_loading: + return False + if self._hints_handler is not handler: + enabled = getattr(handler, "read_hints_enabled", None) + self._hints_enabled = bool(enabled()) if callable(enabled) else False + self._hints_handler = handler + return self._hints_enabled + def _hint(self, request_id: str, keys: list[str], *, arrival: bool = False) -> None: """Give storage the read hint of ``request_id`` once.""" handler = self._handler @@ -1417,8 +1430,10 @@ def _hint(self, request_id: str, keys: list[str], *, arrival: bool = False) -> N with self._hint_lock: if request_id in self._hinted: return - if arrival and request_id not in self._arrived: - return # it ended before the hint was ready + if arrival: + if request_id not in self._arrived: + return # it ended before the hint was ready + self._arrival_keys[request_id] = keys self._hinted.add(request_id) try: handler.hint_read( @@ -1430,6 +1445,7 @@ def _hint(self, request_id: str, keys: list[str], *, arrival: bool = False) -> N def _end_hint(self, request_id: str) -> None: with self._hint_lock: self._arrived.discard(request_id) + self._arrival_keys.pop(request_id, None) if request_id not in self._hinted: return self._hinted.discard(request_id) @@ -1440,15 +1456,22 @@ def _end_hint(self, request_id: str) -> None: except Exception: logger.debug("Maru: read hint end failed", exc_info=True) - def _count_matched_chunks(self, token_ids: list[int]) -> int: + def _count_matched_chunks( + self, token_ids: list[int], keys: list[str] | None = None + ) -> int: """Count how many consecutive prefix chunks are cached in Maru. Uses batch_exists for efficiency: single RPC call checks all chunks. + Args: + token_ids: The prompt. + keys: Its chunk keys, if already computed. + Returns: Number of consecutive cached chunks from the beginning. """ - keys = _chunk_keys(token_ids, self._kv_chunk_tokens) + if keys is None: + keys = _chunk_keys(token_ids, self._kv_chunk_tokens) if not keys: return 0 @@ -1606,7 +1629,15 @@ def _match( return 0, False _t0 = time.monotonic() - num_matched_chunks = self._count_matched_chunks(token_ids) + with self._hint_lock: + keys = self._arrival_keys.get(request.request_id) + if keys is None and self._read_hints_on(): + keys = _chunk_keys(token_ids, self._kv_chunk_tokens) # once, for both + num_matched_chunks = ( + self._count_matched_chunks(token_ids) + if keys is None + else self._count_matched_chunks(token_ids, keys) + ) if self._timing: _emit_timing( f"get_num_new_matched (incl _chunk_keys) {len(token_ids)} tok = " @@ -1644,10 +1675,11 @@ def _match( self._last_match_result[request.request_id] = num_matched_chunks if self._cpu_mode: self._load_starts[request.request_id] = num_computed_tokens - if self._deferred_loading: + if self._read_hints_on(): # The load waits for vLLM to admit it: tell storage what it reads. ct = self._kv_chunk_tokens - keys = _chunk_keys(token_ids, ct) + if keys is None: + keys = _chunk_keys(token_ids, ct) first, last = num_computed_tokens // ct, -(-matched_tokens // ct) self._hint(request.request_id, keys[first:last]) diff --git a/tests/unit/test_remote_cxl_read_hints.py b/tests/unit/test_remote_cxl_read_hints.py index 537e3feb..7c7c3230 100644 --- a/tests/unit/test_remote_cxl_read_hints.py +++ b/tests/unit/test_remote_cxl_read_hints.py @@ -6,11 +6,13 @@ for a device package. """ +import threading import time import pytest import maru_handler.storage.remote_cxl.client as client_module +import maru_handler.storage.remote_cxl.hints as hints_module from maru_remote.region_map import read_handle from maru_remote_cxl_agent.service import MapDenied, NotMapped from tests.unit.test_remote_cxl_access_policy import FakeExtension, FakePolicy @@ -168,3 +170,99 @@ def test_a_probe_says_where_hits_are_only_when_asked(setup): (handle,) = found["regions"].values() region = next(iter(worker.regions.values())) assert handle == read_handle(region.handle) # never the owner's handle + + +def test_a_hint_needs_a_region_some_connection_holds(setup): + w, ext, policy = setup + worker = w.client("a", access_policy="fake") + assert put(worker, "k", b"x" * PAGE)[0] + service = w.node.service + region = next(iter(worker.regions.values())) + m = service._maps[region.handle["region_id"]] + message = { + "action": "ahead", + "hint": True, + "write": False, + "epoch": service.epoch, + "ranges": [{"handle": read_handle(region.handle), "offset": 0, "length": 1}], + } + held = set(m.holders) + m.holders.clear() # the last holder left; the sweep has not released it yet + try: + with pytest.raises(NotMapped): + service.extension_request(10_000, message) + finally: + m.holders.update(held) + + +def test_a_policy_without_hint_hooks_reads_with_three_arguments(monkeypatch): + class ThreeArgumentPolicy(FakePolicy): + def read(self, channel, ranges, transfer): + super().read(channel, ranges, transfer) + + w = World(extension=RecordingExtension()) + try: + policy = ThreeArgumentPolicy() + monkeypatch.setattr(client_module, "load_access_policy", lambda n, c: policy) + worker = w.client("a", access_policy="fake") + assert put(worker, "k", b"\x07" * PAGE)[0] + worker.set_read_request("r1") + assert read(worker, ["k"]) == [b"\x07" * PAGE] + assert not worker.read_hints_enabled + finally: + w.close() + + +class _BlockedPolicy: + """A hinting policy whose first hint blocks until released.""" + + def __init__(self): + self.release = threading.Event() + self.hints = [] + self.ends = [] + + def read_hint(self, channel, request_id, ranges): + self.release.wait(5) + self.hints.append(request_id) + + def read_hint_end(self, channel, request_id): + self.ends.append(request_id) + + +def test_ends_are_kept_when_too_many_hints_wait(monkeypatch): + monkeypatch.setattr(hints_module, "MAX_PENDING_HINTS", 2) + warnings = [] + monkeypatch.setattr( + hints_module.logger, "warning", lambda msg, *a: warnings.append(msg % a) + ) + policy = _BlockedPolicy() + hints = hints_module.ReadHints( + policy, lambda keys: [("agent", [])], lambda a, m: {} + ) + hints.hint("r0", ["k"]) # the thread takes it and blocks + _until(lambda: hints._pending == 0) + hints.hint("r1", ["k"]) + hints.hint("r2", ["k"]) + hints.hint("r3", ["k"]) # dropped + assert len(warnings) == 1 and "read hints dropped" in warnings[0] + for rid in ("r0", "r1", "r2", "r3"): + hints.end(rid) + policy.release.set() + _until(lambda: len(policy.ends) == 3) + assert policy.hints == ["r0", "r1", "r2"] + assert policy.ends == ["r0", "r1", "r2"] # r3 never reached the policy + assert hints.close() + + +def test_hints_after_close_are_ignored(): + policy = _BlockedPolicy() + policy.release.set() + hints = hints_module.ReadHints( + policy, lambda keys: [("agent", [])], lambda a, m: {} + ) + hints.hint("r1", ["k"]) + _until(lambda: policy.hints) + assert hints.close() + hints.hint("r2", ["k"]) + hints.end("r1") + assert hints._thread is None and policy.hints == ["r1"] and not policy.ends diff --git a/tests/unit/test_vllm_read_hints.py b/tests/unit/test_vllm_read_hints.py index 85dc420e..26394b1b 100644 --- a/tests/unit/test_vllm_read_hints.py +++ b/tests/unit/test_vllm_read_hints.py @@ -14,10 +14,14 @@ class Handler: """Scheduler-side MaruHandler stand-in that records hints.""" - def __init__(self, hits): + def __init__(self, hits, hints_enabled=True): self.hits = hits self.hints = [] self.ends = [] + self.enabled = hints_enabled + + def read_hints_enabled(self): + return self.enabled def batch_exists(self, keys, route_key=None): return [i < self.hits for i in range(len(keys))] @@ -137,3 +141,44 @@ def preprocess_add_request(self, request): connector._install_arrival_hint(sched) engine.preprocess_add_request("r2") assert arrived == ["r1", "r2"] + + +def test_storage_without_hints_costs_the_scheduler_nothing(): + sched = make_scheduler(4, 8, extra(maru_async_load=True)) + sched._handler = Handler(hits=5, hints_enabled=False) + request = _request("r1") + matched, load_async = sched.get_num_new_matched_tokens(request, 8) + assert load_async and matched > 0 + sched.request_arrived(request) + sched.request_finished(request, []) + assert sched._handler.hints == [] and sched._handler.ends == [] + assert not sched._hinted and not sched._arrived and sched._arrival_pool is None + + +def test_the_lookup_reuses_the_keys_an_arrival_hint_computed(): + sched = _scheduler(hits=5) + request = _request("r1") + sched.request_arrived(request) + deadline = time.monotonic() + 5 + while not sched._handler.hints: + assert time.monotonic() < deadline + time.sleep(0.01) + seen = [] + sched._count_matched_chunks = lambda tokens, keys=None: seen.append(keys) or 5 + sched.get_num_new_matched_tokens(request, 8) + assert seen == [sched._handler.hints[0][1]] + sched.request_finished(request, []) + assert not sched._arrival_keys + + +def test_arrival_hints_need_asynchronous_loading(monkeypatch): + import maru_vllm.connector as connector + + installed, warnings = [], [] + monkeypatch.setattr(connector, "_install_arrival_hint", installed.append) + monkeypatch.setattr( + connector.logger, "warning", lambda msg, *a: warnings.append(msg % a) + ) + make_scheduler(4, 8, extra(maru_read_hint_on_arrival=True)) + assert installed == [] + assert any("requires maru_async_load" in w for w in warnings) From 92d28dc6a87bfb2a6209274653061e15a2f807ed Mon Sep 17 00:00:00 2001 From: youngrok-XCENA Date: Tue, 27 Oct 2026 12:00:44 +0900 Subject: [PATCH 7/8] docs(remote_cxl): describe access policies with the CXL-SSD case and the Maru/package boundary Name the contracts at the boundary (Maru to policy, policy to Maru through the channel, agent to extension), draw who calls whom, use prefetch for what the device does, and state that the need is wider than remote_cxl. --- docs/source/design_doc/access_policies.md | 172 +++++++++++++--------- 1 file changed, 104 insertions(+), 68 deletions(-) diff --git a/docs/source/design_doc/access_policies.md b/docs/source/design_doc/access_policies.md index d110c0c5..af8c4471 100644 --- a/docs/source/design_doc/access_policies.md +++ b/docs/source/design_doc/access_policies.md @@ -1,89 +1,114 @@ # Access Policies and Read Hints -Some CXL memory answers every address but not at the same speed: a device may -keep a small fast cache in front of larger, slower media, so bytes have to be -brought into the cache before a read and written back after a write. Maru keeps -no knowledge of such devices. The `remote_cxl` backend instead offers two -extension points that a separately installed package fills in, and a read hint -that tells that package early which bytes a load will read. +Some CXL memory makes a page fast to reach only after the device has prepared +it. A CXL-SSD, for example, keeps a DRAM cache in front of an SSD: an RDMA READ +runs at DRAM speed only on a page already prefetched from the SSD into DRAM, +and an RDMA WRITE needs its page held in DRAM and written back to the SSD +before the page holds new data. Only software next to the device can prepare a +page, and it needs to know early which pages are coming. + +Maru keeps no knowledge of such devices. The `remote_cxl` backend instead +offers two extension points that a separately installed package implements: + +- an **access policy** on the GPU node, which decides when each RDMA transfer + runs, and +- a **region extension** in the memory node's remote CXL agent, which drives + the device. + +A **read hint** tells the policy, before the load starts, which pages a +request will read, so the device can prefetch them while the request waits. ## Who owns what +Grey boxes are Maru; orange boxes are the installed package. Every message +between the package's two halves passes through Maru. + ```mermaid -flowchart LR +flowchart TB subgraph gpu["GPU node"] - sched["vLLM scheduler
+ Maru connector"] - worker["vLLM worker
+ Maru connector"] - polS["access policy
(read hints)"] - polW["access policy
(read, write)"] - sched --- polS - worker --- polW + direction LR + pol["access policy
(maru.access_policies)"] + mc["Maru in vLLM
(scheduler, workers)"] end subgraph mem["memory node"] - server["maru-server
(keys, leases)"] - agent["remote CXL agent
(map, register, check)"] - ext["region extension"] - dev[("CXL memory")] - agent --- ext - ext -. "device control" .-> dev + direction LR + ms["maru-server"] + ag["remote CXL agent"] + ext["region extension
(maru.region_extensions)"] + dev[("CXL-SSD
DRAM cache + SSD")] end - sched -- "lookup" --> server - worker -- "lease" --> server - polS -- "read hint" --> agent - polW -- "policy messages" --> agent - worker == "RDMA READ / WRITE" ==> dev + mc -->|"① pages to be read or written
(read hint, then transfer)"| pol + pol -->|"② when to transfer,
messages for the extension"| mc + mc -->|"lookup, lease"| ms + mc -->|"carries messages"| ag + ag -->|"③ after checking the pages"| ext + ext -.->|"prefetch, hold,
write back"| dev + mc ==>|"RDMA READ / WRITE"| dev + classDef maru fill:#e8e8e8,stroke:#555,color:#111 + classDef pkg fill:#fbe3cc,stroke:#c06000,color:#111 + class mc,ms,ag maru + class pol,ext pkg ``` - **Maru** decides which keys hit, which pages hold them, which bytes move and whether a key is published. It maps and registers regions on the memory node and checks every policy message against the regions it maps. -- **The access policy** (entry point group `maru.access_policies`, selected with - `maru_access_policy`) runs in the engine's processes. In the worker it decides - *when* each READ runs and whether a page may be written or reused. In the - scheduler it only hears read hints. The scheduler and every worker build - their own instance, so the policy's constructor runs in each of those - processes. -- **The region extension** (entry point group `maru.region_extensions`, selected - with `--region-extension` on the agent) is the policy's counterpart next to - the memory. It receives the policy's messages after the agent checked them, - and whatever it does to the device is its own business. +- **① Maru → access policy.** Maru tells the policy which pages each transfer + touches, and, with read hints, which pages a request will read before the + load. The policy sees pages and regions, never keys. The scheduler and every + worker build their own instance (entry point group `maru.access_policies`, + selected with `maru_access_policy`). +- **② Access policy → Maru.** The policy decides when each transfer runs: it + can delay a READ until its page is prefetched and a WRITE until its pages are + held, but cannot skip or reorder a READ. It also decides when a page given + back may hold new data. It sends messages to its counterpart through a + channel; Maru carries them to the agent over an authenticated connection + without reading their content. +- **③ Remote CXL agent → region extension.** The agent hands a message to the + extension (entry point group `maru.region_extensions`, selected with + `--region-extension`) only after checking that every page lies inside a + region it maps, under that region's own capability, and that a read-only + capability does not prepare a write. The extension may also refuse to map a + region, and keep a region from going back while the device still works on it. Installing a package selects nothing; only the configured names are loaded. -The hook contracts are documented in `maru_handler/storage/access_policy.py`, +The hook contracts, locks and failure outcomes are documented in +`maru_handler/storage/access_policy.py`, `maru_handler/storage/remote_cxl/hints.py` and `maru_remote_cxl_agent/extension.py`. ## Why read hints -A policy that is told about a read only when the worker starts it has no time -to prepare: the first bytes are not in the fast cache yet, and if the device -fills the cache about as fast as the network reads it, the READs follow the -fill chunk by chunk for the whole load. +A policy that learns about a read only when the worker starts the load has no +time to prepare. If the device prefetches about as fast as the network reads, +the READs trail the prefetch page by page for the whole load. -The engine knows much earlier which keys a request will load. A request that +The engine knows much earlier which pages a request will load. A request that arrives while an engine step runs waits for that step to end before the scheduler picks it, and the scheduler then admits the load before the worker -starts it. A read hint hands that knowledge to the policy at the earliest of -these points, so the device can fill while the request waits. +starts it. A read hint hands that knowledge to the policy at one of these +points, so the device can prefetch while the request waits. ```mermaid sequenceDiagram - participant E as vLLM engine
(input thread) - participant S as Maru connector
(scheduler) - participant H as hint thread - participant A as remote CXL agent
+ region extension - participant W as Maru connector
(worker) - E->>S: request received (maru_read_hint_on_arrival) - S->>H: hint_read(request, keys) + participant E as vLLM engine + participant S as Maru (scheduler) + participant P as access policy
(one per process) + participant A as agent + region extension + participant W as Maru (worker) + participant D as CXL-SSD + E->>S: request received (opt-in hint on arrival) + S->>S: probe locations at maru-server (no lease) + S->>P: read_hint(request, pages) + P->>A: message through Maru's channel + A->>D: prefetch from SSD into DRAM Note over E: request waits for the running step - H->>H: probe with locations: leading hits per region - H->>A: read_hint(request, ranges) - Note over A: device prepares the bytes - S->>S: scheduler picks the request (first lookup: hint once) - W->>W: lease the keys - W->>A: policy.read(..., request): READs run as bytes are ready - S->>H: end_read_hint(request) when the request finishes - H->>A: read_hint_end(request) + E->>W: load the request's hits + W->>P: read(pages, transfer, request) + P->>W: transfer pages as they are ready + W->>D: RDMA READ + E->>S: request finished + S->>P: read_hint_end(request) ``` - **Where hints come from.** By default the scheduler connector hints a request @@ -91,21 +116,32 @@ sequenceDiagram `maru_read_hint_on_arrival` it hints as soon as vLLM's engine receives the request, by wrapping `EngineCore.preprocess_add_request`; the first lookup then finds the request hinted. A request is hinted at most once, never after - it finished, and its end follows its hint. + it finished, and its end follows its hint. Hints need `maru_async_load`. - **No waiting.** The caller only queues the hint. One thread per backend looks - the keys up at maru-server (a probe that also returns where each hit is and a - read handle of its region; it takes no lease) and calls the policy, in the - order hints were given. -- **Advice only.** The agent accepts a hint for a region it maps and some - connection holds, even from a process that holds no region (the scheduler - has no data path), but never as a write. A hint that is late, lost or wrong - costs only the time it would have saved: the load still leases and reads - every key, and the policy's `read` hook is passed `request_id=` the request - it serves, so it can match the load with its hint. + the keys up at maru-server (a probe that also returns where each remote hit + is and a read handle of its region; it takes no lease) and calls the policy, + in the order hints were given. +- **Advice only.** The agent accepts a hint from any connection for a region it + maps and some connection holds (the scheduler holds none), but never as a + write, so the extension must treat every hint as read-only. A hint that is + late, lost or wrong costs only the time it would have saved: the load still + leases and reads every key, and the policy's `read` hook is passed + `request_id=`, so it can match the load with its hint. - **Inexact by design.** A hint on arrival covers the leading hits from the first chunk, before vLLM's own prefix cache is consulted, so the load may read fewer chunks than the hint named. While 1024 hints wait, further hints are dropped with a warning; ends are never dropped. A process that dies sends no end, so a policy should also expire what it prepared. - **Off without a policy.** Without an access policy that defines `read_hint`, - no thread starts, no probe is sent and the scheduler keeps no hint state. + no hint is sent, no probe runs and the scheduler keeps no hint state. + +## Scope + +The need is not specific to `remote_cxl`: a CXL-SSD attached to the GPU node's +own host has the same problem. `remote_cxl` comes first because there the GPU +node cannot reach the device software at all, and Maru's storage client itself +issues every RDMA transfer, so it can hold one back. On the local path the +engine's connector copies the bytes, and the handler plugins +(`maru.handler_plugins`) see a batch only after its lookup. Read hints are sent +only by the vLLM connector; LMCache and SGLang on `remote_cxl` get the policy's +timing but no hints. From bf9bcf2729d9324012c644adf48b1de5de00d043 Mon Sep 17 00:00:00 2001 From: youngrok-XCENA Date: Tue, 27 Oct 2026 12:06:10 +0900 Subject: [PATCH 8/8] test(vllm): skip the read-hint tests where torch or vLLM is missing --- tests/unit/test_vllm_read_hints.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/unit/test_vllm_read_hints.py b/tests/unit/test_vllm_read_hints.py index 26394b1b..afaebcd9 100644 --- a/tests/unit/test_vllm_read_hints.py +++ b/tests/unit/test_vllm_read_hints.py @@ -5,6 +5,11 @@ import time from types import SimpleNamespace +import pytest + +pytest.importorskip("torch") +pytest.importorskip("vllm") + import maru from maru_vllm.connector import _chunk_keys, _create_maru_handler from tests.unit.test_vllm_remote_cxl_config import extra