Repository navigation
Expand file tree
/
Copy pathplugin.py
More file actions
267 lines (221 loc) · 11.4 KB
/
Copy pathplugin.py
File metadata and controls
267 lines (221 loc) · 11.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 XCENA Inc.
"""Out-of-tree (OOT) plugin support for MaruHandler.
Maru core stays vendor-neutral: hardware- or vendor-specific behaviour
(e.g. a CXL device's prefetch/pin hints) lives in a *separate* pip package
that registers itself under the ``maru.handler_plugins`` entry-point group.
MaruHandler discovers those packages at construction time and calls them at a
few well-defined seams — it never imports them directly.
The design mirrors vLLM's plugin system (soft-fail + name allowlist) rather
than PyTorch's device-backend autoload (hard-fail at ``import torch``): a Maru
plugin is an *optional optimization*. A missing or broken plugin must never
break KV-cache operation, so a load/hook failure is logged and skipped, never
raised.
Registering a plugin (in the plugin package's ``pyproject.toml``)::
[project.entry-points."maru.handler_plugins"]
my_plugin = "my_pkg.plugin:MyPlugin"
The entry-point value is a zero-argument callable (a class or factory
function) returning an object that implements any subset of
:class:`MaruHandlerPlugin`.
Selecting plugins at runtime::
MARU_PLUGINS=my_plugin,other # only these names load; unset → all load
"""
from __future__ import annotations
import importlib.metadata
import logging
import os
from typing import TYPE_CHECKING, Protocol, runtime_checkable
if TYPE_CHECKING:
from maru_common import BatchLookupKVResponse
from .handler import MaruHandler
logger = logging.getLogger(__name__)
#: Entry-point group scanned for handler plugins.
PLUGIN_GROUP = "maru.handler_plugins"
#: Comma-separated allowlist of plugin *names* to load. Unset/empty → load all.
PLUGIN_ALLOWLIST_ENV = "MARU_PLUGINS"
#: Process-level cache of the discovered entry points. The installed
#: distribution set is immutable within a process, so scan once (the scan is
#: not free) and reuse across every MaruHandler construction.
_discovered_entry_points: list | None = None
def _discover_entry_points() -> list:
"""Scan (once) the installed ``maru.handler_plugins`` entry points."""
global _discovered_entry_points
if _discovered_entry_points is None:
try:
_discovered_entry_points = list(
importlib.metadata.entry_points(group=PLUGIN_GROUP)
)
except Exception:
logger.exception("failed to enumerate %s entry points", PLUGIN_GROUP)
_discovered_entry_points = []
return _discovered_entry_points
@runtime_checkable
class MaruHandlerPlugin(Protocol):
"""Extension interface invoked by MaruHandler at defined lifecycle seams.
Every hook is **optional** — a plugin implements only the ones it needs;
MaruHandler skips any that are absent. All hooks are best-effort: an
exception raised inside one is logged and swallowed so it can never break
the surrounding core operation. That isolation covers *raised exceptions*
only — a hook that blocks or hangs still stalls the calling core path, so
every hook must return promptly.
"""
def on_init(self, handler: MaruHandler) -> None:
"""Called once, at the end of ``MaruHandler.__init__``.
The handler is constructed but **not yet connected** (``_mapper`` is
``None`` until ``connect()``). Use this only for cheap setup; defer any
work that needs mapped regions to :meth:`on_batch_retrieve`.
Runs synchronously inside the constructor, so keep it fast and
non-blocking. Note this hook is **not paired** with
:meth:`on_close`: ``on_close`` fires only if the handler was connected
(see its note), so do not acquire in ``on_init`` a resource whose
release you rely on ``on_close`` to perform — a construct-then-never-
connect sequence would leak it. Acquire lazily in ``on_batch_retrieve``
instead.
"""
...
def on_batch_retrieve(
self,
handler: MaruHandler,
keys: list[str],
batch_resp: BatchLookupKVResponse,
) -> None:
"""Called at the end of ``batch_retrieve``, after regions are mapped.
``keys[i]`` corresponds to ``batch_resp.entries[i]``. For a found
entry, the KV bytes live at ``entry.kv_offset`` (offset within the
allocation) for ``entry.kv_length`` bytes; ``entry.handle.offset`` is
only the region's mmap base, **not** the payload location, so a
hardware hint must target ``handle.offset + kv_offset`` for
``kv_length`` bytes (mirror ``get_buffer_view``). The region is already
mapped, so a plugin may issue hints (prefetch/pin) against live memory.
Runs on the retrieval hot path — keep it cheap and non-blocking.
**Not serialized against** :meth:`on_close`: ``batch_retrieve`` takes
no lock, so this hook can still be in flight while another thread's
``close()`` unmaps the very regions referenced here. A plugin issuing
HW hints must be thread-safe and treat pin/unpin as idempotent (or
reference-counted) so a hint that lands after the unmap is harmless.
"""
...
def on_close(self, handler: MaruHandler) -> None:
"""Called during ``MaruHandler.close``, while regions are still mapped.
Runs before regions are unmapped and the RPC connection is torn down,
so a plugin can safely release resources tied to mapped memory (e.g.
unpin device ranges) here. Fires **only if the handler was connected** —
a handler that is constructed but never connected does not invoke this
hook (see :meth:`on_init`). A ``metadata_only`` handler (a vLLM
scheduler's) maps no region; it calls this hook before it closes its
RPC connection, after the ``read_hint_end`` of every request it hinted
and did not end (see :meth:`on_memory_hint`).
Dispatched while holding the handler's write lock, so a slow or
blocking implementation stalls ``close()`` and every concurrent writer:
keep it cheap and non-blocking, same as the other hooks.
"""
...
#: Hint actions this plugin takes in :meth:`on_memory_hint` (for example
#: ``("read_hint", "read_hint_end", "read_promote")``). A plugin that
#: names none gets no hints.
hint_actions: tuple[str, ...] = ()
def on_memory_hint(self, handler: MaruHandler, message: dict) -> None:
"""Take one memory hint of the ``cxl`` backend.
With ``read_hints`` or ``read_promotes`` on, a handler of the ``cxl``
backend gives its hints to the plugins whose :attr:`hint_actions` name
the hint's action. ``message`` has the form a remote CXL agent's
region extension gets (:mod:`maru_handler.hints`):
- ``action``: ``"read_hint"`` (``request`` will read ``ranges`` soon,
in this order), ``"read_hint_end"`` (``request`` ended; what was
prepared for it may go) or ``"read_promote"`` (the load of
``request``, which another process hinted, starts now);
- ``request``: the serving engine's request id;
- ``ranges``: for ``read_hint``, the leading hit keys' bytes, each a
``{"handle": {"region_id", "offset", "length"}, "offset",
"length"}`` map: ``length`` bytes at ``offset`` in the region whose
mmap offset in the DAX device is ``handle["offset"]``. Empty for
the other actions.
Called on the handler's hint thread, in the order the hints were
given, never on the caller's path. When the handler closes, every
request it hinted and did not end gets its ``read_hint_end`` before
:meth:`on_close`, and no hint comes after it. If the hints still
queued take longer than the close timeout (5 s), every message still
queued is dropped, ends included, and ``close`` waits without a time
limit for the hint thread's call under way (a metadata lookup or a
plugin call); :meth:`on_close` then stands for the ends of the
requests still open. A
hint is advice: the load reads every key either way. Keep it short;
a slow plugin delays the next hints and the handler's close, not the
loads.
"""
...
def contribute_stats(self) -> dict | None:
"""Return a JSON-serializable stats dict merged into ``get_stats``.
The result is placed under ``stats["plugins"][<plugin-class-name>]``.
The key is the plugin's class name, so two plugins sharing a class name
(from different packages) would overwrite each other here — give the
class a distinctive name if that collision is possible. Return ``None``
(or an empty dict) to contribute nothing.
"""
...
def _get_allowlist() -> set[str] | None:
"""Parse ``MARU_PLUGINS`` into a name set, or ``None`` when unset/empty."""
raw = os.environ.get(PLUGIN_ALLOWLIST_ENV, "").strip()
if not raw:
return None
return {name.strip() for name in raw.split(",") if name.strip()}
def load_handler_plugins(
entry_points_iter=None,
allowlist: set[str] | None = None,
) -> list[object]:
"""Discover and instantiate handler plugins, isolating every failure.
Args:
entry_points_iter: Iterable of entry-point objects (each with a
``name`` attribute and a ``load()`` method). Defaults to the
installed ``maru.handler_plugins`` entry points; injectable for
testing.
allowlist: Explicit set of plugin names to load. When ``None``, the
``MARU_PLUGINS`` env var is consulted (unset → load all).
Returns:
Instantiated plugin objects. Never raises: a plugin that fails to
enumerate, load, or instantiate is logged and skipped.
"""
if allowlist is None:
allowlist = _get_allowlist()
if entry_points_iter is None:
entry_points_iter = _discover_entry_points()
plugins: list[object] = []
seen_names: set[str] = set()
for ep in entry_points_iter:
if allowlist is not None and ep.name not in allowlist:
logger.debug(
"skipping handler plugin %r (not in %s)", ep.name, PLUGIN_ALLOWLIST_ENV
)
continue
if ep.name in seen_names:
# Two installed dists can register the same name (e.g. a stale and
# a fresh copy). First wins — running both would double every hook.
logger.warning(
"duplicate handler plugin name %r, ignoring the later entry", ep.name
)
continue
seen_names.add(ep.name)
try:
factory = ep.load()
plugin = factory()
except Exception:
logger.exception("handler plugin %r failed to load, skipping", ep.name)
continue
if plugin is None:
# A factory that self-disables should return a no-op object, not
# None — a None here would sit in the list as a dead "plugin".
logger.warning("handler plugin %r factory returned None, skipping", ep.name)
continue
plugins.append(plugin)
logger.info("loaded handler plugin: %s", ep.name)
# Surface typos: an allowlist name that matched no installed plugin is
# almost always a misspelling, and otherwise loads nothing silently.
if allowlist is not None:
missing = allowlist - seen_names
if missing:
logger.warning(
"%s names matched no installed plugin: %s",
PLUGIN_ALLOWLIST_ENV,
", ".join(sorted(missing)),
)
return plugins