fix(BACKEND-ROCM): ask the device that will RUN a placed expert tower whether it may keep its blocks

include/vllm/model_executor/device_placement.h

@@ -171,6 +171,19 @@ class MoePlacementPlan {
   // gets the inert answer rather than an exception on the decode path.
   vt::DeviceType DeviceForLayer(int64_t l) const;
 
+  // The device THIS ROUTED-EXPERT TENSOR's block runs on, keyed by the only
+  // thing a weight loader has: the name.
+  //
+  // WHY THE LOADER NEEDS THIS. `RunMoePlaced` hands the placed device to the
+  // forward, but the RESIDENCY of a routed-expert tower is decided far earlier,
+  // by `GgufLoadPolicy::Route`, and that decision is a question about the device
+  // that will EXECUTE the tower — whether it has a `vec_dot` for the encoding.
+  // Asking the engine device about a tower the plan has already sent elsewhere
+  // is how a checkpoint whose experts every placement device can execute is
+  // refused at load. A name with no `blk.<N>.` prefix, and a layer this plan was
+  // not resolved against, both answer the engine device.
+  vt::DeviceType DeviceForRoutedExpertTensor(const std::string& name) const;
+
   // True when at least one layer runs its experts somewhere other than the
   // engine device. The forward reads this to decide whether any of the placement
   // machinery runs at all, so an unplaced model pays nothing.
@@ -212,6 +225,12 @@ class MoePlacementPlan {
 // of them.
 std::vector<std::string> RoutedExpertTensorNames(int64_t layer);
 
+// The decoder-block index a llama.cpp GGUF tensor name carries (`blk.<N>.…`),
+// or -1 when it carries none. The inverse of `RoutedExpertTensorNames`' own
+// prefix, exposed so a caller resolves the layer with the same rule the plan was
+// built with instead of a private copy of it.
+int64_t GgufBlockIndexFromTensorName(const std::string& name);
+
 // ── The placement device's queue, and the active plan ────────────────────────
 
 // The queue a placed group runs on. One per device type, created on FIRST USE and

include/vllm/model_executor/model_loader/gguf_device_fit.h

@@ -35,6 +35,7 @@
 #include <string_view>
 
 #include "vllm/model_executor/model_loader/gguf_keep_quant.h"
+#include "vllm/model_executor/device_placement.h"
 #include "vllm/model_executor/model_loader/gguf_reader.h"
 
 namespace vllm {
@@ -131,6 +132,15 @@ struct GgufStagedFootprint {
   // The arena term actually added (0 when the lane is off, and 0 when it is on
   // but nothing matched).
   size_t arena_bytes = 0;
+  // #2517. How many routed-expert tensors the installed hybrid PLACEMENT took
+  // out of the sum, and how many bytes they would otherwise have contributed.
+  // Both 0 with no plan, and 0 with a plan that places nothing.
+  //
+  // Reported rather than merely subtracted, for the same reason the streamed
+  // pair is: a figure that shrank by 137 GiB without saying which tensors left
+  // it is a number an operator cannot check.
+  size_t placed_tensor_count = 0;
+  size_t placed_bytes = 0;
 };
 
 // The per-expert SLICE size of a stacked expert tower, which is the slot size
@@ -254,10 +264,28 @@ inline bool GgufPolicyForcesFullExpand(const GgufLoadPolicy& policy) {
 // available on that device) where the min-based bound under-counted by
 // roughly 4x and the load reached an uncaught `hipMalloc: out of memory`
 // instead of this refusal.
+//
+// `placement` (#2517) is the resolved hybrid-placement plan, or null for no
+// plan. A routed-expert tensor whose layer the plan places away from the plan's
+// own engine device is moved into `placed_*` and leaves the bound, because a
+// layer whose MoE block only ever runs on the placement device is never staged
+// to the accelerator -- `RunMoePlaced` hands that device to the block and
+// `ResidentWeight` aliases host bytes on a CPU `Dev` rather than uploading them
+// (`moe_placement_seam.h`). NULL, or a plan that places nothing, reproduces the
+// pre-#2517 answer byte for byte, which is every load in this tree that
+// configured no placement.
+//
+// WHY THIS IS NOT OPTIONAL BOOKKEEPING. `InstallMoePlacementPlan` runs 220 lines
+// ABOVE the one `CheckDeviceWeightFit` call site (`model_loader.cpp:2672` and
+// `:2895`), and before this parameter existed the refusal quoted the UN-reduced
+// footprint on the line after the plan announced it had reduced it. On
+// `strix:gpu0` that contradiction was the whole distance between a placed
+// GLM-5.3 load and a forward.
 GgufStagedFootprint GgufStagedWeightFootprint(
     const GgufFile& gguf, size_t model_dtype_bytes = 2,
     const StreamedExpertLane& lane = {},
-    bool policy_forces_full_expand = false);
+    bool policy_forces_full_expand = false,
+    const MoePlacementPlan* placement = nullptr);
 
 // The budget to compare a footprint against, in bytes, or 0 for UNKNOWN.
 //
@@ -383,6 +411,7 @@ DeviceWeightFit CheckDeviceWeightFit(const GgufFile& gguf,
                                      size_t budget_bytes,
                                      size_t model_dtype_bytes = 2,
                                      const StreamedExpertLane& lane = {},
-                                     bool policy_forces_full_expand = false);
+                                     bool policy_forces_full_expand = false,
+                                     const MoePlacementPlan* placement = nullptr);
 
 }  // namespace vllm

include/vllm/model_executor/model_loader/gguf_keep_quant.h

@@ -381,6 +381,24 @@ struct GgufLoadPolicy {
   // Route one tensor and notify `audit`. This is the ONLY entry point the
   // loader uses, so every routed tensor is observable.
   GgufResidency Route(const GgufTensorInfo& tensor, GgufTensorRole role) const;
+
+  // The device that will EXECUTE this tensor, which is what a residency decision
+  // is about and is not always `device`.
+  //
+  // It differs for exactly one role. A routed-expert tower whose layer the
+  // installed `MoePlacementPlan` places away from the engine is computed on the
+  // PLACEMENT device (`RunMoePlaced` hands that device to the MoE block), so
+  // asking the engine whether its `vec_dot` covers the encoding refuses a
+  // checkpoint the placement device can execute perfectly well. Every other
+  // role, and every load with no placement installed, answers `device`
+  // unchanged -- see the implementation for the four inertness terms and why the
+  // last one is required rather than defensive.
+  //
+  // Public because `PeekRoute` must resolve the device THE SAME WAY `Route`
+  // does; a second spelling is how a bound and a forward come to disagree about
+  // one file.
+  vt::DeviceType ComputeDeviceFor(const std::string& name,
+                                  GgufTensorRole role) const;
 };
 
 // A policy copy with the KEEP-QUANT residency disabled.

src/vllm/entrypoints/model_loader.cpp

@@ -2892,12 +2892,29 @@ std::unique_ptr<LoadedEngine> LoadedEngine::FromModelDir(
       // defaults) should run. Using the narrower predicate here is what makes
       // this refusal reachable on ROCm without moving any of the other one's
       // consumers; the row's spec records why that flag stays untouched.
+      // #2517: THE PLAN INSTALLED 220 LINES ABOVE, credited here.
+      //
+      // `InstallMoePlacementPlan` runs at the top of this branch, before any
+      // weight I/O, and announces what it placed. Until this argument existed
+      // the refusal on the very next line quoted the UN-reduced footprint: one
+      // load printed "56 layers run their routed experts on cpu ... to bring a
+      // 216433205760 B footprint under a 68719476736 B budget" and then refused
+      // needing 216433205760 B. Two lines of one load contradicting each other,
+      // and on `strix:gpu0` the contradiction was the whole distance between a
+      // placed GLM-5.3 load and a forward.
+      //
+      // The GLOBAL rather than a local, because `SetActiveMoePlacementPlan` is
+      // where the plan lives and the forward reads the same object; a copy taken
+      // here could be the one the seam never sees. It is installed
+      // unconditionally, including when nothing is placed, so this is never a
+      // stale plan from a previous load in the same process.
       const DeviceWeightFit fit = CheckDeviceWeightFit(
           gguf, vt::DeviceTypeName(target.device_type()),
           target.allocates_bounded_device_memory(),
           DeviceWeightBudgetBytes(
               target.residency_policy().device_memory_total_bytes),
-          /*model_dtype_bytes=*/2, lane, policy_forces_full_expand);
+          /*model_dtype_bytes=*/2, lane, policy_forces_full_expand,
+          &vllm::ActiveMoePlacementPlan());
       if (fit.refuse) throw std::runtime_error(fit.message);
     }
     // QUANT-QWEN38-27B-GGUF-ARM (#821): refuse a qwen3_5-family GGUF carrying

src/vllm/model_executor/device_placement.cpp

@@ -142,6 +142,46 @@ vt::DeviceType MoePlacementPlan::DeviceForLayer(int64_t l) const {
   return per_layer_[static_cast<size_t>(l)];
 }
 
+int64_t GgufBlockIndexFromTensorName(const std::string& name) {
+  // `blk.<N>.` is llama.cpp's GGUF spelling for a decoder block, and it is the
+  // ONLY key a loader has: `GgufTensorInfo` carries a name, never a layer.
+  // `RoutedExpertTensorNames` composes the same prefix and
+  // `LlmFfnExpsBlockRegex` matches it, so this is the inverse of two functions
+  // that already exist rather than a new naming convention.
+  //
+  // ANCHORED AT THE START, deliberately. `regex_search`'s unanchored semantics
+  // are right for an operator's `-ot` pattern and wrong here: a name that merely
+  // CONTAINS `blk.3.` is not a block-3 tensor, and answering as though it were
+  // would move a weight no plan claimed.
+  static constexpr char kPrefix[] = "blk.";
+  const size_t plen = sizeof(kPrefix) - 1;
+  if (name.size() <= plen || name.compare(0, plen, kPrefix) != 0) return -1;
+  size_t i = plen;
+  int64_t idx = 0;
+  bool any = false;
+  while (i < name.size() && name[i] >= '0' && name[i] <= '9') {
+    // A count that cannot be a layer index is not a layer index. Refusing
+    // instead of wrapping keeps a malformed name out of `per_layer_`'s range
+    // rather than aliasing it onto a real layer.
+    if (idx > (INT64_MAX - 9) / 10) return -1;
+    idx = idx * 10 + (name[i] - '0');
+    any = true;
+    ++i;
+  }
+  if (!any || i >= name.size() || name[i] != '.') return -1;
+  return idx;
+}
+
+vt::DeviceType MoePlacementPlan::DeviceForRoutedExpertTensor(
+    const std::string& name) const {
+  const int64_t layer = GgufBlockIndexFromTensorName(name);
+  // A name carrying no block index belongs to no layer, so no placement claims
+  // it. The engine's own device is the inert answer, which is `DeviceForLayer`'s
+  // own out-of-range behaviour rather than a second convention.
+  if (layer < 0) return engine_device_;
+  return DeviceForLayer(layer);
+}
+
 std::string MoePlacementPlan::Describe() const {
   if (placed_ == 0) return "";
   std::string out = std::to_string(placed_);

src/vllm/model_executor/model_loader/gguf_device_fit.cpp

@@ -131,12 +131,47 @@ bool GgufExpertTowersReachSlotLane(const GgufFile& gguf,
   return matched;
 }
 
+namespace {
+
+// #2517. Is this tensor a ROUTED-EXPERT tower whose layer the installed plan
+// runs somewhere other than the engine device?
+//
+// The name test is `_exps.weight`, which is the class the plan's own
+// `RoutedExpertTensorNames` composes and the class `RunMoePlaced` moves. The
+// layer term goes through `DeviceForRoutedExpertTensor`, so the bound and the
+// forward resolve the placement with ONE rule rather than two.
+//
+// The engine-device comparison is against the PLAN's engine device, not the
+// caller's: a plan resolved for a different device than the one being priced is
+// a caller bug, and answering "placed" for a tensor that will in fact stage here
+// would DELETE a correct refusal -- the unsafe direction #1378 names.
+bool PlacedAwayFromEngine(const MoePlacementPlan* plan,
+                          const std::string& name) {
+  if (plan == nullptr || !plan->PlacesAnything()) return false;
+  if (!NameHasSuffix(name, "_exps.weight")) return false;
+  return plan->DeviceForRoutedExpertTensor(name) != plan->engine_device();
+}
+
+}  // namespace
+
 GgufStagedFootprint GgufStagedWeightFootprint(const GgufFile& gguf,
                                               size_t model_dtype_bytes,
                                               const StreamedExpertLane& lane,
-                                              bool policy_forces_full_expand) {
+                                              bool policy_forces_full_expand,
+                                              const MoePlacementPlan* placement) {
   GgufStagedFootprint out;
   for (const GgufTensorInfo& t : gguf.Tensors()) {
+    // BEFORE the lane test, because a placed tower does not reach the lane
+    // either: `RunMoePlaced` hands the block a CPU `Dev`, and the expert-stream
+    // seam's own admission test (`expert_stream_seam.cpp:431`) is about the
+    // ENGINE platform. Counting one tensor in both classes would report the same
+    // bytes twice in a message whose whole job is to account for them once.
+    if (PlacedAwayFromEngine(placement, t.name)) {
+      ++out.placed_tensor_count;
+      out.placed_bytes +=
+          StagedBytes(t, model_dtype_bytes, policy_forces_full_expand);
+      continue;
+    }
     // W0d: a tensor the slot lane serves is never staged, so it contributes
     // nothing to the bound and cannot be the largest single allocation either.
     // It is still COUNTED, in its own two fields, so the caller can say what was
@@ -216,7 +251,8 @@ DeviceWeightFit CheckDeviceWeightFit(const GgufFile& gguf,
                                      size_t budget_bytes,
                                      size_t model_dtype_bytes,
                                      const StreamedExpertLane& lane,
-                                     bool policy_forces_full_expand) {
+                                     bool policy_forces_full_expand,
+                                     const MoePlacementPlan* placement) {
   DeviceWeightFit fit;
   fit.budget_bytes = budget_bytes;
   // A platform that does not stage weights reads them where they already are, so
@@ -229,7 +265,7 @@ DeviceWeightFit CheckDeviceWeightFit(const GgufFile& gguf,
   if (budget_bytes == 0) return fit;
 
   const GgufStagedFootprint fp = GgufStagedWeightFootprint(
-      gguf, model_dtype_bytes, lane, policy_forces_full_expand);
+      gguf, model_dtype_bytes, lane, policy_forces_full_expand, placement);
   fit.needed_bytes = fp.lower_bound_bytes;
   if (fp.lower_bound_bytes <= budget_bytes) return fit;
 
@@ -282,6 +318,21 @@ DeviceWeightFit CheckDeviceWeightFit(const GgufFile& gguf,
   // correction is appended rather than left to be read as a stale claim. The
   // lane-OFF message is untouched, byte for byte, because every CPU and discrete
   // user reads that one and it is still true for them.
+  // #2517. The plan announced a reduction on the line above this refusal and
+  // the refusal used to quote the un-reduced figure, which is two contradictory
+  // numbers in consecutive lines of one load. Whatever the verdict, say what the
+  // placement took out of it.
+  if (fp.placed_tensor_count > 0) {
+    fit.message +=
+        " NOTE: an installed hybrid MoE placement runs " +
+        std::to_string(fp.placed_tensor_count) +
+        " routed-expert tensors (" + std::to_string(fp.placed_bytes) +
+        " bytes, " + Gib(fp.placed_bytes) + ") on another device, so they are "
+        "ALREADY EXCLUDED from the figure above rather than being a reduction "
+        "still available. What does not fit is the remainder. Placing more "
+        "layers (\"cpu_moe\", \"n_cpu_moe\", VT_CPU_MOE=1) is the knob that "
+        "shrinks it further.";
+  }
   if (fp.streamed_tensor_count > 0) {
     fit.message +=
         " NOTE: the expert-stream lane IS active for this load, so the "

src/vllm/model_executor/model_loader/gguf_keep_quant.cpp

@@ -7,6 +7,7 @@
 #include <cstring>
 
 #include "vllm/config/weight_residency.h"
+#include "vllm/model_executor/device_placement.h"
 #include "vt/ops.h"
 #include "vt/quant.h"
 
@@ -398,11 +399,52 @@ GgufLoadPolicy GgufLoadPolicy::FromEnv(vt::DeviceType dev) {
   return p;
 }
 
+// THE DEVICE A RESIDENCY DECISION IS ACTUALLY ABOUT (#2516).
+//
+// `GgufLoadPolicy::device` is the ENGINE's resolved device, and its own field
+// comment says every device-dependent decision in the struct reads it. Hybrid
+// placement (#2023, #2314) introduced a SECOND, narrower answer for exactly one
+// class of tensor: `ActiveMoePlacementPlan().DeviceForLayer(l)` is the device a
+// layer's routed experts are COMPUTED on, and `RunMoePlaced` hands that device
+// to `MoeBlock`. A keep-quant decision asks "does the device that will execute
+// this GEMM have a `vec_dot` for this encoding", so for a PLACED routed-expert
+// tower it is a question about the placement device, not about the engine.
+//
+// Measured, on GLM-5.3 `UD-IQ1_S` on `strix:gpu0`: `DeviceKeepQuantSupported`
+// serves {Q8_0, Q4_K, Q5_K, Q6_K} on ROCm, so all 228 IQ1_S/IQ3_XXS/IQ2_XXS/
+// IQ4_XS/Q2_K/Q3_K towers routed `kExpandBf16` and `LoadStackedExperts` refused
+// the load by name -- for towers whose bytes never reach the GPU at all (that
+// model reads a tower only through `GlmExpertSlice`, never through
+// `ResidentWeight`) and which the installed plan had already sent to the CPU,
+// whose `vec_dot` table covers every one of those six encodings.
+//
+// THIS IS #1136 AND #2406 ONE SEAM FURTHER ALONG. Both were the same shape: a
+// residency decision resolved against a device other than the one that would
+// read the bytes. The repair is the same shape too -- ask the device that will
+// run it.
+//
+// INERT BY CONSTRUCTION, in four terms, and the last one is the load-bearing
+// one. The plan is a process-global that a load with no placement never writes,
+// and a DEFAULT-CONSTRUCTED plan's `engine_device()` is `kCPU`. Returning its
+// answer unconditionally would therefore route a CUDA load's experts as though
+// they were on the host. So the placed answer is ADOPTED only when the plan
+// actively moves THIS tensor somewhere other than the device the plan itself was
+// resolved for; every other path returns the policy's own device unchanged.
+vt::DeviceType GgufLoadPolicy::ComputeDeviceFor(const std::string& name,
+                                                GgufTensorRole role) const {
+  if (role != GgufTensorRole::kStackedExpertWeight) return device;
+  const MoePlacementPlan& plan = ActiveMoePlacementPlan();
+  if (!plan.PlacesAnything()) return device;
+  const vt::DeviceType placed = plan.DeviceForRoutedExpertTensor(name);
+  if (placed == plan.engine_device()) return device;
+  return placed;
+}
+
 GgufResidency GgufLoadPolicy::Route(const GgufTensorInfo& tensor,
                                     GgufTensorRole role) const {
-  const GgufResidency r =
-      RouteGgufTensor(keep_quant, keep_f16, nvfp4_fp4, cpu_ref, role,
-                      tensor.ggml_type, tensor.shape, device);
+  const GgufResidency r = RouteGgufTensor(
+      keep_quant, keep_f16, nvfp4_fp4, cpu_ref, role, tensor.ggml_type,
+      tensor.shape, ComputeDeviceFor(tensor.name, role));
   if (audit) audit(tensor.name, role, r);
   return r;
 }
@@ -415,9 +457,15 @@ GgufLoadPolicy NoKeepQuant(const GgufLoadPolicy& policy) {
 
 GgufResidency PeekRoute(const GgufLoadPolicy& policy, const GgufTensorInfo& tensor,
                         GgufTensorRole role) {
+  // THROUGH THE SAME RESOLVER `Route` USES, not a copy of the expression. This
+  // function exists so a caller can ask what the loader WILL do without
+  // notifying the audit, and `GgufExpertTowersReachSlotLane` is one such caller
+  // -- it peeks the very tensors the loader then routes. Two spellings of the
+  // device term is exactly how the bound and the forward come to disagree about
+  // one file, which is the defect #1378 records.
   return RouteGgufTensor(policy.keep_quant, policy.keep_f16, policy.nvfp4_fp4,
                          policy.cpu_ref, role, tensor.ggml_type, tensor.shape,
-                         policy.device);
+                         policy.ComputeDeviceFor(tensor.name, role));
 }
 
 }  // namespace vllm