build: bump vllm to 0.29.0

3rdparty/TensorRT-LLM-workspace/_backend.py

@@ -242,9 +242,14 @@ def build_wheel(wheel_directory, config_settings=None, metadata_directory=None):
         )
         mirror_dir.mkdir(parents=True, exist_ok=True)
         mirror = mirror_dir / wheel.name
-        if wheel.resolve() != mirror.resolve():
+        # The directory is content-addressed, so an existing mirror is this
+        # wheel already; skipping the copy keeps a stage that inherits the
+        # mirror from an earlier layer from duplicating it into its own.
+        if mirror.exists():
+            print(f"[trtllm-backend] Mirror already present: {mirror}", flush=True)
+        else:
             shutil.copy2(wheel, mirror)
-        print(f"[trtllm-backend] Mirrored cached wheel to: {mirror}", flush=True)
+            print(f"[trtllm-backend] Mirrored cached wheel to: {mirror}", flush=True)
 
     destination = Path(wheel_directory) / wheel.name
     shutil.copy2(wheel, destination)

docker/Dockerfile

@@ -374,6 +374,12 @@ set +e
 # and churning it here would copy files up into this layer.
 # The throwaway venv must be symlink-mode: hardlinks to files in the dependency layer's uv
 # cache make overlayfs copy them up into this layer, and the copies outlive the rm -rf below.
+# TRTLLM_WHEEL_CACHE_MIRROR_DIR keeps a copy of the wheel in this image layer.
+# CI creates a fresh buildx builder for every job, so the cache mount above only
+# lives for the duration of one build: when a later build reuses this layer from
+# the registry cache and re-executes the release stage (any NeMo-RL source
+# change does that), the mount is empty there. The release stage re-seeds it
+# from this mirror instead of failing on TRTLLM_REQUIRE_CACHED_WHEEL.
 UV_CACHE_DIR=/root/.cache/uv \
     TRTLLM_WHEEL_CACHE_DIR=/root/.cache/trtllm-wheels \
     TRTLLM_WHEEL_CACHE_MIRROR_DIR=/opt/trtllm_wheels \

docs/design-docs/dynamo-integration.md

@@ -50,7 +50,7 @@ cache mode determines where it runs.
 `BUILD_DYNAMO=1` adds a Python 3.12 `/opt/dynamo_venv` to the standard image.
 It contains only `ai-dynamo[vllm]==1.3.0.post1`, its pinned vLLM 0.23.0, etcd,
 and NATS. NeMo-RL's normal Ray and engine environments are unchanged; the
-standard NeMo-RL vLLM environment currently uses vLLM 0.25.1.
+standard NeMo-RL vLLM environment currently uses vLLM 0.29.0.
 
 vLLM 0.23.0 predates PR #44814, which fixes layerwise reload accounting for
 composed loaders. The installer asserts the exact vLLM version, checks and

docs/guides/dynamo-generation.md

@@ -29,7 +29,7 @@ docker buildx build \
 The opt-in layer installs `ai-dynamo[vllm]==1.3.0.post1` in isolated Python
 3.12 under `/opt/dynamo_venv`, along with etcd v3.5.21 and NATS Server v2.11.6.
 It does not replace NeMo RL's normal Ray or vLLM dependencies: the standard
-NeMo RL vLLM environment currently uses vLLM 0.25.1, while this isolated
+NeMo RL vLLM environment currently uses vLLM 0.29.0, while this isolated
 Dynamo environment uses Dynamo's vLLM 0.23.0 pin. Both environments pin
 `nvidia-nccl-cu13==2.30.7` so their NCCL communicators use the same release.
 For a local source checkout, the same environment can be installed under

docs/guides/grpo-audio-visual.md

@@ -12,7 +12,7 @@ The NeMo RL container does not ship `torchaudio`, `torchcodec`, or system FFmpeg
 bash tools/install_audio_deps.sh
 ```
 
-This installs system FFmpeg (required by `torchcodec` at runtime) and pins `torchaudio==2.11.0` + `torchcodec>=0.3.0` into the container venv. The script is idempotent — re-running it on a machine where the deps are already present exits immediately.
+This installs system FFmpeg (required by `torchcodec` at runtime) and pins `torchaudio==2.11.0` + `torchcodec==0.16.0` into the container venv. The script is idempotent — re-running it on a machine where the deps are already present exits immediately.
 
 > **Note:** `decord` and `av` (PyAV) are **not** used. Audio and video are decoded via `torchcodec` (backed by system FFmpeg) and `torchaudio` respectively.
 

nemo_rl/distributed/collectives.py

@@ -44,9 +44,7 @@ def rebalance_nd_tensor(
         dtype=torch.int64,
         device=torch.cuda.current_device(),
     )
-    torch.distributed.all_gather_into_tensor(
-        batch_num_per_rank, num_samples, group=group
-    )
+    torch.distributed.all_gather_single(batch_num_per_rank, num_samples, group=group)
 
     B = batch_num_per_rank.sum()
     other_dims = tensor.shape[1:]

nemo_rl/modelopt/models/generation/vllm_quant_backend.py

@@ -23,6 +23,9 @@
 import zmq
 from modelopt.torch.quantization.nn.modules.tensor_quantizer import TensorQuantizer
 
+from nemo_rl.modelopt.models.generation.vllm_quant_moe_amax import (
+    route_moe_input_quantizer_amax,
+)
 from nemo_rl.modelopt.utils import (
     MODELOPT_REAL_QUANT_ZMQ_TIMEOUT_MS,
     matches_quant_ignore_pattern,
@@ -149,10 +152,11 @@ def _batch_fused_modelopt_moe_weights(
     loader still requires an expert id, so only the tiny per-expert global
     scales are exposed as scalar views.
 
-    Gated ``w13`` payloads are the exception on vLLM >= 0.25: they are emitted
-    as per-expert 2-D shards instead, because ``RoutedExperts.load_weights``'
-    fused-3D branch mis-transposes packed NVFP4. See the comment at the
-    emission site below.
+    ``w13`` payloads are the exception on vLLM >= 0.25: they are emitted as
+    per-expert 2-D shards instead, because ``RoutedExperts.load_weights``'
+    fused-3D branch mis-transposes packed NVFP4 (gated models) and, for
+    non-gated models that have no fused mapping at all, keeps only the first
+    gate/up half of each expert. See the comments at the emission sites below.
     """
     batched: list[tuple[str, torch.Tensor]] = []
     for name, tensor in weights:
@@ -171,11 +175,22 @@ def _batch_fused_modelopt_moe_weights(
         if target in {"w13_weight", "w13_weight_scale"}:
             target_suffix = "weight" if target == "w13_weight" else "weight_scale"
             if w13_num_shards_by_prefix.get(prefix) == 1:
-                batched.append(
+                # Non-gated experts (Nemotron-H: `ckpt_gate_proj_name="up_proj"`,
+                # no up shard) get no fused gate/up mapping from vLLM >= 0.28
+                # ("Unexpected gate/up projection names ... will be skipped"),
+                # so a batched 3-D tensor under `experts.0.up_proj` falls into
+                # `RoutedExperts.load_weights`' fused branch, which assumes a
+                # gate/up concatenation and keeps `chunk(2, dim=1)[0]`: half of
+                # every expert's rows. Half of w13 plus all of w2 is the
+                # "134701312/179601664 elements" layerwise-reload failure seen
+                # on the nanov3 w4a16 recipe. Emit per-expert 2-D shards, the
+                # same path the initial disk load takes.
+                batched.extend(
                     (
-                        f"{prefix}.experts.0.up_proj.{target_suffix}",
-                        tensor,
+                        f"{prefix}.experts.{expert_id}.up_proj.{target_suffix}",
+                        expert_weight,
                     )
+                    for expert_id, expert_weight in enumerate(tensor.unbind(0))
                 )
                 continue
             if tensor.ndim < 2 or tensor.shape[1] % 2 != 0:
@@ -667,6 +682,19 @@ def _load_weights(self, weights):
                         source_storage_ptrs,
                     )
 
+        # vLLM 0.28 routes every ``experts.*`` name through
+        # ``RoutedExperts.load_weights``, which resolves the rewritten name with a
+        # single ``getattr`` and therefore cannot reach the dotted quantizer
+        # buffers (``w13_input_quantizer._amax``). Fan the per-expert amax values
+        # into the fused quantizers here and keep them away from vLLM's loader.
+        # Checkpoint names (e.g. Nemotron-H's ``backbone.*``) only turn into the
+        # module's vLLM ``layer_name`` through the model's hf_to_vllm_mapper.
+        weights = route_moe_input_quantizer_amax(
+            self.model_runner.model,
+            weights,
+            mapper=getattr(self.model_runner.model, "hf_to_vllm_mapper", None),
+        )
+
         # MBridge exports K/V amax with the HF-semantic attention path, such as
         # ``self_attn.k_bmm_quantizer._amax``. ModelOpt installs these quantizers
         # on vLLM's inner Attention module, whose runtime path contains ``.attn``.

nemo_rl/modelopt/models/generation/vllm_quant_moe_amax.py

@@ -0,0 +1,139 @@
+# Copyright (c) 2026, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Route per-expert fakequant amax buffers around vLLM's expert weight loader.
+
+The Megatron side exports one ``input_quantizer._amax`` per expert projection,
+e.g. ``model.layers.1.mixer.experts.16.up_proj.input_quantizer._amax``. On the
+vLLM side the ModelOpt MoE quant module owns a single fused quantizer per
+projection group (``w13_input_quantizer`` / ``w2_input_quantizer``), so the
+incoming values fan in with ``max``.
+
+vLLM 0.25 loaded these through the model-level parameter dict, which NeMo-RL
+patches to include quantizer buffers. vLLM 0.28's ``AutoWeightsLoader`` hands
+every ``experts.*`` name to ``RoutedExperts.load_weights`` instead, which
+rewrites the checkpoint name with the expert mapping and resolves the result
+with a single ``getattr(self, param_name)``. A dotted quantizer-buffer name such
+as ``w13_input_quantizer._amax`` cannot resolve that way and the loader raises
+``AttributeError: Layer <experts> has no parameter 'w13_input_quantizer._amax'``.
+
+This module applies the same rewrite vLLM does (replace the mapping's
+``weight_name`` with its ``param_name``, strip the layer prefix) and then walks
+the dotted path on the expert module itself, so the buffers are updated before
+the remaining weights reach vLLM's loader.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable
+from functools import reduce
+from typing import Any
+
+import torch
+
+INPUT_QUANTIZER_AMAX_SUFFIX = "input_quantizer._amax"
+
+
+def _expert_modules(model: torch.nn.Module) -> list[Any]:
+    """Return every module that owns an expert mapping and a vLLM layer name."""
+    return [
+        module
+        for _, module in model.named_modules()
+        if callable(getattr(module, "get_expert_mapping", None))
+        and isinstance(getattr(module, "layer_name", None), str)
+    ]
+
+
+def _resolve_dotted(module: Any, path: str) -> Any | None:
+    try:
+        return reduce(getattr, path.split("."), module)
+    except AttributeError:
+        return None
+
+
+def route_moe_input_quantizer_amax(
+    model: torch.nn.Module,
+    weights: Iterable[tuple[str, torch.Tensor]],
+    *,
+    reduce_fn: Callable[[torch.Tensor, torch.Tensor], torch.Tensor] = torch.max,
+    mapper: Any | None = None,
+) -> list[tuple[str, torch.Tensor]]:
+    """Fan per-expert ``input_quantizer._amax`` values into the fused quantizers.
+
+    Returns the weights that were *not* consumed, in their original order, so
+    the caller can hand them to vLLM's own loader unchanged. Names that end
+    with the amax suffix but do not belong to an expert module (dense layers,
+    attention quantizers) pass through untouched.
+
+    ``mapper`` is the model's ``hf_to_vllm_mapper`` (a vLLM ``WeightsMapper``),
+    when it has one. Refit sends checkpoint names and vLLM's ``layer_name`` /
+    expert mapping use vLLM names; ``load_weights`` applies the mapper before
+    its own matching, so the same rename has to happen here or e.g. Nemotron-H's
+    ``backbone.layers.N.mixer.experts.*`` never matches the module's
+    ``model.layers.N.mixer.experts`` prefix and falls through to the loader
+    that cannot resolve it. Names the mapper drops (returns ``None``) are
+    passed through untouched.
+
+    Raises:
+        KeyError: an expert amax name matched an expert module's mapping but
+            the rewritten quantizer path does not exist on that module. This
+            is the same condition vLLM would have raised on, surfaced with the
+            resolved path so the layout mismatch is visible.
+    """
+    experts = _expert_modules(model)
+    if not experts:
+        return list(weights)
+
+    # (layer prefix, module, [(param_name, weight_name)]) — computed once.
+    routes = []
+    for module in experts:
+        mapping = module.get_expert_mapping()
+        pairs = [(entry[0], entry[1]) for entry in mapping]
+        routes.append((f"{module.layer_name}.", module, pairs))
+
+    remaining: list[tuple[str, torch.Tensor]] = []
+    for name, tensor in weights:
+        if not name.endswith(INPUT_QUANTIZER_AMAX_SUFFIX):
+            remaining.append((name, tensor))
+            continue
+        # Match on the vLLM-side name, exactly as AutoWeightsLoader will.
+        vllm_name = mapper._map_name(name) if mapper is not None else name
+        if vllm_name is None:
+            remaining.append((name, tensor))
+            continue
+        handled = False
+        for prefix, module, pairs in routes:
+            if not vllm_name.startswith(prefix):
+                continue
+            for param_name, weight_name in pairs:
+                if weight_name not in vllm_name:
+                    continue
+                target_path = vllm_name.replace(weight_name, param_name).removeprefix(
+                    prefix
+                )
+                buf = _resolve_dotted(module, target_path)
+                if not isinstance(buf, torch.Tensor):
+                    raise KeyError(
+                        f"Expert module {module.layer_name!r} has no quantizer "
+                        f"buffer {target_path!r} for incoming amax {name!r} "
+                        f"(vLLM name {vllm_name!r})"
+                    )
+                with torch.no_grad():
+                    buf.copy_(reduce_fn(buf, tensor.to(buf.device, buf.dtype)))
+                handled = True
+                break
+            if handled:
+                break
+        if not handled:
+            remaining.append((name, tensor))
+    return remaining

nemo_rl/models/generation/sglang/utils/train_utils.py

@@ -20,6 +20,8 @@
 import torch
 from torch.multiprocessing import reductions
 
+from nemo_rl.utils.cuda_ipc import normalize_cuda_ipc_handle
+
 
 class MultiprocessingSerializer:  # pragma: no cover
     """Serialize/deserialize Python objects using ForkingPickler for IPC.
@@ -85,6 +87,7 @@ def monkey_patch_torch_reductions():
 # The signature has not been changed for years, and we will not need this when the next version is released,
 # so it looks safe to use a constant.
 _REDUCE_TENSOR_ARG_DEVICE_INDEX = 6
+_REDUCE_TENSOR_ARG_STORAGE_HANDLE_INDEX = 7
 
 
 def _reduce_tensor_modified(*args, **kwargs):
@@ -103,6 +106,11 @@ def _reduce_tensor_modified(*args, **kwargs):
 
 def _rebuild_cuda_tensor_modified(*args):
     args = _modify_tuple(args, _REDUCE_TENSOR_ARG_DEVICE_INDEX, _device_from_maybe_uuid)
+    # The training venv that produced the handle may run a newer torch than
+    # this sglang venv; see nemo_rl.utils.cuda_ipc for the version-byte rewrite.
+    args = _modify_tuple(
+        args, _REDUCE_TENSOR_ARG_STORAGE_HANDLE_INDEX, normalize_cuda_ipc_handle
+    )
     return reductions._rebuild_cuda_tensor_original(*args)
 
 

nemo_rl/models/generation/vllm/patches.py

@@ -285,7 +285,17 @@ def _patch_vllm_ray_executor_v2_tcpstore_port(logger) -> None:
     and falls through to ``get_open_port()`` — straight back to ``VLLM_PORT``.
     That is exactly the port the MessageQueue takes. See RL-1104.
 
-    Returns without raising when the snippet is missing, but logs at warning
+    vLLM 0.29 fixes the race upstream (vllm-project/vllm#53666, #50969): the
+    rank-0 actor now binds the TCPStore itself, on a kernel-assigned port, and
+    *holds* that socket (``self._dist_init_store = store``) until
+    ``init_process_group`` reuses it, so there is no probe/bind window for the
+    MessageQueue to land in. That is not the TOCTOU pattern the reserved band
+    guards against (the port is never released between selection and use), and
+    ``_select_tcpstore_port`` no longer exists to patch. When that upstream
+    marker is present this function logs at info level and leaves the file
+    alone.
+
+    Returns without raising when neither form is found, but logs at warning
     level so a silent no-op is visible in worker logs.
     """
     try:
@@ -297,6 +307,9 @@ def _patch_vllm_ray_executor_v2_tcpstore_port(logger) -> None:
         )
         return
 
+    # vLLM >= 0.29: RayWorkerV2.create_dist_init_method binds and keeps the
+    # TCPStore before publishing its port (vllm-project/vllm#50969).
+    upstream_fix_marker = "self._dist_init_store = store"
     marker = "start_port=envs.VLLM_PORT + 32"
     old_snippet = (
         "        if local_dp_rank is None:\n            return get_open_port()\n"
@@ -330,6 +343,12 @@ def _patch_vllm_ray_executor_v2_tcpstore_port(logger) -> None:
         if marker in content:
             logger.info("vLLM RayExecutorV2 TCPStore port patch already applied.")
             return
+        if upstream_fix_marker in content:
+            logger.info(
+                "vLLM binds the RayExecutorV2 TCPStore before publishing its port "
+                "(vllm-project/vllm#50969); NeMo-RL TCPStore port patch not needed."
+            )
+            return
 
         if old_snippet not in content:
             logger.warning(
@@ -363,13 +382,12 @@ def _patch_vllm_ray_executor_v2_tcpstore_port(logger) -> None:
 
 
 def _patch_vllm_shm_broadcast_bind_retry(logger) -> None:
-    """Make MessageQueue's remote socket survive losing a port race.
+    """Keep MessageQueue's remote socket in the reserved band with bind retries.
 
-    ``MessageQueue.__init__`` picks the port for its remote (TCP) socket with
-    ``remote_subscribe_port = get_open_port()``, which *probes a port and
-    releases it*, and only binds it with ZMQ several statements later
-    (``shm_broadcast.py``: ``self.remote_socket.bind(socket_addr)``). The
-    window between the probe and the bind is a TOCTOU race.
+    vLLM 0.28 binds port zero directly, avoiding the old probe/bind race but
+    ignoring ``VLLM_PORT``. Restore reserved-band selection with retries so
+    engine sockets do not consume the ephemeral ports used by other services.
+    A probe alone releases its socket before ZMQ binds, leaving a TOCTOU race.
 
     On vLLM 0.25 that race is lost reliably, not occasionally. Every
     ``RayWorkerProc`` on a **non-driver** node takes ``n_local_reader=0``
@@ -421,10 +439,14 @@ def _patch_vllm_shm_broadcast_bind_retry(logger) -> None:
 
     marker = "_nrl_bind_attempts"
     old_snippet = (
-        '            socket_addr = f"tcp://{connect_ip}:{remote_subscribe_port}"\n'
-        "            self.remote_socket.bind(socket_addr)\n"
+        '            self.remote_socket.bind(f"tcp://{connect_ip}:0")\n'
+        "            last_endpoint = self.remote_socket.getsockopt(zmq.LAST_ENDPOINT)\n"
+        '            remote_subscribe_port = last_endpoint.decode().rsplit(":", 1)[1]\n'
     )
     new_snippet = (
+        "            from vllm.utils.network_utils import get_open_port, _get_open_port\n"
+        "\n"
+        "            remote_subscribe_port = get_open_port()\n"
         "            # NeMo-RL: get_open_port() above probed this port and then\n"
         "            # released it; ZMQ only binds it for real here. Every worker\n"
         "            # on a non-driver node builds its response queue at the same\n"
@@ -445,8 +467,6 @@ def _patch_vllm_shm_broadcast_bind_retry(logger) -> None:
         "                except zmq.ZMQError:\n"
         "                    if _nrl_bind_attempt == _nrl_bind_attempts - 1:\n"
         "                        raise\n"
-        "                    from vllm.utils.network_utils import _get_open_port\n"
-        "\n"
         "                    logger.info(\n"
         '                        "Port %s was taken between probe and bind; '
         'retrying.",\n'