diff --git a/hyper_parallel/core/activation_memory/_backend.py b/hyper_parallel/core/activation_memory/_backend.py index ef52e81abd..c874b8911e 100644 --- a/hyper_parallel/core/activation_memory/_backend.py +++ b/hyper_parallel/core/activation_memory/_backend.py @@ -21,6 +21,7 @@ "get_stream_context", "get_current_stream", "new_event", + "record_stream", "no_grad", "preserve_version_counter", "cat", @@ -37,26 +38,58 @@ from torch import Tensor from hyper_parallel.core.utils.communication import get_device_handle +from hyper_parallel.core.utils.simulation import ( + hold_for_simulated_stream, + is_device_runtime_simulated, + simulated_device_handle, +) + + +def _stream_owner(): + """Return the namespace owning streams and events for this process. + + Under logical simulation the process never binds an accelerator, so stream + and event primitives come from the simulated handle: every transfer then + completes in host program order, which preserves the storage lifecycle the + swap state machine expresses while dropping only device-side overlap. + """ + if is_device_runtime_simulated(): + return simulated_device_handle() + return get_device_handle() def new_stream(): """Create a new device stream on the current accelerator.""" - return get_device_handle().Stream() + return _stream_owner().Stream() def get_stream_context(): """Return the stream context manager (``torch.npu.stream`` / ``torch.cuda.stream``).""" - return get_device_handle().stream + return _stream_owner().stream def get_current_stream(): """Return the current device stream.""" - return get_device_handle().current_stream() + return _stream_owner().current_stream() def new_event(): """Create a new device event on the current accelerator.""" - return get_device_handle().Event() + return _stream_owner().Event() + + +def record_stream(tensor, stream) -> None: + """Reserve ``tensor``'s storage for ``stream`` until its pending work ends. + + Under simulation the reference is held until the next simulated wait — + the same point a real caching allocator could first reclaim it — so the + logical lifetime of staging buffers is conservative, never shorter than + the real one. + """ + if is_device_runtime_simulated(): + hold_for_simulated_stream(tensor) + return + tensor.record_stream(stream) def no_grad(): @@ -76,6 +109,10 @@ def cat(tensors, dim=0): def empty_like(tensor, *, dtype=None, device=None, pin_memory=False): """Allocate an uninitialized tensor shaped like *tensor*.""" + if pin_memory and is_device_runtime_simulated(): + # Pinning needs a live device runtime; logically it only affects DMA + # speed, never which host bytes are allocated. + pin_memory = False return torch.empty_like(tensor, dtype=dtype, device=device, pin_memory=pin_memory) @@ -91,8 +128,10 @@ def get_element_size(tensor) -> int: def alloc_tensor_buffer(numel: int, dtype, device="cpu", pin_memory: bool = False): """Allocate an uninitialized 1-D tensor buffer.""" - if pin_memory: + if pin_memory and not is_device_runtime_simulated(): return torch.empty(numel, dtype=dtype, device="cpu", pin_memory=True) + if pin_memory: + return torch.empty(numel, dtype=dtype, device="cpu") return torch.empty(numel, dtype=dtype, device=device) diff --git a/hyper_parallel/core/activation_memory/pinned_memory_pool.py b/hyper_parallel/core/activation_memory/pinned_memory_pool.py index ac31432d6f..89cddbe0f1 100644 --- a/hyper_parallel/core/activation_memory/pinned_memory_pool.py +++ b/hyper_parallel/core/activation_memory/pinned_memory_pool.py @@ -46,9 +46,15 @@ def _valid_positive_int(value: int, name: str) -> None: def _storage_key(tensor: Any) -> int: - """Return a stable identity for a tensor's underlying storage.""" + """Return a stable identity for a tensor's underlying storage. + + Identity of the cached storage PyObject, not ``data_ptr()``: the pool's + buffers are unpinned CPU tensors under logical simulation and would all + report pointer 0. The pool owns every underlying buffer for its whole + lifetime, so the id cannot be recycled while a key is registered. + """ try: - return int(tensor.untyped_storage().data_ptr()) + return id(tensor.untyped_storage()) except (AttributeError, TypeError, ValueError) as exc: raise ValueError("release() expects a tensor with an identifiable storage.") from exc diff --git a/hyper_parallel/core/activation_memory/swap.py b/hyper_parallel/core/activation_memory/swap.py index 32667bcb66..3997059d97 100644 --- a/hyper_parallel/core/activation_memory/swap.py +++ b/hyper_parallel/core/activation_memory/swap.py @@ -76,18 +76,25 @@ def _return_cpu_pinned_buf(buf): _CPU_PINNED_POOL[str(buf.dtype)].append(buf) -def _collect_device_storage_ptrs(tensors: Any) -> Set[int]: - """Collect device storage pointers from a nested tensor structure.""" - storage_ptrs = set() +def _collect_device_storage_ids(tensors: Any) -> Set[int]: + """Collect device storage identities from a nested tensor structure. + + Identity of the cached storage PyObject, not ``data_ptr()``: empty tensors + and FakeTensors report pointer 0 for distinct storages, which would make + every such tensor look like an alias of the protected set. The ids stay + valid while the owning tensors are alive, which the caller guarantees for + the duration of one protect pass. + """ + storage_ids = set() def _collect(x): local_tensor = x.to_local() if isinstance(x, DTensor) else x if isinstance(local_tensor, _backend.Tensor) and str(local_tensor.device).lower() != "cpu": - storage_ptrs.add(local_tensor.untyped_storage().data_ptr()) + storage_ids.add(id(local_tensor.untyped_storage())) return x _backend.tree_map(_collect, tensors) - return storage_ptrs + return storage_ids class SwapTensor: @@ -122,13 +129,21 @@ def __init__(self, val: Any, funcname: str, group_swap: bool = False, cpu_pool=N self.storage_size = 0 def dedup_key(self): - """Return a stable identity key for duplicate-swap detection.""" + """Return a stable identity key for duplicate-swap detection. + + Keyed by the storage PyObject's identity rather than ``data_ptr()``: + the pointer is 0 for every FakeTensor and empty storage, which would + collapse distinct registrations into false duplicates. The SwapTensor + keeps ``self.val`` (and with it the storage object) alive for the + whole group lifecycle, so the id cannot be recycled while the key is + in a group's seen-set. + """ if self._state == self.STATE_NON_TENSOR: return None val_storage = self.val.untyped_storage() return ( str(self.val.device), - val_storage.data_ptr(), + id(val_storage), self.val.storage_offset(), val_storage.size(), tuple(self.val.stride()), @@ -138,11 +153,11 @@ def mark_duplicate_swap(self) -> None: """Mark this wrapper as a duplicate registration in the same swap group.""" self._duplicate_swap = True - def protect_if_aliases(self, alias_storage_ptrs: Set[int]) -> None: + def protect_if_aliases(self, alias_storage_ids: Set[int]) -> None: """Keep tensors that alias externally-owned tensors on device.""" if self._state == self.STATE_NON_TENSOR: return - if self.val.untyped_storage().data_ptr() in alias_storage_ptrs: + if id(self.val.untyped_storage()) in alias_storage_ids: self._keep_on_device = True def get_val(self) -> Any: @@ -394,14 +409,14 @@ def mark_duplicate_swaps(self, seen_keys) -> int: seen_keys.add(dedup_key) return duplicate_count - def protect_alias_storage_ptrs(self, alias_storage_ptrs: Set[int]): + def protect_alias_storage_ids(self, alias_storage_ids: Set[int]): """Avoid offloading swap entries that alias externally-owned storage.""" - if not alias_storage_ptrs: + if not alias_storage_ids: return def _protect_tensor(x): if isinstance(x, SwapTensor): - x.protect_if_aliases(alias_storage_ptrs) + x.protect_if_aliases(alias_storage_ids) return x for storage_list in self.values(): @@ -501,7 +516,7 @@ def __init__(self, group_name: str): self._group_device_buf = None # temp device bufs; cleared after each phase # Persistent dedup set accumulated across add() calls; avoids O(N²) rebuild. # mark_duplicate_swaps mutates it in-place, so new keys are added automatically. - # Reset at wait_load() so stale data_ptrs don't leak into the next iteration. + # Reset at wait_load() so stale storage ids don't leak into the next iteration. self._seen_dedup_keys: set = set() # Per-bucket SwapTensor lists built in _collect_packable_tensors and consumed # in launch_offload, eliminating a redundant pass over _packed_tensor_info. @@ -518,11 +533,11 @@ def add(self, storage): def protect_alias_tensors(self, tensors: Any): """Protect externally-owned tensors from premature offload.""" - alias_storage_ptrs = _collect_device_storage_ptrs(tensors) - if not alias_storage_ptrs: + alias_storage_ids = _collect_device_storage_ids(tensors) + if not alias_storage_ids: return for storage in self._storages: - storage.protect_alias_storage_ptrs(alias_storage_ptrs) + storage.protect_alias_storage_ids(alias_storage_ids) @staticmethod def _bucket_tensor(candidate_buckets, dtype_key, x): @@ -658,7 +673,7 @@ def _protect_device_bufs(group_device_bufs, copy_stream): for buf in group_device_bufs.values(): device = getattr(buf, "device", None) if device is not None and device.type not in ("cpu", "meta"): - buf.record_stream(copy_stream) + _backend.record_stream(buf, copy_stream) def _acquire_bucket_cpu_buf(self, bucket_key, bucket): """Acquire the pinned CPU buffer that receives one bucket's D2H copy.""" diff --git a/hyper_parallel/core/dtensor/dtensor_base.py b/hyper_parallel/core/dtensor/dtensor_base.py index ddfd39bd54..49d8276b86 100644 --- a/hyper_parallel/core/dtensor/dtensor_base.py +++ b/hyper_parallel/core/dtensor/dtensor_base.py @@ -17,6 +17,109 @@ from typing import Tuple, Dict, Any, Optional import torch from torch import Tensor +from torch._subclasses.fake_tensor import FakeTensor, unset_fake_temporarily + + +def _make_shell_subclass(cls, local_tensor: Tensor) -> Tensor: + """Wrap ``local_tensor`` in subclass ``cls``, aliasing or shadowing its storage. + + Real local tensors share their storage with the wrapper (the normal path, + byte-identical to a plain ``_make_subclass``). ``Tensor._make_subclass`` + cannot re-wrap a FakeTensor because a FakeTensor is already a Tensor + subclass, so storage-less locals get a meta shell with identical metadata + instead — built as a :func:`_fake_local_variant` of ``cls`` so every + FakeTensor-specific accessor lives on that variant class and the real + training path never executes a fake-aware branch. + + Args: + cls: The DTensor subclass to instantiate. + local_tensor: The local shard the wrapper represents. + + Returns: + Tensor: An uninitialized ``cls`` instance (``__init_data__`` pending). + """ + if isinstance(local_tensor, FakeTensor): + # Inside an active FakeTensorMode even meta factories return + # FakeTensors, which _make_subclass rejects; build the shell outside. + with unset_fake_temporarily(): + shell = torch.empty_strided( + local_tensor.size(), + local_tensor.stride(), + dtype=local_tensor.dtype, + device="meta", + ) + return Tensor._make_subclass( + _fake_local_variant(cls), shell, local_tensor.requires_grad + ) + return Tensor._make_subclass(cls, local_tensor, local_tensor.requires_grad) + + +_FAKE_LOCAL_VARIANTS: Dict[type, type] = {} + + +def _fake_local_variant(cls: type) -> type: + """Return the meta-shell variant of DTensor subclass ``cls``. + + The variant overrides exactly the accessors whose base implementation + assumes the wrapper owns real storage. Keeping these overrides on a + separate class (instead of ``if is_meta_shell`` branches in the base) + means real-tensor instances resolve the unmodified base methods — the + hot ``grad``/``data``/``zero_grad`` paths cost nothing extra when + dry-run is not in use. Variants are cached per subclass and are + idempotent (a variant is its own variant), so ``self.__class__(...)`` + reconstruction in ``detach``/``to``/``type`` stays type-stable, which + ``nn.Parameter`` requires of ``detach``. + """ + if getattr(cls, "_is_fake_local_variant", False): + return cls + variant = _FAKE_LOCAL_VARIANTS.get(cls) + if variant is None: + variant = type(f"_FakeLocal{cls.__name__}", (cls,), { + "_is_fake_local_variant": True, + "is_meta_shell": _variant_is_meta_shell, + "grad": property(_variant_grad_get, _variant_grad_set), + "data": property(_variant_data_get, _variant_data_set), + }) + _FAKE_LOCAL_VARIANTS[cls] = variant + return variant + + +def _variant_is_meta_shell(self) -> bool: + """Meta-shell wrappers have no storage of their own.""" + return True + + +def _variant_grad_get(self) -> Optional[Tensor]: + """Return the stored DTensor gradient wrapper while it owns the leaf's grad.""" + local_grad = self._local_tensor.grad + wrapper = getattr(self, "_grad_dtensor_wrapper", None) + if wrapper is not None and wrapper._local_tensor is local_grad: + return wrapper + return local_grad + + +def _variant_grad_set(self, value: Optional[Tensor]) -> None: + """Store DTensor gradients on the local leaf; autograd rejects meta-device grads.""" + if isinstance(value, DTensorBase) and value.is_meta_shell(): + self._local_tensor.grad = value._local_tensor + self._grad_dtensor_wrapper = value + return + self._local_tensor.grad = value + if getattr(self, "_grad_dtensor_wrapper", None) is not None: + self._grad_dtensor_wrapper = None + + +def _variant_data_get(self): + """A meta shell owns no storage; the local tensor's data is the payload.""" + with torch._C.DisableTorchFunctionSubclass(): + return self._local_tensor.data + + +def _variant_data_set(self, value) -> None: + """Rebind only the local tensor; the meta shell cannot adopt device storage.""" + local_value = value.to_local() if isinstance(value, DTensorBase) else value + with torch._C.DisableTorchFunctionSubclass(): + Tensor.data.__set__(self._local_tensor, local_value) class DTensorBase(Tensor): @@ -40,7 +143,7 @@ def __new__(cls, local_tensor, device_mesh=None, placements=None, layout=None, s """ if isinstance(local_tensor, DTensorBase): # Copy from existing DTensorBase — use alias_placements to preserve multi-axis ordering - t = Tensor._make_subclass(cls, local_tensor._local_tensor, local_tensor._local_tensor.requires_grad) + t = _make_shell_subclass(cls, local_tensor._local_tensor) copy_placements = local_tensor.layout.alias_placements if local_tensor.layout else local_tensor.placements t.__init_data__( local_tensor._local_tensor, @@ -56,7 +159,8 @@ def __new__(cls, local_tensor, device_mesh=None, placements=None, layout=None, s raise ValueError("placements is None, must provide placements") # Create Tensor subclass instance, sharing local_tensor's underlying storage - t = Tensor._make_subclass(cls, local_tensor, local_tensor.requires_grad) + # (or shadowing it with a meta shell when the local is a FakeTensor). + t = _make_shell_subclass(cls, local_tensor) t.__init_data__(local_tensor, device_mesh, placements, layout, shape) return t @@ -85,6 +189,12 @@ def __torch_function__( Any: The result of the dispatched operation, typically a DTensor or tuple of DTensors. """ kwargs = kwargs or {} + if func in (torch.numel, Tensor.numel): + # numel must keep logical global semantics (DTensor.numel) even for + # functional / unbound call forms and under SkipDTensorDispatch, + # where the bypass would otherwise leak the local shard's capacity. + tensor = args[0] if args else kwargs["input"] + return tensor.numel() # pylint: disable=C0415 from hyper_parallel.core.shard._op_dispatch import _OP_DISPATCHER out = _OP_DISPATCHER.dispatch(func, args, kwargs) @@ -110,6 +220,15 @@ def grad(self, value: Optional[Tensor]) -> None: """ self._local_tensor.grad = value + def is_meta_shell(self) -> bool: + """Return whether this wrapper is a storage-less meta shell. + + Always False here; the ``_fake_local_variant`` subclass used for + FakeTensor locals overrides this (and the storage-touching accessors) + so the real-tensor path carries no fake-aware branches. + """ + return False + @property def requires_grad(self) -> bool: """ @@ -344,6 +463,19 @@ def data_ptr(self) -> int: # Force return local_tensor's data pointer (ensure address consistency) return self._local_tensor.data_ptr() + def untyped_storage(self) -> torch.UntypedStorage: + """ + Get the untyped storage of the local tensor. + + Like ``data_ptr``, storage queries answer for the local shard: the + wrapper either aliases the same storage (real locals) or holds a + storage-less meta shell (FakeTensor locals). + + Returns: + torch.UntypedStorage: The local tensor's untyped storage. + """ + return self._local_tensor.untyped_storage() + def numel(self) -> int: """ Get the total number of elements in this tensor. @@ -375,7 +507,7 @@ def to(self, *args, **kwargs): DTensorBase: A new DTensor with the converted local tensor. """ new_local = self._local_tensor.to(*args, **kwargs) - new_dt = Tensor._make_subclass(type(self), new_local, new_local.requires_grad) + new_dt = _make_shell_subclass(type(self), new_local) new_dt.__init_data__( new_local, self._device_mesh, diff --git a/hyper_parallel/core/dtensor/layout.py b/hyper_parallel/core/dtensor/layout.py index 3d6c2ce76b..79436490f8 100644 --- a/hyper_parallel/core/dtensor/layout.py +++ b/hyper_parallel/core/dtensor/layout.py @@ -316,12 +316,17 @@ def __call__(self, *alias_tensor_map): return self._process_alias_layout(obj, alias_tensor_map) def __deepcopy__(self, memo): - """Deep copy layout without rebuilding the underlying device mesh.""" + """Deep copy layout without rebuilding the underlying device mesh. + + ``_mesh`` is shared by identity: a DeviceMesh is process-wide topology + (created once, never mutated in place) and deep-copying it would clone + its mesh tensor and process-group bookkeeping on every layout copy. + """ cls = self.__class__ result = cls.__new__(cls) memo[id(self)] = result for k, v in self.__dict__.items(): - setattr(result, k, copy.deepcopy(v, memo)) + setattr(result, k, v if k == "_mesh" else copy.deepcopy(v, memo)) return result @staticmethod diff --git a/hyper_parallel/core/fully_shard/hsdp_param.py b/hyper_parallel/core/fully_shard/hsdp_param.py index 0c29313c85..e8aaa0f135 100755 --- a/hyper_parallel/core/fully_shard/hsdp_param.py +++ b/hyper_parallel/core/fully_shard/hsdp_param.py @@ -925,11 +925,11 @@ def _is_same_sharded_local_tensor(self, local_tensor: torch.Tensor) -> bool: """Return whether communication storage already aliases the local tensor.""" if not isinstance(self._sharded_param_data, torch.Tensor): return False - sharded_data_ptr = self._sharded_param_data.untyped_storage().data_ptr() + # Storage identity instead of data_ptr: empty shards and FakeTensors + # report data_ptr 0 for distinct storages, which faked aliasing here. return ( - # Empty shards may have a zero data pointer and must still be rebuilt. - sharded_data_ptr > 0 - and sharded_data_ptr == local_tensor.untyped_storage().data_ptr() + self._sharded_param_data.untyped_storage() + is local_tensor.untyped_storage() ) def _validate_reset_local_tensor( @@ -1037,9 +1037,11 @@ def reset_sharded_param(self) -> None: def _update_shardedparam_storage_forcely(self,): sharded_param_data = self.sharded_param.data local_tensor_data = self.sharded_param._local_tensor.data + # Storage identity instead of data_ptr: data_ptr is 0 for empty shards + # and FakeTensors, so distinct storages would look identical here. if ( sharded_param_data.device != local_tensor_data.device - or sharded_param_data.data_ptr() != local_tensor_data.data_ptr() + or sharded_param_data.untyped_storage() is not local_tensor_data.untyped_storage() ): local_tensor_data.requires_grad_(self.sharded_param.requires_grad) # TensorImpl keeps storage_ptr, storage_offset, sizes, strides, dtype and metadatas diff --git a/hyper_parallel/core/fully_shard/hsdp_state.py b/hyper_parallel/core/fully_shard/hsdp_state.py index cd7eb80cd7..a1f2fb2de1 100755 --- a/hyper_parallel/core/fully_shard/hsdp_state.py +++ b/hyper_parallel/core/fully_shard/hsdp_state.py @@ -35,6 +35,7 @@ ) from hyper_parallel.core.fully_shard.hsdp_param import HSDPParamV2 from hyper_parallel.core.fully_shard.param_group import HSDPParamGroup, AllReduceParamGroup +from hyper_parallel.core.utils.simulation import is_device_runtime_simulated logger = get_logger("FSDP") @@ -622,6 +623,10 @@ def set_requires_all_reduce(self, requires_all_reduce: bool) -> None: def _sync_current_stream_if_needed(self, need_synchronize): if need_synchronize: + if is_device_runtime_simulated(): + # Logical simulation: the D2H grad copy completed in program + # order; there is no device stream to drain. + return if self.device.type == "npu": torch.npu.current_stream().synchronize() elif self.device.type == "cuda": diff --git a/hyper_parallel/core/fully_shard/param_group.py b/hyper_parallel/core/fully_shard/param_group.py index 935fc9259a..40419b6927 100644 --- a/hyper_parallel/core/fully_shard/param_group.py +++ b/hyper_parallel/core/fully_shard/param_group.py @@ -141,9 +141,12 @@ def is_flat_buffer_valid(self) -> bool: """ if self.flat_param_buffer is None: return False - flat_storage_ptr = self.flat_param_buffer.untyped_storage().data_ptr() + # Storage identity instead of data_ptr: empty shards and FakeTensors + # report data_ptr 0 for distinct storages, which would validate a + # buffer the parameters do not actually alias. + flat_storage = self.flat_param_buffer.untyped_storage() return all( - hsdp_param._sharded_param_data.untyped_storage().data_ptr() == flat_storage_ptr + hsdp_param._sharded_param_data.untyped_storage() is flat_storage for hsdp_param in self.hsdp_params ) diff --git a/hyper_parallel/core/optimizer/muon.py b/hyper_parallel/core/optimizer/muon.py index f99d43ff08..ab83c3b908 100644 --- a/hyper_parallel/core/optimizer/muon.py +++ b/hyper_parallel/core/optimizer/muon.py @@ -33,6 +33,7 @@ build_owner_by_size, ) from hyper_parallel.core.optimizer.muon_shard import ( + _comm_device_for_group, _debug_param_shard_metadata, build_pad_ns_inputs, build_param_shard_metadata_for_group, @@ -42,6 +43,7 @@ logger = logging.getLogger(__name__) + # Legacy: single-coefficient quintic NS (Keller Jordan / Moonlight). # Same (a, b, c) applied every step; typically 5 steps. _NS_LEGACY_COEFFS: Tuple[float, float, float] = (3.4445, -4.7750, 2.0315) @@ -763,7 +765,7 @@ def _reshape_ns_input( ) for reshaped_input in reshaped_inputs: - if reshaped_input.untyped_storage().data_ptr() != working_input.untyped_storage().data_ptr(): + if reshaped_input.untyped_storage() is not working_input.untyped_storage(): raise ValueError("reshape_fn must return views that share storage with the working NS input tensor") return working_input, reshaped_inputs @@ -788,7 +790,7 @@ def restore_view_updates(updates: List[torch.Tensor], output: torch.Tensor) -> N """Write reshaped NS updates back into the views and the working input.""" for reshaped_input, update in zip(reshaped_inputs, updates): reshaped_input.copy_(update.contiguous().view_as(reshaped_input)) - if output.untyped_storage().data_ptr() != working_input.untyped_storage().data_ptr(): + if output.untyped_storage() is not working_input.untyped_storage(): output.copy_(working_input) return working_input, NSInputTransform(tensors=reshaped_inputs, restore=restore_view_updates) @@ -977,7 +979,7 @@ def _compute_batched_ns_updates_fast( for param in p_list: update = ns_inputs[param].view(state["origin_shapes"][param]) working_input = state["working_inputs"][param] - if working_input.untyped_storage().data_ptr() != update.untyped_storage().data_ptr(): + if working_input.untyped_storage() is not update.untyped_storage(): update.copy_(working_input) state["restored"][param] = update return state["restored"] @@ -1052,9 +1054,7 @@ def _compute_batched_ns_updates_with_transform( for p in p_list: reshaped_update = ns_inputs[p].view(state["origin_shapes"][p]) - - if (state["working_inputs"][p].untyped_storage().data_ptr() - != reshaped_update.untyped_storage().data_ptr()): + if state["working_inputs"][p].untyped_storage() is not reshaped_update.untyped_storage(): reshaped_update.copy_(state["working_inputs"][p]) state["restored"][p] = reshaped_update @@ -1073,6 +1073,12 @@ def _fused_broadcast_and_apply( weight_decay = group["weight_decay"] shard_sizes, local_coords, shard_pgs, total_shard_size = self._get_shard_info(hsdp_assign) device = to_local_if_dtensor(valid_params[0].data).device + # The relay broadcast runs on the shard groups' backend; with CPU + # offload the parameter locals are on host, so stage the pack buffers + # on the first real shard group's device instead. + first_pg = next((pg for pg in shard_pgs if pg is not None), None) + if first_pg is not None: + device = _comm_device_for_group(first_pg, device) coord_groups: Dict[Tuple[int, ...], List[torch.nn.Parameter]] = defaultdict(list) for p in valid_params: @@ -1132,6 +1138,10 @@ def _fused_broadcast_and_apply( local_param = to_local_if_dtensor(p.data) all_local_params.append(local_param) + if update_to_apply.device != local_param.device: + # CPU offload: updates were broadcast on the accelerator + # while the parameter local lives on host. + update_to_apply = update_to_apply.to(local_param.device) all_update_shards.append(update_to_apply.view(local_param.shape)) if not all_local_params: diff --git a/hyper_parallel/core/optimizer/muon_shard.py b/hyper_parallel/core/optimizer/muon_shard.py index 91e93aa1b0..8b3bb8e3dc 100644 --- a/hyper_parallel/core/optimizer/muon_shard.py +++ b/hyper_parallel/core/optimizer/muon_shard.py @@ -32,6 +32,7 @@ import torch.distributed as dist from hyper_parallel.core.optimizer.dtensor_compat import to_local_if_dtensor +from hyper_parallel.core.utils import simulation from hyper_parallel.core.optimizer.sharding_category import ( HSDPCommGroup, ParamLayoutSpec, @@ -126,6 +127,31 @@ def _debug_param_shard_metadata(hsdp_group, param_to_meta: Dict[torch.nn.Paramet ) +def _comm_device_for_group( + shard_pg: Optional[dist.ProcessGroup], + device: torch.device, +) -> torch.device: + """Return a device the group's backend can run collectives on. + + With CPU offload the parameter locals live on host while the shard group + is an accelerator backend (HCCL/NCCL) with no CPU path; the metadata + gather must then run on the accelerator. Under logical simulation the + group is a fake backend and the simulated accelerator plays that role, so + the staging allocations land where a real run puts them. + """ + if device.type != "cpu" or shard_pg is None: + return device + backend = dist.get_backend(shard_pg) + if backend == "hccl": + return torch.device("npu", torch.npu.current_device()) + if backend == "nccl": + return torch.device("cuda", torch.cuda.current_device()) + lied_accelerator = simulation.simulated_accelerator() + if backend == "fake" and lied_accelerator is not None: + return torch.device(lied_accelerator, 0) + return device + + def _allgather_shapes_for_group( local_shapes: List[Tuple[int, ...]], shard_pg: Optional[dist.ProcessGroup], @@ -136,6 +162,7 @@ def _allgather_shapes_for_group( if shard_pg is None or shard_size <= 1: return [[tuple(shape) for shape in local_shapes]] + device = _comm_device_for_group(shard_pg, device) shape_tensor = torch.tensor(local_shapes, dtype=torch.int64, device=device) gathered = torch.empty((shard_size, *shape_tensor.shape), dtype=torch.int64, device=device) dist.all_gather_into_tensor(gathered.view(-1), shape_tensor.contiguous().view(-1), group=shard_pg) @@ -380,10 +407,17 @@ def _pack_and_allgather( shard_size: int, buffer_cache: Optional[Dict], ) -> torch.Tensor: - """Pack local shards into one buffer, all-gather, return gathered view.""" + """Pack local shards into one buffer, all-gather, return gathered view. + + The pack and gather buffers live on a device the group's backend supports: + with CPU offload the shards live on host while the shard group is an + accelerator backend, so the collective is staged there and the gathered + view is brought back to the shards' device for the NS computation. + """ + comm_device = _comm_device_for_group(shard_pg, device) pack_buffer = _get_or_alloc_buffer( - buffer_cache, ("fused_allgather", axis_idx, dtype, device), total_padded_numel, - dtype, device, + buffer_cache, ("fused_allgather", axis_idx, dtype, comm_device), total_padded_numel, + dtype, comm_device, )[:total_padded_numel] pack_buffer.zero_() @@ -392,12 +426,14 @@ def _pack_and_allgather( gathered_buffer = _get_or_alloc_buffer( buffer_cache, - ("fused_allgather_out", axis_idx, dtype, device), + ("fused_allgather_out", axis_idx, dtype, comm_device), total_padded_numel * shard_size, - dtype, device, + dtype, comm_device, )[:total_padded_numel * shard_size] dist.all_gather_into_tensor(gathered_buffer, pack_buffer, group=shard_pg) + if comm_device != device: + gathered_buffer = gathered_buffer.to(device) return gathered_buffer.view(shard_size, total_padded_numel) diff --git a/hyper_parallel/core/optimizer/swap_optimizer_base.py b/hyper_parallel/core/optimizer/swap_optimizer_base.py index 6c28a81341..14bdea6a8e 100644 --- a/hyper_parallel/core/optimizer/swap_optimizer_base.py +++ b/hyper_parallel/core/optimizer/swap_optimizer_base.py @@ -28,6 +28,8 @@ import torch # pylint: disable=forbidden-backend-import from torch.optim import optimizer as torch_optimizer # pylint: disable=forbidden-backend-import +from hyper_parallel.core.utils import simulation + logger = logging.getLogger(__name__) # Logical optimizer-state keys every swap family in this codebase understands. @@ -627,7 +629,12 @@ def _pack_dtype_slots(self, dtype: Any, dtype_slots: Sequence[SwapSlot]) -> Any: afterwards, because the host view now owns the only valid values. """ total_numel = sum(slot.numel for slot in dtype_slots) - host_buffer = torch.empty(total_numel, dtype=dtype, device="cpu", pin_memory=True) + try: + host_buffer = torch.empty(total_numel, dtype=dtype, device="cpu", pin_memory=True) + except RuntimeError: + # No live device runtime (logical simulation on a zero-device + # host); pinning only affects DMA speed, not the byte account. + host_buffer = torch.empty(total_numel, dtype=dtype, device="cpu") host_offset = 0 for slot in dtype_slots: flat_view = host_buffer.narrow(0, host_offset, slot.numel) @@ -778,6 +785,11 @@ def release_device_storage(self, slot: SwapSlot) -> None: @staticmethod def device_handle() -> Any: """Return the Torch device module (``torch.npu``) that owns streams and events.""" + if simulation.is_device_runtime_simulated(): + # Logical simulation: streams and events complete in program + # order, keeping the swap pipeline's host-visible storage + # lifecycle without touching an accelerator runtime. + return simulation.simulated_device_handle() try: return getattr(torch, _DEVICE_TYPE) except AttributeError as error: @@ -868,7 +880,8 @@ def begin_packed_step(self, batches: Sequence[Sequence[UpdateUnit]]) -> None: # FSDP may leave gradient reductions/reshards queued on auxiliary # streams. A current-stream event cannot cover those streams, so the # device-wide synchronization is required before optimizer reads state. - getattr(torch, first_slot.device.type).synchronize(first_slot.device) + if not simulation.is_device_runtime_simulated(): + getattr(torch, first_slot.device.type).synchronize(first_slot.device) self._packed_tail_event = None self._packed_batch_plans = [self._build_packed_batch_plan(batch) for batch in batches] dtype_layouts, total_bytes, device = self._packed_layout(self._packed_batch_plans) diff --git a/hyper_parallel/core/utils/clip_grad.py b/hyper_parallel/core/utils/clip_grad.py index 7474cd443d..e57feb3bd9 100644 --- a/hyper_parallel/core/utils/clip_grad.py +++ b/hyper_parallel/core/utils/clip_grad.py @@ -49,6 +49,7 @@ import torch.distributed as dist from hyper_parallel.core.dtensor.placement_types import Partial +from hyper_parallel.core.utils import simulation if TYPE_CHECKING: from hyper_parallel.core.dtensor.layout import Layout @@ -219,7 +220,7 @@ def dense_grad_norm_sq_and_refs(model: torch.nn.Module, for name in getattr(getattr(mesh_context, "device_mesh", None), "mesh_dim_names", ()) or (): axis = mesh_context.device_mesh[name] if axis.size() > 1: - dist.all_reduce(local_sq, op=dist.ReduceOp.SUM, group=axis.get_group()) + _all_reduce_scalar(local_sq, dist.ReduceOp.SUM, axis.get_group()) return local_sq, refs @@ -321,6 +322,34 @@ def _param_device(param: torch.Tensor) -> torch.device: return param.device +def _all_reduce_scalar(scalar: torch.Tensor, reduce_op, group) -> torch.Tensor: + """All-reduce a norm scalar on a device the group's backend supports. + + With CPU offload the gradients — and therefore the local norm scalar — + live on host, while the process group is an accelerator backend + (HCCL/NCCL) with no CPU path. The scalar is staged on the backend's + device for the collective and brought back, which preserves the caller's + device so subsequent arithmetic stays where the gradients are. + """ + try: + backend = dist.get_backend(group) + except (ValueError, RuntimeError): + # Test doubles answer every device; reduce in place. + backend = None + backend_device = {"hccl": "npu", "nccl": "cuda"}.get(backend) + if backend_device is None and backend == "fake": + # Logical simulation: stage where a real run would, so the simulated + # allocation story matches the accelerator run byte for byte. + backend_device = simulation.simulated_accelerator() + if backend_device is None or scalar.device.type == backend_device: + dist.all_reduce(scalar, op=reduce_op, group=group) + return scalar + staged = scalar.to(backend_device) + dist.all_reduce(staged, op=reduce_op, group=group) + scalar.copy_(staged) + return scalar + + def _get_grad_obj(param: torch.nn.Parameter) -> Optional[torch.Tensor]: """Return the gradient object for *param*. @@ -608,10 +637,7 @@ def _total_norm_inf( # pylint: disable=R0913,R0917 if mesh_id is not None: mesh = mesh_cache[mesh_id] for dim in shard_dims: - dist.all_reduce( - local_norm, op=reduce_op, - group=mesh.get_group(dim), - ) + _all_reduce_scalar(local_norm, reduce_op, mesh.get_group(dim)) group_norms.append(local_norm) if not group_norms: if norm_type == -math.inf: @@ -629,10 +655,7 @@ def _total_norm_sum(grad_groups, norm_type, mesh_cache, device): if mesh_id is not None: mesh = mesh_cache[mesh_id] for dim in shard_dims: - dist.all_reduce( - local_val, op=dist.ReduceOp.SUM, - group=mesh.get_group(dim), - ) + _all_reduce_scalar(local_val, dist.ReduceOp.SUM, mesh.get_group(dim)) total.add_(local_val) return total @@ -732,7 +755,7 @@ def _total_norm_fsdp2_aligned(grad_groups, norm_type, mesh_cache, device, local_p = torch.tensor(0.0, device=device, dtype=torch.float32) for group in sig_groups[sig]: - dist.all_reduce(local_p, op=dist.ReduceOp.SUM, group=group) + _all_reduce_scalar(local_p, dist.ReduceOp.SUM, group) total_p = total_p + local_p diff --git a/hyper_parallel/core/utils/simulation.py b/hyper_parallel/core/utils/simulation.py new file mode 100644 index 0000000000..ae258e0ee2 --- /dev/null +++ b/hyper_parallel/core/utils/simulation.py @@ -0,0 +1,184 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Process-wide device-runtime simulation switch with no-op stream primitives. + +Logical simulation (the dry-run tool) executes unmodified trainer and core +code over FakeTensors without ever binding an accelerator or initializing its +runtime. Components that schedule asynchronous transfers — activation swap, +optimizer-state swap, FSDP CPU offload — fetch their stream and event +primitives through seams that consult this switch. Under simulation every +transfer degenerates to its host-visible program order: the same storages are +allocated, copied, resized, and released; only device-side overlap disappears, +which is exactly the part a logical memory tracker does not account anyway. +""" + +import contextlib +from typing import Any, Optional + +__all__ = [ + "SimulatedEvent", + "SimulatedStream", + "hold_for_simulated_stream", + "is_device_runtime_simulated", + "release_simulated_stream_holds", + "simulate_device_runtime", + "simulated_accelerator", + "simulated_device_handle", +] + +_SIMULATED_DEVICE_RUNTIME = False +_SIMULATED_ACCELERATOR: Optional[str] = None +_PENDING_STREAM_HOLDS: list = [] + + +def simulate_device_runtime(enabled: bool = True, accelerator: Optional[str] = None) -> None: + """Toggle no-op device effects for logical (FakeTensor) simulation. + + Args: + enabled: Whether device-side effects become no-ops. + accelerator: Device type the simulation lies as (``npu``/``cuda``), or + ``None`` for plain CPU simulation. Components that pick a + collective staging device consult it so simulated runs allocate + communication buffers on the same (fake) device real runs do. + """ + global _SIMULATED_DEVICE_RUNTIME, _SIMULATED_ACCELERATOR # pylint: disable=global-statement + _SIMULATED_DEVICE_RUNTIME = bool(enabled) + _SIMULATED_ACCELERATOR = accelerator if enabled else None + if not enabled: + _PENDING_STREAM_HOLDS.clear() + + +def is_device_runtime_simulated() -> bool: + """Return whether device-side effects are currently simulated.""" + return _SIMULATED_DEVICE_RUNTIME + + +def simulated_accelerator() -> Optional[str]: + """Return the accelerator type the simulation lies as, or ``None``.""" + return _SIMULATED_ACCELERATOR + + +def hold_for_simulated_stream(tensor: Any) -> None: + """Keep ``tensor`` alive until the next simulated synchronization point. + + This is the simulation analogue of ``Tensor.record_stream``: a real + caching allocator retains a staging buffer until the recorded stream's + pending work completes, even after the last Python reference dies. The + stand-in holds the reference until an event wait or stream synchronize is + observed, so the logical lifetime is conservative (never shorter than the + real one) without modeling stream timing. + """ + _PENDING_STREAM_HOLDS.append(tensor) + + +def release_simulated_stream_holds() -> None: + """Drop stream-held references; called from simulated waits.""" + _PENDING_STREAM_HOLDS.clear() + + +class SimulatedEvent: + """Event stand-in whose completion is always immediate. + + Host code that records, waits on, or queries an event observes the same + control flow as a synchronous device: ``query`` answers ``True`` so + event-gated reuse (for example pinned-buffer pools) proceeds eagerly. + Waits release any stream-held staging references (see + :func:`hold_for_simulated_stream`) — the same point a real allocator would + be allowed to reclaim them. + """ + + def record(self, stream: Optional[Any] = None) -> "SimulatedEvent": + """Accept any stream; completion is immediate.""" + del stream + return self + + def wait(self, stream: Optional[Any] = None) -> None: + """Accept any stream; pending stream holds are now reclaimable.""" + del stream + release_simulated_stream_holds() + + def query(self) -> bool: + """Report the event as elapsed.""" + return True + + def synchronize(self) -> None: + """Pending stream holds are now reclaimable.""" + release_simulated_stream_holds() + + +class SimulatedStream: + """Stream stand-in on which every submission completes in program order.""" + + def synchronize(self) -> None: + """Pending stream holds are now reclaimable.""" + release_simulated_stream_holds() + + def query(self) -> bool: + """Report the stream as drained.""" + return True + + def wait_event(self, event: Any) -> None: + """Accept any event; pending stream holds are now reclaimable.""" + del event + release_simulated_stream_holds() + + def wait_stream(self, stream: Any) -> None: + """Accept any stream; pending stream holds are now reclaimable.""" + del stream + release_simulated_stream_holds() + + def record_event(self, event: Optional[Any] = None) -> Any: + """Return an elapsed event.""" + return event if event is not None else SimulatedEvent() + + +class _SimulatedDeviceHandle: + """``torch.``-shaped namespace whose effects are no-ops. + + Only the stream/event/synchronize surface is provided on purpose: a + simulated process must never reach device state queries that have no + meaningful logical answer, and an ``AttributeError`` names the gap. + """ + + Stream = SimulatedStream + Event = SimulatedEvent + + def __init__(self) -> None: + self._current_stream = SimulatedStream() + + def current_stream(self, device: Optional[Any] = None) -> SimulatedStream: + """Return the process-wide simulated current stream.""" + del device + return self._current_stream + + @staticmethod + def stream(stream: Optional[Any] = None) -> Any: + """Return a context manager that activates ``stream`` logically.""" + del stream + return contextlib.nullcontext() + + @staticmethod + def synchronize(device: Optional[Any] = None) -> None: + """Pending stream holds are now reclaimable.""" + del device + release_simulated_stream_holds() + + +_SIMULATED_DEVICE_HANDLE = _SimulatedDeviceHandle() + + +def simulated_device_handle() -> _SimulatedDeviceHandle: + """Return the process-wide simulated device handle.""" + return _SIMULATED_DEVICE_HANDLE diff --git a/hyper_parallel/tools/dryrun/DESIGN.md b/hyper_parallel/tools/dryrun/DESIGN.md new file mode 100644 index 0000000000..793fd517c2 --- /dev/null +++ b/hyper_parallel/tools/dryrun/DESIGN.md @@ -0,0 +1,273 @@ +# HyperParallel Dry-run Design + +## Goal + +Dry-run drives the unmodified training step — `TextTrainer.train_step`, the +real loss aggregation, gradient normalization, clipping, and optimizer +sequence — over FakeTensors. It reads the first real CPU batch for shape and +token statistics, then executes one logical training step without loading +model weights, allocating device memory, binding an accelerator, or creating +real communication domains. + +The guiding principle is **identical by construction, not by imitation**: +dry-run owns only the environment (fake process group, FakeTensor +materialization, host-scalar resolution, device-effect suppression); the step +logic itself is the trainer's own code. Whatever the trainer's step loop does +— per-micro-step FSDP gradient-sync flags, reshard scheduling, aux-loss +binding, token-weighted global loss — dry-run inherits automatically. + +The tracker records tensor storage lifetimes only. It does not classify tensors +as parameters, gradients, optimizer state, activations, forward, backward, or +recompute. The snapshot follows the `segments` and `device_traces` shape used by +PyTorch memory visualization. + +**Process-lifetime contract**: dry-run owns the process. It installs +process-local compatibility patches, enables the simulated device runtime, and +disables the FakeTensor dispatch cache for the remainder of the process. The +supported model is invoke → simulate → write snapshots → exit; running normal +training in the same process afterward is outside the contract. + +## Package Layout + +All dry-run implementation lives under `hyper_parallel/tools/dryrun`: + +- `runner.py`: environment preparation and real-trainer orchestration. +- `runtime.py`: value replay, distributed emulation, host-scalar shim, + batch replay iterator, and tensor collection. +- `memory.py`: logical storage tracker and snapshot writer. +- `compatibility.py`: scoped AutoModel/optimizer-subgroup/Muon/CSA2 patches. +- `operator_support.py`: built-in and user-defined FakeTensor kernels. +- `config.py`: YAML configuration dataclasses. +- `data.py`: one-shot CPU data probe. +- `examples/deepseek_v41.py`: DeepSeek-specific scoped replay handlers. + +The reconciliation ST lives in `tests/torch/tools/test_dryrun_memory_sync.py`: +it runs one recipe through both the simulator and a real 8-card training step, +then asserts the logical peak stays inside a tolerance band of the real trace +peak and that every large real allocation call site appears in the dry-run +trace. A step-loop divergence surfaces there as a named `file:line` before it +is visible in the peak. + +## Configuration Interface + +```yaml +dry_run: + enabled: true + target_device: npu + ranks: [0, 7] + virtual_world_size: 8 + snapshot: + enabled: true + path: outputs/dryrun/deepseek_v41 + capture_stacks: true + stack_depth: 64 +``` + +`ranks` defaults to `[0]`. `virtual_world_size` defaults to 1 without torchrun. +`target_device` states the accelerator the plan is for and takes precedence +over environment detection, so capacity planning runs on hosts without that +accelerator — including hosts with no accelerator at all (simulation then +falls back to CPU FakeTensors and reports `simulation_device` in metadata; +the logical peak is identical to the accelerator-lying simulation). +`snapshot.path` is an output root, not a file template. Dry-run creates: + +```text +/rank_/memory.pickle +``` + +No separate report or JSON file is generated. Run metadata is stored under the +snapshot `metadata` key. Progress and errors use standard Python logging. + +## Launch Modes + +Single process, sequential selected ranks: + +```bash +python -m hyper_parallel.tools.dryrun config.yaml +``` + +Existing torchrun launch: + +```bash +torchrun --nproc_per_node=8 -m hyper_parallel.tools.dryrun config.yaml +``` + +All torchrun workers parse the config, but only listed ranks build and execute +the fake model. In torchrun mode, a configured `virtual_world_size` must equal +`WORLD_SIZE`. + +## Execution Principle + +1. Resolve and validate the dry-run config and selected ranks. +2. Build a CPU data path through the real TextTrainer data stages and save the + first batch structure and token counts. +3. Initialize a fake process group with the configured logical topology and + enable the simulated device runtime (`set_device`/`synchronize`/ + `empty_cache` and allocator history become no-ops; the process never binds + an accelerator). +4. Build and parallelize the model without materializing or loading weights. +5. Replace meta state with FakeTensors. +6. Construct the real BaseTrainer state through its own build stages (loss, + optimizer, lr scheduler, callbacks, model-integration session) and run one + unmodified `TextTrainer.train_step` over a replayed mock batch. +7. Track storage allocation, alias, resize, and release. +8. Write one native-compatible protocol-4 pickle per simulated rank. + +## Host Scalars + +The real step loop reads a handful of host scalars (`loss.item()` for logging +accumulators, token counts for the gradient-normalization denominator, +`grad_norm` casts for diagnostics). FakeTensors have no values, so a scoped +`TorchFunctionMode` shim resolves `item`/`__float__`/`__int__`/`__bool__` on +FakeTensors: logging-only scalars read as zero without touching tensor shapes, +while token counts — the only value-bearing quantities the step needs — are +pinned to the true values the CPU probe observed. Real tensors pass through +untouched. + +## Value-Dependent Replay + +FakeTensors preserve shape and dtype but not runtime values. Data-dependent +operations therefore cannot always execute unchanged. + +MoE forced balancing (`fix_router=True`) fixes the intended expert assignment, +but it is not sufficient by itself. The generic EP path still needs concrete +values for: + +- `argsort` permutations. +- Expert token counts. +- `.tolist()` host integers. +- Value-sized slicing and output shapes. + +Registered value-dependency handlers automatically discover supported targets. +Each handler declares a default replay path. The built-in `moe_routing` handler +uses `balanced`; DeepSeek V4.1 registers a model-specific Engram handler when +its example module is imported. + +There is no user-facing `value_dependencies` rule list. The handlers are scoped +to the fake step and restore patched methods afterward. + +## Custom Operators + +Users register a shape-only kernel with: + +```python +from hyper_parallel.tools.dryrun import register_fake_op + + +@register_fake_op("custom::my_forward") +def my_forward_fake(input_tensor, weight): + return torch.empty_like(input_tensor) +``` + +Registration occurs immediately when the defining module is imported. There is +no `op_plugins` enable switch. Model integration code must import the registration +module before dry-run constructs the model. + +When FakeTensor encounters an unsupported operator, the error explains how to +register a kernel and lists currently registered operators. + +## Compatibility Boundary + +DTensor supports FakeTensor locals natively: ``DTensorBase.__new__`` builds a +storage-less meta shell for a FakeTensor local, and every metadata, grad, and +storage accessor answers for the local tensor. Dry-run and real training share +the same dispatch path; no DTensor methods are patched. + +The runner installs the remaining process-local compatibility patches before +model build: + +- Deferred, shape-only AutoModel materialization. +- Fake process-group subgroup creation for optimizer replicas. +- Layout-derived Muon gather metadata for uneven and empty shards. +- CSA2 host-value validation bypasses limited to tensors with `fake_mode`. + +Normal components do not import dry-run config or DeepSeek modules. + +Storage alias checks in core (Muon, FSDP) compare `untyped_storage()` identity +directly. This works for real tensors and FakeTensors. `data_ptr()` is not used +because FakeTensors have no physical address and can return identical zero +values for independent storages (empty shards behave the same way). + +## Offload and Swap Simulation + +CPU offload and swap features trade device bytes for host bytes, and their +storage lifecycle is entirely host-ordered: device memory is released by +`untyped_storage().resize_(0)` inside `wait_offload`, and restored by +`resize_(nbytes)` plus a copy inside `launch_load`/`prefetch` — all called +from Python hooks in a fixed order. Stream events only decide how much the +transfers overlap compute, never which bytes are live, so the logical tracker +reproduces the lifecycle exactly. Three features simulate on this basis: + +- **FSDP CPU offload** (`fsdp_config.enable_offload`): sharded params and + reduced grads live on host; each all-gather stages a transient device copy. +- **Optimizer-state swap** (`optimizer.swap.enabled`): state tensors mirror + to pinned host memory between updates, with two device staging arenas. +- **Activation swap** (`activation_swap: attention`): per-layer attention + activations offload after forward and prefetch before backward. + +The components reach their stream/event primitives through seams +(`core/utils/simulation.py`): under simulation the seams answer with +immediate-completion stand-ins, so the unmodified swap state machines run in +host program order without binding an accelerator. Host-side pinning +degrades to ordinary CPU allocations (pinning changes DMA speed, not bytes). +One refinement keeps staging lifetimes conservative: + +- **Stream holds**: the `record_stream` stand-in retains the staging buffer + reference until the next simulated event wait or stream synchronize — the + same point a real caching allocator could first reclaim it — so logical + staging lifetimes are never shorter than real ones, without modeling + stream timing. + +The snapshot gains a host ledger: fake CPU storages the step allocates +(offload mirrors, swapped state) are accounted under the `host` key and +`summary.host_peak_bytes`, separate from the native device keys, so a plan +reads both sides of the trade — device bytes saved and pinned host bytes +spent. Real CPU tensors that pre-exist the step (the probed batch) stay out. +The reconciliation ST asserts both sides: the device census against the real +trace, and the host ledger's coverage of the device saving against the +baseline scenario (the host side has no real counterpart trace). + +Known limit: under CPU-fallback simulation (no accelerator backend on the +planning host) device and host tensors share one fake device, so host +mirrors land on the device ledger and `host_peak_bytes` stays zero. This is +accepted: the fallback exists for coarse capacity planning, host RAM is +rarely the binding constraint, and exact offload/swap accounting is one +accelerator-lying run away. + +What is not reproduced is, as everywhere else, only device-side overlap: a +real run may briefly hold both the device storage and its in-flight host +copy across an asynchronous boundary that simulation collapses to program +order. Measured against the 8-die validation crop, the FSDP-offload logical +peak matches the real trace peak to within 0.1%. + +## Snapshot Semantics + +Each live storage produces a logical synthetic segment. The tracker records +requested storage bytes and native-style alloc/free events. It does not simulate: + +- Caching allocator segments or fragmentation. +- Device workspace and communication library reserves (fused vendor kernels' + aclnn workspaces are the main unaccounted item when `use_fused_ops` paths + are active). +- Asynchronous device-stream overlap (the host-visible pipelining of async + communication — buffers issued in one micro-step and waited in the next — + **is** reproduced, because the real scheduler code runs; only the + kernel-time interleaving on device streams is out of scope). +- HCCL/NCCL persistent allocations. + +The resulting peak is a logical capacity estimate, not a reservation guarantee. +Measured against 8-die DeepSeek-V4.1 validation crops, the logical peak lands +at 94–100% of the real trace-level peak across EP/FSDP/TP+SP/CP mixes, and at +100–102% across the offload/swap scenarios. + +## Unsupported Scope + +The current implementation rejects: + +- Pipeline parallelism (the eager trainer has no PP path yet; dry-run will + follow once it does). +- PEFT mutation. +- `torch.compile`. + +Dry-run must fail fast when a configuration requests behavior whose memory +lifetime cannot be represented by the current logical tracker. diff --git a/hyper_parallel/tools/dryrun/README.md b/hyper_parallel/tools/dryrun/README.md new file mode 100644 index 0000000000..e62063ae77 --- /dev/null +++ b/hyper_parallel/tools/dryrun/README.md @@ -0,0 +1,95 @@ +# HyperParallel Dry-run + +`hyper_parallel.tools.dryrun` executes the configured training path with +FakeTensors and writes a PyTorch-memory-viz-compatible logical snapshot. Model +weights and activations are not allocated on an accelerator. + +## Run + +```bash +python -m hyper_parallel.tools.dryrun path/to/training.yaml +``` + +For an existing torchrun launch, each worker reads the same YAML and only ranks +listed in `dry_run.ranks` execute: + +```bash +torchrun --nproc_per_node=8 -m hyper_parallel.tools.dryrun path/to/training.yaml +``` + +Without torchrun, the CLI runs the configured ranks sequentially in one process, +using `virtual_world_size` as the logical topology. The default is rank 0. + +The YAML uses the normal trainer sections plus: + +```yaml +dry_run: + enabled: true + target_device: npu + ranks: [0, 7] + virtual_world_size: 8 + snapshot: + enabled: true + path: "outputs/dry_run" + capture_stacks: true + stack_depth: 64 +``` + +`snapshot.path` is the output root. Dry-run creates +`rank_{rank}/memory.pickle` under it automatically. Setting +`snapshot.enabled: false` keeps the in-memory result but writes no pickle. +`capture_stacks` and `stack_depth` control Python allocation stacks in the +snapshot and can reduce overhead for very large models. +`enabled: false` makes the dry-run CLI exit without constructing a model. + +`ranks` must be unique, non-negative, and smaller than the logical world size. +Under torchrun, `virtual_world_size`, when set, must equal the launched +`WORLD_SIZE`. + +The only dry-run data product is the pickle selected by `snapshot.path`; run +metadata is embedded under the snapshot's `metadata` key. Progress and errors +use the normal Python logger and therefore follow the launch process's console +or trainer logging configuration. + +## Add A Custom Operator + +If dry-run reports `UnsupportedOperatorException`, add a shape-only FakeTensor +kernel for the missing operator. Do not copy values or launch device work. + +```python +# my_project/dryrun_ops.py +import torch + +from hyper_parallel.tools.dryrun import register_fake_op + + +@register_fake_op("custom::my_forward") +def my_forward_fake(input_tensor, weight, *, scale=1.0): + del weight, scale + return torch.empty_like(input_tensor) +``` + +There is no separate enable switch. The decorator registers immediately when +the defining module is imported. Import that module from the model integration +code before dry-run constructs the model. + +## Value Dependencies + +FakeTensor removes runtime values, so data-dependent routing and control flow +cannot execute unchanged. Dry-run automatically discovers targets supported by +registered value-dependency handlers and applies each handler's default balanced +path. There is no user-facing `value_dependencies` switch or rule list. + +For MoE, `fix_router=True` is necessary but not sufficient: it fixes the intended +expert assignments, while FakeTensor still cannot materialize `argsort` +permutations, expert counts, `.tolist()` host ints, or value-sized slices. +The automatic MoE handler supplies those shapes during the fake step. + +DeepSeek V4.1 additionally registers its Engram replay handler when the example +model module is imported. Models without value-dependent branches simply have no +discovered targets. + +The built-in DeepSeek-V4.1 example is in +`hyper_parallel/tools/dryrun/examples/deepseek_v41.yaml`. +The full interface and execution design is in +`hyper_parallel/tools/dryrun/DESIGN.md`. diff --git a/hyper_parallel/tools/dryrun/__init__.py b/hyper_parallel/tools/dryrun/__init__.py new file mode 100644 index 0000000000..301edaeb76 --- /dev/null +++ b/hyper_parallel/tools/dryrun/__init__.py @@ -0,0 +1,67 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Public entry points for model-independent logical memory dry-run.""" + +from importlib import import_module +from typing import Any + +__all__ = [ + "DryRunBatchMocker", + "DryRunConfig", + "DryRunOpRegistry", + "DryRunRuntime", + "DryRunSnapshotConfig", + "HyperModelsDryRunRunner", + "LogicalMemoryTracker", + "ValueDependencyHandler", + "ValueDependencyStage", + "get_dryrun_op_registry", + "install_dryrun_compatibility", + "register_builtin_dryrun_ops", + "register_environment_patch", + "register_fake_op", + "register_value_dependency_handler", + "write_memory_snapshot", +] + +_EXPORTS = { + "DryRunBatchMocker": ("runtime", "DryRunBatchMocker"), + "DryRunConfig": ("config", "DryRunConfig"), + "DryRunOpRegistry": ("operator_support", "DryRunOpRegistry"), + "DryRunRuntime": ("runtime", "DryRunRuntime"), + "DryRunSnapshotConfig": ("config", "DryRunSnapshotConfig"), + "HyperModelsDryRunRunner": ("runner", "HyperModelsDryRunRunner"), + "LogicalMemoryTracker": ("memory", "LogicalMemoryTracker"), + "ValueDependencyHandler": ("runtime", "ValueDependencyHandler"), + "ValueDependencyStage": ("runtime", "ValueDependencyStage"), + "get_dryrun_op_registry": ("operator_support", "get_dryrun_op_registry"), + "install_dryrun_compatibility": ("compatibility", "install_dryrun_compatibility"), + "register_builtin_dryrun_ops": ("operator_support", "register_builtin_dryrun_ops"), + "register_environment_patch": ("compatibility", "register_environment_patch"), + "register_fake_op": ("operator_support", "register_fake_op"), + "register_value_dependency_handler": ("runtime", "register_value_dependency_handler"), + "write_memory_snapshot": ("memory", "write_memory_snapshot"), +} + + +def __getattr__(name: str) -> Any: + """Load a public dry-run symbol without importing the runner eagerly.""" + try: + module_name, attribute = _EXPORTS[name] + except KeyError as error: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from error + value = getattr(import_module(f"{__name__}.{module_name}"), attribute) + globals()[name] = value + return value diff --git a/hyper_parallel/tools/dryrun/__main__.py b/hyper_parallel/tools/dryrun/__main__.py new file mode 100644 index 0000000000..02b1d443a9 --- /dev/null +++ b/hyper_parallel/tools/dryrun/__main__.py @@ -0,0 +1,98 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Run one configured logical training step without entering the real Trainer.""" + +import copy +import logging +import os +from collections.abc import Mapping + +from hyper_parallel.trainer.config.parser import parse_training_args +from hyper_parallel.tools.dryrun import HyperModelsDryRunRunner +from hyper_parallel.tools.dryrun.config import DryRunConfig +from hyper_parallel.tools.dryrun.runtime import DryRunRuntime + +logger = logging.getLogger(__name__) + + +def _resolve_config(value: object) -> DryRunConfig: + """Normalize the trainer's generic dry-run mapping.""" + if isinstance(value, DryRunConfig): + return value + if isinstance(value, Mapping): + return DryRunConfig.from_mapping(value) + raise TypeError("The training config must contain a dry_run mapping") + + +def _torchrun_identity() -> tuple[int, int, int] | None: + """Return rank identity when the CLI was launched by torchrun.""" + names = ("RANK", "WORLD_SIZE", "LOCAL_RANK") + if not all(name in os.environ for name in names): + return None + return tuple(int(os.environ[name]) for name in names) + + +def _configured_ranks(dry_run: DryRunConfig, world_size: int) -> list[int]: + """Resolve and validate the ranks selected by the config.""" + if isinstance(world_size, bool) or not isinstance(world_size, int) or world_size < 1: + raise ValueError("dry_run.virtual_world_size must be a positive integer") + ranks = dry_run.ranks if dry_run.ranks is not None else [0] + if ( + not ranks + or any(isinstance(rank, bool) or not isinstance(rank, int) or rank < 0 for rank in ranks) + or len(set(ranks)) != len(ranks) + or any(rank >= world_size for rank in ranks) + ): + raise ValueError( + "dry_run.ranks must be a non-empty list of unique non-negative " + f"integers smaller than world size {world_size}" + ) + return ranks + + +def main() -> None: + """Run dry-run from a resolved training YAML and optional CLI overrides.""" + config = parse_training_args() + dry_run = _resolve_config(config.dry_run) + if not dry_run.enabled: + logger.info("Dry-run is disabled by dry_run.enabled=false") + return + config.dry_run = dry_run + torchrun_identity = _torchrun_identity() + if torchrun_identity is not None: + rank, world_size, _ = torchrun_identity + if dry_run.virtual_world_size is not None and dry_run.virtual_world_size != world_size: + raise ValueError( + "dry_run.virtual_world_size must match torchrun WORLD_SIZE: " + f"{dry_run.virtual_world_size} versus {world_size}" + ) + if rank not in _configured_ranks(dry_run, world_size): + logger.info("Skipping dry-run rank %s; dry_run.ranks=%s", rank, dry_run.ranks) + return + HyperModelsDryRunRunner(config).run() + return + + world_size = dry_run.virtual_world_size or 1 + for rank in _configured_ranks(dry_run, world_size): + logger.info("Running dry-run rank %s/%s", rank, world_size) + rank_config = copy.deepcopy(config) + HyperModelsDryRunRunner( + rank_config, + DryRunRuntime(rank=rank, world_size=world_size, local_rank=0), + ).run() + + +if __name__ == "__main__": + main() diff --git a/hyper_parallel/tools/dryrun/compatibility.py b/hyper_parallel/tools/dryrun/compatibility.py new file mode 100644 index 0000000000..01c68bf315 --- /dev/null +++ b/hyper_parallel/tools/dryrun/compatibility.py @@ -0,0 +1,285 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Process-local compatibility patches used only by the dry-run tool.""" + +from __future__ import annotations + +from contextlib import contextmanager +from contextvars import ContextVar +from typing import Any, Iterator + +import torch +import torch.distributed as dist +from torch import nn +from torch._subclasses.fake_tensor import unset_fake_temporarily +from transformers.modeling_utils import ContextManagers + +from hyper_parallel import init_empty_weights + + +_DEFER_MODEL_MATERIALIZATION: ContextVar[bool] = ContextVar( + "dryrun_defer_model_materialization", + default=False, +) +_MUON_LAYOUT_STAGE_STATE: ContextVar[Any] = ContextVar( + "dryrun_muon_layout_stage_state", + default=None, +) +_INSTALLED = False +# Model-specific environment patches, registered by example/plugin modules at +# import time (same contract as value-dependency handlers). The generic +# compatibility layer never imports model-specific code; the dependency +# direction is plugins -> dry-run adapters, not the reverse. +_ENVIRONMENT_PATCHES: dict[str, Any] = {} +_ENVIRONMENT_PATCHES_INSTALLED: set[str] = set() + + +def register_environment_patch(name: str, installer: Any) -> None: + """Register a model-specific process-local patch for dry-run environments. + + Args: + name: Unique patch identity, e.g. ``"deepseek_v41.csa2"``. + installer: Zero-argument callable applying the patch idempotently. + """ + if not callable(installer): + raise TypeError("environment patch installer must be callable") + _ENVIRONMENT_PATCHES[name] = installer + # A patch registered after install_dryrun_compatibility (import order is + # the plugin's choice) still has to take effect for this process. + if _INSTALLED and name not in _ENVIRONMENT_PATCHES_INSTALLED: + installer() + _ENVIRONMENT_PATCHES_INSTALLED.add(name) + + +@contextmanager +def model_build_context() -> Iterator[None]: + """Build a model without materializing weights or reading device storage.""" + token = _DEFER_MODEL_MATERIALIZATION.set(True) + try: + yield + finally: + _DEFER_MODEL_MATERIALIZATION.reset(token) + + +def is_model_materialization_deferred() -> bool: + """Return whether the current build is owned by dry-run.""" + return _DEFER_MODEL_MATERIALIZATION.get() + + +def _install_model_build_patches() -> None: + """Keep shape-only construction out of AutoModel business branches.""" + from hyper_parallel.models._transformers import auto_model, model_builder + + if getattr(model_builder, "_dryrun_compatibility_installed", False): + return + + original_materialize = model_builder._materialize_and_load_model + original_init_context = auto_model._model_init_context + + def materialize_without_storage(model: nn.Module, **kwargs: Any) -> nn.Module: + if is_model_materialization_deferred(): + return model + return original_materialize(model, **kwargs) + + def model_init_context(is_meta_device: bool) -> Any: + if not is_model_materialization_deferred(): + return original_init_context(is_meta_device) + + try: + from transformers.modeling_utils import no_init_weights + except ImportError: + from transformers.initialization import no_init_weights + + contexts = [unset_fake_temporarily(), no_init_weights(), init_empty_weights()] + return ContextManagers(contexts) + + model_builder._materialize_and_load_model = materialize_without_storage + model_builder.model_build_context = model_build_context + model_builder.is_model_materialization_deferred = is_model_materialization_deferred + auto_model._model_init_context = model_init_context + model_builder._dryrun_compatibility_installed = True + + +def _install_fake_subgroup_patch() -> None: + """Create optimizer replica subgroups directly on PyTorch's fake backend.""" + from hyper_parallel.core.optimizer.optimizer import BaseDistributedOptimizer + + if getattr(BaseDistributedOptimizer, "_dryrun_subgroup_patch_installed", False): + return + original = BaseDistributedOptimizer._get_or_create_sub_pg + + def get_or_create_sub_pg( + self: Any, + parent_pg: dist.ProcessGroup, + sub_size: int, + ) -> dist.ProcessGroup: + if dist.get_backend(parent_pg) != "fake": + return original(self, parent_pg, sub_size) + parent_ranks = tuple(sorted(dist.get_process_group_ranks(parent_pg))) + cache_key = (parent_ranks, sub_size) + cached = self._split_sub_pg_cache.get(cache_key) + if cached is not None: + for subgroup in cached.values(): + if subgroup is not None: + return subgroup + + rank = dist.get_rank() + subgroups = {} + local_group = None + for index in range(len(parent_ranks) // sub_size): + ranks = list(parent_ranks[index * sub_size:(index + 1) * sub_size]) + if rank not in ranks: + subgroups[index] = None + continue + local_group = dist.new_group(ranks=ranks, use_local_synchronization=True) + subgroups[index] = local_group + self._split_sub_pg_cache[cache_key] = subgroups + if local_group is None: + raise RuntimeError(f"Rank {rank} is absent from optimizer subgroup {parent_ranks}") + return local_group + + BaseDistributedOptimizer._get_or_create_sub_pg = get_or_create_sub_pg + BaseDistributedOptimizer._dryrun_subgroup_patch_installed = True + + +def _infer_gather_rank_shapes( + layout: Any, + global_shape: tuple[int, ...], + coordinate: tuple[int, ...], + shard_axes: tuple[tuple[int, int], ...], +) -> tuple[tuple[tuple[int, ...], ...], ...]: + """Derive successive all-gather shapes from DTensor slice geometry.""" + from itertools import product + + from hyper_parallel.core.dtensor.layout import infer_slice_area_by_layout + + shapes = {} + for rank, mesh_coordinate in enumerate( + product(*(range(size) for size in layout.mesh_shape)) + ): + area = infer_slice_area_by_layout(layout, rank, global_shape) + shapes[mesh_coordinate] = tuple(end - start for start, end in area) + result = [] + for mesh_dim, tensor_dim in shard_axes: + group_size = layout.mesh_shape[mesh_dim] + peers = [ + coordinate[:mesh_dim] + (rank,) + coordinate[mesh_dim + 1:] + for rank in range(group_size) + ] + result.append(tuple(shapes[peer] for peer in peers)) + next_shapes = {} + for key, shape in shapes.items(): + group_peers = [ + key[:mesh_dim] + (rank,) + key[mesh_dim + 1:] + for rank in range(group_size) + ] + output = list(shape) + output[tensor_dim] = sum(shapes[peer][tensor_dim] for peer in group_peers) + next_shapes[key] = tuple(output) + shapes = next_shapes + return tuple(result) + + +def _install_muon_metadata_patch() -> None: + """Supply FakeTensor Muon shard metadata without changing optimizer code.""" + from hyper_parallel.core.dtensor.layout import Layout + from hyper_parallel.core.optimizer import muon + from hyper_parallel.core.optimizer import muon_shard + from hyper_parallel.core.optimizer.dtensor_compat import to_local_if_dtensor + + if getattr(muon_shard, "_dryrun_metadata_patch_installed", False): + return + original_build = muon_shard.build_param_shard_metadata_for_group + original_all_gather = muon_shard._allgather_shapes_for_group + + def all_gather_shapes_for_group( + local_shapes: list[tuple[int, ...]], + shard_pg: Any, + shard_size: int, + device: torch.device, + ) -> list[list[tuple[int, ...]]]: + state = _MUON_LAYOUT_STAGE_STATE.get() + if state is None: + return original_all_gather(local_shapes, shard_pg, shard_size, device) + layout_shapes, stage = state + _MUON_LAYOUT_STAGE_STATE.set((layout_shapes, stage + 1)) + return [ + [stages[stage][rank] for stages in layout_shapes] + for rank in range(shard_size) + ] + + def build_param_shard_metadata_for_group(hsdp_group: Any) -> dict[Any, Any]: + layout_spec = hsdp_group.layout_spec + if ( + layout_spec is None + or not layout_spec.shard_axes + or not hsdp_group.records + or not all( + isinstance(getattr(record.param, "layout", None), Layout) + for record in hsdp_group.records + ) + ): + return original_build(hsdp_group) + layout_shapes = [] + for record in hsdp_group.records: + parameter = record.param + coordinate = tuple(parameter.device_mesh.get_coordinate()) + stages = _infer_gather_rank_shapes( + parameter.layout, + tuple(parameter.shape), + coordinate, + layout_spec.shard_axes, + ) + inferred = stages[0][coordinate[layout_spec.shard_axes[0][0]]] + local_shape = tuple(to_local_if_dtensor(parameter.data).shape) + if inferred != local_shape: + raise ValueError( + f"Muon shard shape disagrees with layout: inferred={inferred}, " + f"local={local_shape}, global={tuple(parameter.shape)}" + ) + layout_shapes.append(stages) + token = _MUON_LAYOUT_STAGE_STATE.set((layout_shapes, 0)) + try: + return original_build(hsdp_group) + finally: + _MUON_LAYOUT_STAGE_STATE.reset(token) + + muon_shard.build_param_shard_metadata_for_group = build_param_shard_metadata_for_group + muon.build_param_shard_metadata_for_group = build_param_shard_metadata_for_group + muon_shard._allgather_shapes_for_group = all_gather_shapes_for_group + muon_shard._dryrun_metadata_patch_installed = True + + +def install_dryrun_compatibility() -> None: + """Install all process-local compatibility patches used by dry-run.""" + global _INSTALLED # pylint: disable=global-statement + if _INSTALLED: + return + _install_model_build_patches() + _install_fake_subgroup_patch() + _install_muon_metadata_patch() + for name, installer in _ENVIRONMENT_PATCHES.items(): + if name not in _ENVIRONMENT_PATCHES_INSTALLED: + installer() + _ENVIRONMENT_PATCHES_INSTALLED.add(name) + _INSTALLED = True + + +__all__ = [ + "install_dryrun_compatibility", + "is_model_materialization_deferred", + "model_build_context", + "register_environment_patch", +] diff --git a/hyper_parallel/tools/dryrun/config.py b/hyper_parallel/tools/dryrun/config.py new file mode 100644 index 0000000000..538ceb2c3e --- /dev/null +++ b/hyper_parallel/tools/dryrun/config.py @@ -0,0 +1,49 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Dry-run configuration for logical LLM memory analysis.""" + +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import Any, Literal, Optional + + +@dataclass +class DryRunSnapshotConfig: + """Logical memory snapshot output settings.""" + + enabled: bool = True + path: str = "outputs/dry_run" + capture_stacks: bool = True + stack_depth: int = 64 + + +@dataclass +class DryRunConfig: + """Shape and value-profile configuration for logical LLM memory analysis.""" + + enabled: bool = True + target_device: Literal["npu", "cuda"] = "npu" + ranks: Optional[list[int]] = None + snapshot: DryRunSnapshotConfig = field(default_factory=DryRunSnapshotConfig) + virtual_world_size: Optional[int] = None + + @classmethod + def from_mapping(cls, value: Mapping[str, Any]) -> "DryRunConfig": + """Build a config while resolving the nested snapshot mapping.""" + fields = dict(value) + snapshot = fields.get("snapshot") + if isinstance(snapshot, Mapping): + fields["snapshot"] = DryRunSnapshotConfig(**dict(snapshot)) + return cls(**fields) diff --git a/hyper_parallel/tools/dryrun/data.py b/hyper_parallel/tools/dryrun/data.py new file mode 100644 index 0000000000..9a14b88242 --- /dev/null +++ b/hyper_parallel/tools/dryrun/data.py @@ -0,0 +1,82 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""One-shot normal-training data probe owned exclusively by Dry-run.""" +# Dry-run deliberately reuses TextTrainer's existing private build stages. +# pylint: disable=protected-access + +from dataclasses import dataclass +from typing import Any, Mapping + +from hyper_parallel.trainer.base import BaseTrainer +from hyper_parallel.trainer.runtime.loss_aggregation import count_loss_token +from hyper_parallel.trainer.text_trainer import TextTrainer + + +@dataclass(frozen=True) +class PreparedDryRunBatch: + """One normal-training batch and its CPU-derived token counts.""" + + model_inputs: Mapping[str, Any] + loss_inputs: Mapping[str, Any] + token_counts: Mapping[str, int] + + +class DryRunDataProbe: + """Reuse TextTrainer data methods without constructing a real trainer.""" + + def __init__(self, base: BaseTrainer) -> None: + """Attach a temporary TextTrainer facade to Dry-run's BaseTrainer state.""" + self.base = base + self._iterator: Any = None + self._trainer = TextTrainer.__new__(TextTrainer) + self._trainer.base = base + + def build(self) -> None: + """Invoke the unchanged TextTrainer data construction sequence.""" + self._trainer._build_model_assets() + self._trainer._build_data_transform() + self.base._build_dataset() + self._trainer._build_collate_fn() + self.base._build_dataloader() + self._trainer._build_get_batch() + + def read_first_batch(self) -> PreparedDryRunBatch: + """Read this rank's first fully prepared training micro-batch.""" + dataloader = getattr(self.base, "train_dataloader", None) + if dataloader is None: + raise ValueError("A non-empty train dataloader is required") + self._iterator = iter(dataloader) + try: + model_inputs, loss_inputs = self.base.get_batch(self._iterator) + except StopIteration as exc: + raise ValueError("Train dataloader produced no batches") from exc + if not isinstance(model_inputs, Mapping) or not isinstance(loss_inputs, Mapping): + raise ValueError("dataloader.get_batch must return model_inputs and loss_inputs mappings") + counts = { + name: int(value.item()) + for name, value in count_loss_token(dict(loss_inputs)).items() + } + return PreparedDryRunBatch(dict(model_inputs), dict(loss_inputs), counts) + + def close(self) -> None: + """Stop workers owned by the one-shot iterator, if any.""" + iterator = self._iterator + self._iterator = None + shutdown_workers = getattr(iterator, "_shutdown_workers", None) + if callable(shutdown_workers): + shutdown_workers() # pylint: disable=not-callable + + +__all__ = ["DryRunDataProbe", "PreparedDryRunBatch"] diff --git a/hyper_parallel/tools/dryrun/examples/__init__.py b/hyper_parallel/tools/dryrun/examples/__init__.py new file mode 100644 index 0000000000..4a6b2084a4 --- /dev/null +++ b/hyper_parallel/tools/dryrun/examples/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Model-specific dry-run examples built on the generic tool package.""" diff --git a/hyper_parallel/tools/dryrun/examples/deepseek_v41.py b/hyper_parallel/tools/dryrun/examples/deepseek_v41.py new file mode 100644 index 0000000000..34fabb9cfc --- /dev/null +++ b/hyper_parallel/tools/dryrun/examples/deepseek_v41.py @@ -0,0 +1,288 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""DeepSeek V4.1 text-model recipe for logical memory simulation.""" + +from contextlib import AbstractContextManager, nullcontext +from types import MethodType +from typing import Any + +import torch # pylint: disable=forbidden-backend-import +from torch.nn import functional # pylint: disable=forbidden-backend-import +from torch.utils.checkpoint import checkpoint # pylint: disable=forbidden-backend-import +from transformers.modeling_outputs import CausalLMOutputWithPast + +from hyper_parallel.components.modules.engram import EngramModule +from hyper_parallel.tools.dryrun.compatibility import register_environment_patch +from hyper_parallel.tools.dryrun.runtime import ( + ValueDependencyHandler, + ValueDependencyStage, + register_value_dependency_handler, +) +from hyper_parallel.distributed.expert_parallel.collectives import ( + _EPAllToAllPadded, + ep_all_to_all, +) +from hyper_parallel.models import HyperAutoModelForCausalLM +from hyper_parallel.models.deepseek_v41.adapter.policies import activation_checkpointing +from hyper_parallel.models.deepseek_v41.adapter.validation.cropped_model import build_deepseek_v41_validation_config + + +def _balanced_sparse_lookup( + hash_ids: torch.Tensor, + embedding: torch.nn.Embedding, + *, + ep_group: Any, + ep_rank: int, + ep_size: int, + padded_num_embeddings: int, +) -> torch.Tensor: + """Replay Engram routing with balanced, value-free EP traffic.""" + num_local_rows = padded_num_embeddings // ep_size + row_start = ep_rank * num_local_rows + flat_ids = hash_ids.reshape(-1) + owner = torch.div(flat_ids, num_local_rows, rounding_mode="floor") + base, remainder = divmod(int(flat_ids.numel()), ep_size) + counts = [base + int(index < remainder) for index in range(ep_size)] + sort_indices = torch.argsort(owner.float(), stable=True) + sorted_ids = flat_ids.index_select(0, sort_indices) + received_ids = ep_all_to_all(sorted_ids, counts, counts, ep_group) + values = embedding(received_ids - row_start) + returned = ep_all_to_all(values, counts, counts, ep_group) + restored = torch.zeros_like(returned) + restored.scatter_add_( + 0, + sort_indices.unsqueeze(1).expand(-1, returned.shape[1]), + returned, + ) + return restored.view(*hash_ids.shape, embedding.embedding_dim) + + +def _fake_padded_all_to_all( + ctx: Any, + x: torch.Tensor, + send_counts: list[int], + recv_counts: list[int], + group: Any, +) -> torch.Tensor: + """Replay padded all-to-all without reading a value from fake MAX reduction.""" + ctx.send_counts = send_counts + ctx.recv_counts = recv_counts + ctx.group = group + ctx.pad_to = pad_to = max([*send_counts, *recv_counts, 1]) + recv = _EPAllToAllPadded._pad_and_exchange(x, send_counts, pad_to, group) + return _EPAllToAllPadded._unpad(recv, recv_counts, pad_to) + + +class _DeepSeekV41FakeValues(AbstractContextManager): + """Install DeepSeek-specific value replay only for the fake training step.""" + + def __enter__(self) -> "_DeepSeekV41FakeValues": + """Patch value-dependent Engram and padded EP boundaries.""" + self._engram_lookup = EngramModule._sparse_lookup # pylint: disable=protected-access + self._padded_forward = _EPAllToAllPadded.forward # pylint: disable=protected-access + EngramModule._sparse_lookup = staticmethod(_balanced_sparse_lookup) # pylint: disable=protected-access + _EPAllToAllPadded.forward = staticmethod(_fake_padded_all_to_all) # pylint: disable=protected-access + return self + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> bool: + """Restore production methods after the fake step.""" + del exc_type, exc_value, traceback + EngramModule._sparse_lookup = staticmethod(self._engram_lookup) # pylint: disable=protected-access + _EPAllToAllPadded.forward = staticmethod(self._padded_forward) # pylint: disable=protected-access + return False + + +class _DeepSeekV41ValueDependency(ValueDependencyHandler): + """Own DeepSeek-specific fake-value boundaries outside business modules.""" + + name = "deepseek_v41" + paths = ("balanced",) + default_path = "balanced" + + def discover(self, model: Any, context: Any) -> list[str]: + """Expose one target when the parallelized model contains Engram.""" + del context + # The handler registry is process-global; another recipe's model (or + # a test double without nn.Module surface) is simply not a target. + modules = getattr(model, "modules", None) + if not callable(modules): + return [] + return ["deepseek_v41"] if any(isinstance(module, EngramModule) for module in modules()) else [] + + def install( + self, + stage: ValueDependencyStage, + target: str, + path: str, + inputs: dict[str, Any], + context: Any, + ) -> AbstractContextManager: + """Install the scoped DeepSeek fake-value adapter.""" + del target, path, inputs, context + if stage is ValueDependencyStage.FAKE_STEP: + return _DeepSeekV41FakeValues() + return nullcontext() + + +register_value_dependency_handler(_DeepSeekV41ValueDependency.name, _DeepSeekV41ValueDependency) + + +def _synthetic_document_boundaries(sequence: Any) -> tuple[int, ...]: + """Keep document count while assigning equal spans to value-free inputs.""" + document_count = max(int(sequence.cu_seq_lens.numel()) - 1, 1) + span, remainder = divmod(sequence.global_sequence_length, document_count) + boundaries = [0] + for index in range(1, document_count + 1): + boundaries.append(boundaries[-1] + span + (index <= remainder)) + return tuple(boundaries) + + +def _install_csa2_patches() -> None: + """Make CSA2 host-side validations shape-only under FakeTensor. + + Model-specific environment patch, registered from this example module so + the generic dry-run compatibility layer never imports DeepSeek code. + """ + from hyper_parallel.components.modules import ( # pylint: disable=C0415 + shared_compressed_dsa_attention as csa2, + ) + + sequence_type = csa2.SharedCompressedPackedSequence + attention_type = csa2.SharedCompressedDSAAttentionBase + if getattr(sequence_type, "_dryrun_compatibility_installed", False): + return + + original_document_boundaries = sequence_type.document_boundaries + original_local_segment_starts = sequence_type.local_segment_starts + original_validate_cp_inputs = attention_type._validate_cp_inputs # pylint: disable=protected-access + + def document_boundaries(sequence: Any) -> tuple[int, ...]: + if not hasattr(sequence.cu_seq_lens, "fake_mode"): + return original_document_boundaries(sequence) + if "boundaries" not in sequence._snapshot: # pylint: disable=protected-access + sequence._snapshot["boundaries"] = _synthetic_document_boundaries(sequence) # pylint: disable=protected-access + return sequence._snapshot["boundaries"] # pylint: disable=protected-access + + def local_segment_starts(sequence: Any, device: torch.device) -> torch.Tensor: + if not hasattr(sequence.cu_seq_lens, "fake_mode"): + return original_local_segment_starts(sequence, device) + cached = sequence._prepared_starts.get(torch.device(device)) # pylint: disable=protected-access + if cached is not None: + return cached + boundaries = torch.tensor( + document_boundaries(sequence), + dtype=torch.long, + device=device, + ) + query_positions = sequence.local_positions(device).squeeze(0) + segment_ids = torch.bucketize(query_positions, boundaries[1:], right=True) + result = boundaries[:-1].index_select(0, segment_ids).unsqueeze(0) + sequence._prepared_starts[torch.device(device)] = result # pylint: disable=protected-access + return result + + def validate_cp_inputs( + cp_context: Any, + position_ids: torch.Tensor, + sequence_length: int, + ) -> tuple[int, int]: + if not hasattr(position_ids, "fake_mode"): + return original_validate_cp_inputs(cp_context, position_ids, sequence_length) + if cp_context.size <= 1 or not 0 <= cp_context.rank < cp_context.size: + raise ValueError( + "shared compressed attention CP requires size > 1 and " + f"0 <= rank < size, got rank={cp_context.rank}, size={cp_context.size}" + ) + return cp_context.rank * sequence_length, sequence_length * cp_context.size + + sequence_type.document_boundaries = document_boundaries + sequence_type.local_segment_starts = local_segment_starts + attention_type._validate_cp_inputs = staticmethod(validate_cp_inputs) # pylint: disable=protected-access + sequence_type._dryrun_compatibility_installed = True # pylint: disable=protected-access + + +register_environment_patch("deepseek_v41.csa2", _install_csa2_patches) + + +def _bind_chunk_loss(model: Any, chunks: int) -> None: + """Compute a checkpointed output-head loss without full-sequence logits.""" + if chunks < 1: + raise ValueError("loss_chunks must be positive") + + def _loss_chunk(hidden: torch.Tensor, targets: torch.Tensor) -> torch.Tensor: + """Compute one output-head cross-entropy chunk.""" + logits = model.lm_head(hidden).float() + return functional.cross_entropy( + logits.reshape(-1, logits.size(-1)), targets.reshape(-1), + ignore_index=-100, reduction="sum", + ) + + def _forward(current_model: Any, *, labels: torch.Tensor, **inputs: Any) -> CausalLMOutputWithPast: + """Run the text model and checkpoint each output-head loss chunk.""" + hidden = current_model.model(**inputs).last_hidden_state + targets = functional.pad(labels[..., 1:], (0, 1), value=-100) + if hidden.size(1) % chunks: + raise ValueError("CP-local sequence length must be divisible by loss_chunks") + loss = torch.zeros((), dtype=torch.float32, device=hidden.device) + for hidden_chunk, target_chunk in zip(hidden.chunk(chunks, dim=1), targets.chunk(chunks, dim=1)): + loss = loss + checkpoint(_loss_chunk, hidden_chunk, target_chunk, use_reentrant=False) + return CausalLMOutputWithPast(loss=loss / targets.ne(-100).sum().clamp_min(1)) + + model.forward = MethodType(_forward, model) + + +def build_text_model( + config_path: str, + engram_assets_path: str, + *, + num_hidden_layers: int | None = None, + text_parameter_divisor: int = 1, + num_routed_experts: int = 384, + loss_chunks: int = 8, + recompute_tier: str = "all", + **kwargs: Any, +) -> Any: + """Build a released-size (or explicitly cropped) text model with chunk loss. + + Args: + config_path: Directory containing the released config.json. + engram_assets_path: Engram validation asset generated for that config. + num_hidden_layers: Optional decoder crop depth; ``None`` keeps all layers. + text_parameter_divisor: 1 retains released text widths. + num_routed_experts: 384 retains all released routed experts. + loss_chunks: Number of CP-local loss chunks. + recompute_tier: Model-owned attention recomputation policy. + **kwargs: Normal HyperAutoModel build settings supplied by Trainer. + + Returns: + Config-only model parallelized through the usual AutoModel path. + """ + config = build_deepseek_v41_validation_config( + config_path, engram_assets_path, + num_hidden_layers=num_hidden_layers, + text_parameter_divisor=text_parameter_divisor, + num_routed_experts=num_routed_experts, + exercise_post_training_indexer=False, + ) + config.v41_model_mode = "full" + if recompute_tier not in ("none", "reuse", "reuse_reindex", "all"): + raise ValueError(f"Unknown recompute_tier: {recompute_tier}") + previous_tier = activation_checkpointing.ATTENTION_RECOMPUTE_TIER + activation_checkpointing.ATTENTION_RECOMPUTE_TIER = recompute_tier + try: + model = HyperAutoModelForCausalLM.from_config(config, **kwargs) + finally: + activation_checkpointing.ATTENTION_RECOMPUTE_TIER = previous_tier + _bind_chunk_loss(model, loss_chunks) + return model diff --git a/hyper_parallel/tools/dryrun/examples/deepseek_v41.yaml b/hyper_parallel/tools/dryrun/examples/deepseek_v41.yaml new file mode 100644 index 0000000000..5f92e2dee7 --- /dev/null +++ b/hyper_parallel/tools/dryrun/examples/deepseek_v41.yaml @@ -0,0 +1,139 @@ +# Supply local model, Engram asset, tokenizer and data paths via CLI overrides. +model: + _target_: hyper_parallel.tools.dryrun.examples.deepseek_v41.build_text_model + config_path: /path/to/DeepSeek-V4.1-Flash + engram_assets_path: /path/to/engram.json + torch_dtype: bfloat16 + text_parameter_divisor: 1 + num_routed_experts: 384 + loss_chunks: 8 + recompute_tier: all + validate_placement: false +model_init_dtype: float32 +training: {train_iters: 1, global_batch_size: 16, micro_batch_size: 1, + backend: hccl, init_device: meta, loss_aggregation: token_weighted, + max_grad_norm: 1.0, seed: 42} +accelerator: {tp_size: 2, cp_size: 4, ep_size: 128, pp_size: 1, + sequence_parallel: false, loss_parallel: false} +fsdp_config: + dp_shard_size: 64 + edp_shard_size: 1 + mix_precision: {param_dtype: bfloat16, reduce_dtype: float32, cast_forward_inputs: false} + reshard_after_forward: true + reshard_after_backward: true + requires_grad_sync: true + forward_prefetch_depth: 0 + backward_prefetch_depth: 0 + comm_fusion: false +plan_overrides: + - match: [model.layers.*.attn_hc, model.layers.*.ffn_hc] + module_type: hyper_parallel.models.deepseek_v41.modeling_deepseek_v41.DeepseekV41PipelinedHyperConnection + replace_module: + _target_: hyper_parallel.components.modules.mhc.PipelinedMhcModule + - match: model.layers.*.engram + module_type: hyper_parallel.models.deepseek_v41.modeling_deepseek_v41.DeepseekV41Engram + replace_module: + _target_: hyper_parallel.components.modules.engram.EngramModule + - match: model.layers.*.self_attn + module_type: hyper_parallel.models.deepseek_v41.modeling_deepseek_v41.DeepseekV41Attention + replace_module: + _target_: hyper_parallel.components.modules.shared_compressed_dsa_attention.SharedCompressedDSAAttention + - match: '*.self_attn' + tp_divide_attrs: [num_groups] + - match: '*.self_attn' + region_dispatch: false + inner_target: self + inner_wrapper: + _target_: hyper_parallel.models.deepseek_v41.adapter.distributed.shared_attention_context_parallel.deepseek_v41_shared_attention_parallel_wrapper + - match: model.layers.*.*_hc + region_dispatch: false + params: + fn: {tp: replicate} + base: {tp: replicate} + scale: {tp: replicate} + in_src: + hidden_streams: {tp: replicate, cp: 'shard(1)', ep: replicate} + in_dst: + hidden_streams: {tp: replicate, cp: 'shard(1)', ep: replicate} + out_src: + pre: {tp: replicate, cp: 'shard(1)', ep: replicate} + post: {tp: replicate, cp: 'shard(1)', ep: replicate} + residual: {tp: replicate, cp: 'shard(1)', ep: replicate} + out_dst: + pre: {tp: replicate, cp: 'shard(1)', ep: replicate} + post: {tp: replicate, cp: 'shard(1)', ep: replicate} + residual: {tp: replicate, cp: 'shard(1)', ep: replicate} + - match: model.layers.*.engram + params: + q_weight: {tp: replicate} + k_weight: {tp: replicate} + embed.weight: {ep: 'shard(0)'} + wkv.weight: {tp: replicate} + - match: model.layers.*.engram + when: ep + region_dispatch: false + local_compute_fn: + _target_: hyper_parallel.models.deepseek_v41.adapter.distributed.moe_engram_expert_parallel.deepseek_v41_engram_compute_fn + - match: '*.mlp' + when: ep + region_dispatch: false + local_compute_fn: + _target_: hyper_parallel.models.deepseek_v41.adapter.distributed.moe_engram_expert_parallel.deepseek_v41_ep_compute_fn + use_grouped_gemm: false +activation_checkpoint: + mode: selective + selection: {source: model_adapter_safe_regions, layer_count: 40} +dataset: + _target_: hyper_parallel.data.text.build_dataset.build_online_text_mapping_dataset + data_path: /path/to/online_4k.jsonl + data_config: {create_attention_mask_in_dataloader: true, labels_are_shifted: true} + model_assets: + chat_template: null + tokenizer: + _target_: hyper_parallel.data.text.build_tokenizer.AutoTokenizer.from_pretrained + pretrained_model_name_or_path: /path/to/DeepSeek-V4.1-Flash + local_files_only: true + use_fast: true + trust_remote_code: false + data_transform: + _target_: hyper_parallel.data.text.text_transform.build_text_transform + data_type: plaintext + text_keys: text + max_seq_len: 4096 +dataloader: + _target_: hyper_parallel.data.batching.TokenBatchLoader + min_buffered_samples: 1 + collate_fn: {_target_: hyper_parallel.data.batching.build_online_text_collate_fn} + get_batch: + _target_: hyper_parallel.data.batching.TextParallelBatch + source_type: online + attention_mode: compressed + runtime_input_adapter: + _target_: hyper_parallel.models.deepseek_v41.adapter.data.runtime.DeepseekV41Runtime + include_position_ids: false + include_image_sequence_start: false + sampler_type: single + data_sharding: false + drop_last: true + use_background_prefetcher: false + num_workers: 0 + pin_memory: false +optimizer: + _target_: hyper_parallel.components.optim.Muon + fp32_main_params: true + no_decay_params: [bias, norm, ln_] + muon_config: {muon_lr: 3.0e-05, muon_weight_decay: 0.0, muon_momentum: 0.95, + muon_nesterov: true, muon_ns_steps: 5, muon_ns_variant: asym5} + adamw_config: {adamw_lr: 1.0e-05, adamw_weight_decay: 0.01, + adamw_betas: [0.9, 0.999], adamw_eps: 1.0e-08} +checkpoint: {save_ckpt: false, checkpoint_dir: outputs/dryrun/deepseek_v41/checkpoints} +dry_run: + enabled: true + target_device: npu + ranks: [0] + virtual_world_size: 128 + snapshot: + enabled: true + path: "outputs/dryrun/deepseek_v41" + capture_stacks: true + stack_depth: 64 diff --git a/hyper_parallel/tools/dryrun/memory.py b/hyper_parallel/tools/dryrun/memory.py new file mode 100644 index 0000000000..7d8d1bf7f0 --- /dev/null +++ b/hyper_parallel/tools/dryrun/memory.py @@ -0,0 +1,366 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Model-independent logical storage tracking with native snapshot output.""" + +import copy +import functools +import os +import pickle +import sys +import tempfile +import weakref +from pathlib import Path +from typing import Any, Iterable + +import torch # pylint: disable=forbidden-backend-import +from torch._guards import active_fake_mode # pylint: disable=forbidden-backend-import +# Register shape-only collective implementations for FakeTensor execution. +import torch.distributed._tools.fake_collectives # pylint: disable=unused-import,forbidden-backend-import +from torch.distributed._tools.common_utils import get_untyped_storages # pylint: disable=forbidden-backend-import +from torch.utils._python_dispatch import TorchDispatchMode # pylint: disable=forbidden-backend-import +from torch.utils._pytree import tree_flatten # pylint: disable=forbidden-backend-import + + +_ADDRESS_BASE = 0xF00000000000 +_ADDRESS_ALIGNMENT = 512 + + +class LogicalMemoryTracker(TorchDispatchMode): + """Observe storage capacity and lifetime without classifying tensor roles. + + Args: + device_type: Logical device type of tensors being simulated. + device_index: Device index used for CPU FakeTensor fallback. + + Note: + Segments are synthetic, one per live allocation. There is no caching, + fragmentation, workspace, or asynchronous stream simulation. + When the simulated device is an accelerator, fake CPU storages the + step allocates (offload mirrors, swapped optimizer state) are + accounted on a separate host ledger: offload features trade device + bytes for host bytes, and pinned host memory is its own capacity + limit. Real CPU tensors (the probed data batch, host scalars) are + process state, not step allocations, and stay out of the ledger. + """ + + _HOST_DEVICE = "host" + + def __init__( + self, + device_type: str, + device_index: int = 0, + *, + capture_stacks: bool = True, + stack_depth: int = 64, + ) -> None: + """Initialize empty storage identities, allocation history, and totals.""" + super().__init__() + if device_index < 0: + raise ValueError("device_index must be non-negative") + if stack_depth < 1: + raise ValueError("stack_depth must be positive") + self.device_type = device_type + self.device_index = device_index + self.capture_stacks = capture_stacks + self.stack_depth = stack_depth + self._allocations: dict[int, dict[str, Any]] = {} + self._storage_refs: dict[int, Any] = {} + self._traces: list[list[dict[str, Any]]] = [] + self._host_trace: list[dict[str, Any]] = [] + self._current: dict[Any, int] = {} + self._peaks: dict[Any, int] = {} + self._peak_bytes = 0 + self._next_address = _ADDRESS_BASE + self._previous_resize = None + self._recording = False + self._stack_cache: dict[tuple, list[dict[str, Any]]] = {} + self._host_mark_observer = None + + def __enter__(self) -> "LogicalMemoryTracker": + """Start dispatch tracking and observe direct storage resizes.""" + if self._recording: + raise RuntimeError("LogicalMemoryTracker is already recording") + super().__enter__() + self._recording = True + self._previous_resize = torch.UntypedStorage.resize_ + previous_resize = self._previous_resize + + @functools.wraps(previous_resize) + def resize(storage: Any, size: int) -> Any: + """Record capacity transitions after the storage resize completes.""" + result = previous_resize(storage, size) + key = int(storage._cdata) # pylint: disable=protected-access + allocation = self._allocations.get(key) + if allocation is not None: + self._observe_storage(storage, allocation["device"]) + return result + + torch.UntypedStorage.resize_ = resize + return self + + def __exit__(self, *args: Any) -> Any: + """Restore the storage resize hook and freeze the recorded state.""" + torch.UntypedStorage.resize_ = self._previous_resize + self._previous_resize = None + self._recording = False + return super().__exit__(*args) + + def _frames(self) -> list[dict[str, Any]]: + if not self.capture_stacks: + return [] + frame = sys._getframe().f_back # pylint: disable=protected-access + frames = [] + while frame is not None and len(frames) < self.stack_depth: + code = frame.f_code + if code.co_filename != __file__: + frames.append((code.co_filename, frame.f_lineno, code.co_name)) + frame = frame.f_back + key = tuple(frames) + if key not in self._stack_cache: + self._stack_cache[key] = [ + {"filename": filename, "line": line, "name": name} + for filename, line, name in frames + ] + return self._stack_cache[key] + + def _event(self, action: str, allocation: dict[str, Any]) -> None: + device = allocation["device"] + if device == self._HOST_DEVICE: + trace = self._host_trace + else: + while len(self._traces) <= device: + self._traces.append([]) + trace = self._traces[device] + trace.append({ + "action": action, + "addr": allocation["address"], + "size": allocation["size"], + "stream": 0, + "frames": allocation["frames"], + "pool_id": (0, 0), + }) + + def _release(self, key: int) -> None: + allocation = self._allocations.pop(key, None) + self._storage_refs.pop(key, None) + if allocation is None: + return + if allocation["size"] == 0: + return + allocation = {**allocation, "frames": self._frames()} + self._event("free_requested", allocation) + self._event("free_completed", allocation) + self._event("segment_free", allocation) + self._current[allocation["device"]] -= allocation["size"] + + def _observe_storage(self, storage: Any, device: Any) -> None: + key = int(storage._cdata) # pylint: disable=protected-access + size = int(storage.nbytes()) + allocation = self._allocations.get(key) + if allocation is not None: + if allocation["size"] == size and allocation["device"] == device: + return + # A size change is a reallocation; a device change migrates the + # ledger (a storage first seen as device, then marked host by an + # offload seam, must not stay on the device account). + self._release(key) + tracker_ref = weakref.ref(self) + + def released(storage_ref: Any) -> None: + """Close the allocation only if this is its current storage reference.""" + tracker = tracker_ref() + if tracker is not None and tracker._recording and tracker._storage_refs.get(key) is storage_ref: + tracker._release(key) + + # Keep zero-capacity storage identities so a later resize can be observed. + self._storage_refs[key] = weakref.ref(storage, released) + if size == 0: + self._allocations[key] = { + "device": device, "size": 0, "address": 0, "frames": [], + } + return + address = self._next_address + self._next_address += ((size + _ADDRESS_ALIGNMENT - 1) // _ADDRESS_ALIGNMENT) * _ADDRESS_ALIGNMENT + allocation = {"device": device, "size": size, "address": address, "frames": self._frames()} + self._allocations[key] = allocation + self._event("segment_alloc", allocation) + self._event("alloc", allocation) + self._current[device] = self._current.get(device, 0) + size + self._peaks[device] = max(self._peaks.get(device, 0), self._current[device]) + self._peak_bytes = max(self._peak_bytes, self._accelerator_current_bytes()) + + def _accelerator_current_bytes(self) -> int: + """Return live bytes on the simulated accelerator, excluding the host ledger.""" + return sum( + current for device, current in self._current.items() + if device != self._HOST_DEVICE + ) + + def track(self, tensors: Iterable[torch.Tensor]) -> None: + """Register existing tensors without retaining them or their storages. + + Args: + tensors: Existing inputs, state, or newly returned tensors. + """ + if not self._recording: + raise RuntimeError("Enter LogicalMemoryTracker before registering tensors") + for tensor in tensors: + if tensor.device.type == self.device_type: + device: Any = tensor.device.index + if device is None: + device = self.device_index + elif ( + tensor.device.type == "cpu" + and self.device_type != "cpu" + and hasattr(tensor, "fake_mode") + ): + # A fake CPU storage is one the simulated step itself + # allocated — an offload mirror, a swapped optimizer mirror, + # or a host scalar. Real CPU tensors (the probed batch) are + # pre-existing process state and stay untracked. + # + # Known limit: under CPU-fallback simulation (device_type == + # "cpu") device and host share one fake device, so host + # mirrors land on the device ledger and host_peak_bytes + # stays zero. Accepted: the fallback exists for coarse + # capacity planning without the accelerator's torch build, + # and host RAM is rarely the binding constraint. Use an + # accelerator-lying run for exact offload/swap accounting. + device = self._HOST_DEVICE + else: + continue + for storage in get_untyped_storages(tensor): + self._observe_storage(storage, device) + + def __torch_dispatch__(self, func: Any, types: Any, args: tuple = (), kwargs: Any = None) -> Any: + del types + input_values, _ = tree_flatten((args, kwargs)) + # FakeTensor constant propagation must not execute collective kernels + # on tiny materialized scalars in the virtual process group. + if str(func).startswith("c10d."): + for value in input_values: + if isinstance(value, torch.Tensor) and hasattr(value, "fake_mode"): + value.constant = None + self.track(value for value in input_values if isinstance(value, torch.Tensor)) + # Fake wait_tensor's Meta kernel returns a new storage, unlike a real wait. + if func is torch.ops._c10d_functional.wait_tensor.default and active_fake_mode() is not None: + result = args[0] + else: + result = func(*args, **(kwargs or {})) + output_values, _ = tree_flatten(result) + self.track(value for value in (*input_values, *output_values) if isinstance(value, torch.Tensor)) + return result + + def snapshot(self) -> dict[str, Any]: + """Return an independent native-compatible snapshot of logical memory. + + Returns: + Native segments and device traces, with logical summary and + simulation assumptions. Event list order is the logical timeline. + """ + segments = [] + host_segments = [] + for allocation in self._allocations.values(): + size = allocation["size"] + if size == 0: + continue + segment = { + "device": allocation["device"], + "address": allocation["address"], + "total_size": size, + "allocated_size": size, + "active_size": size, + "requested_size": size, + "stream": 0, + "segment_type": "small" if size <= 2 ** 20 else "large", + "segment_pool_id": (0, 0), + "is_expandable": False, + "frames": allocation["frames"], + "blocks": [{ + "address": allocation["address"], + "size": size, + "requested_size": size, + "state": "active_allocated", + "frames": allocation["frames"], + }], + } + if allocation["device"] == self._HOST_DEVICE: + segment["device"] = 0 + host_segments.append(segment) + else: + segments.append(segment) + # Copies isolate saved snapshots from subsequent events and releases. + return copy.deepcopy({ + "segments": sorted(segments, key=lambda segment: (segment["device"], segment["address"])), + "device_traces": self._traces, + # Host-side storages the simulated step allocated (offload mirrors, + # swapped state). Kept out of the native keys so accelerator + # tooling reads the device story unchanged. + "host": { + "segments": sorted(host_segments, key=lambda segment: segment["address"]), + "trace": self._host_trace, + "current_bytes": self._current.get(self._HOST_DEVICE, 0), + "peak_bytes": self._peaks.get(self._HOST_DEVICE, 0), + }, + "summary": { + "current_bytes": self._accelerator_current_bytes(), + "peak_bytes": self._peak_bytes, + "peak_gib": self._peak_bytes / 2 ** 30, + "host_current_bytes": self._current.get(self._HOST_DEVICE, 0), + "host_peak_bytes": self._peaks.get(self._HOST_DEVICE, 0), + "devices": { + device: {"current_bytes": current_bytes, "peak_bytes": self._peaks[device]} + for device, current_bytes in self._current.items() + if device != self._HOST_DEVICE + }, + }, + "simulation": { + "memory": "logical storage bytes", + "segments": "synthetic, one per allocation, immediately returned on free", + "timeline": "event order, not wall-clock time", + "streams": "synchronous, stream 0", + "workspace": "not simulated", + "fragmentation": "not simulated", + }, + }) + + +def write_memory_snapshot(snapshot: dict[str, Any], output_path: str) -> str: + """Atomically write a native-compatible pickle snapshot. + + Args: + snapshot: LogicalMemoryTracker snapshot, optionally with run metadata. + output_path: Destination pickle path. + + Returns: + Absolute destination path. + """ + destination = Path(output_path).expanduser().resolve() + destination.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_path = tempfile.mkstemp(prefix=f".{destination.name}.", dir=str(destination.parent)) + try: + with os.fdopen(descriptor, "wb") as snapshot_file: + # The browser viewer supports a subset of pickle, with protocol 4 + # also accepted by older deployed MemoryViz versions. + pickle.dump(snapshot, snapshot_file, protocol=4) + os.replace(temporary_path, destination) + except Exception: + if os.path.exists(temporary_path): + os.unlink(temporary_path) + raise + return str(destination) + + +__all__ = ["LogicalMemoryTracker", "write_memory_snapshot"] diff --git a/hyper_parallel/tools/dryrun/operator_support.py b/hyper_parallel/tools/dryrun/operator_support.py new file mode 100644 index 0000000000..c5ac6eff71 --- /dev/null +++ b/hyper_parallel/tools/dryrun/operator_support.py @@ -0,0 +1,270 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Extensible FakeTensor operator support for dry-run. + +The built-in kernels cover Ascend operators used by DeepSeek-V4.1. Users can +register additional shape-only kernels with :func:`register_fake_op`. +Registration takes effect immediately when the defining module is imported. + +Every kernel below only derives output *shapes*; gradients mirror the +differentiated forward inputs. +""" + +import logging +from collections.abc import Callable +from typing import Any + +import torch # pylint: disable=forbidden-backend-import + +logger = logging.getLogger(__name__) + + +class DryRunOpRegistry: + """Register and load shape-only implementations for custom operators.""" + + def __init__(self) -> None: + """Create an empty registry tied to the current process.""" + self._registered: set[str] = set() + + def register_fake( + self, + op_name: str, + kernel: Callable[..., Any] | None = None, + *, + allow_missing: bool = False, + ) -> Callable[..., Any]: + """Register a FakeTensor kernel. + + Args: + op_name: Operator name such as ``custom::my_forward``. + kernel: Shape-only implementation; omit it when using this method + as a decorator. + allow_missing: Keep optional vendor kernels non-fatal when the + operator is unavailable in the current backend build. + + Returns: + The registered kernel or a decorator when ``kernel`` is omitted. + """ + if not isinstance(op_name, str) or "::" not in op_name: + raise ValueError("op_name must use the 'namespace::operator' form") + + def decorator(function: Callable[..., Any]) -> Callable[..., Any]: + if not callable(function): + raise TypeError("FakeTensor kernel must be callable") + if op_name in self._registered: + return function + try: + torch.library.register_fake(op_name)(function) + except RuntimeError as error: + if allow_missing and "does not exist" in str(error): + logger.debug("Optional operator %s is unavailable: %s", op_name, error) + return function + raise RuntimeError( + f"Unable to register dry-run kernel for {op_name!r}: {error}" + ) from error + self._registered.add(op_name) + logger.info("Registered Dry-run FakeTensor kernel for %s", op_name) + return function + + return decorator if kernel is None else decorator(kernel) + + @property + def registered_ops(self) -> tuple[str, ...]: + """Return registered operator names for diagnostics.""" + return tuple(sorted(self._registered)) + + +_REGISTRY = DryRunOpRegistry() +_BUILTINS_REGISTERED = False + + +def get_dryrun_op_registry() -> DryRunOpRegistry: + """Return the process-wide dry-run operator registry.""" + return _REGISTRY + + +def register_fake_op( + op_name: str, + kernel: Callable[..., Any] | None = None, + *, + allow_missing: bool = False, +) -> Callable[..., Any]: + """Register a custom FakeTensor kernel on the global dry-run registry.""" + return _REGISTRY.register_fake(op_name, kernel, allow_missing=allow_missing) + + +def _register(op_name, kernel): + """Register one Meta kernel if the vendor operator is present.""" + op = getattr(getattr(torch.ops, "custom", None), op_name, None) + if op is None: + return False + full_name = f"custom::{op_name}" + try: + _REGISTRY.register_fake(full_name, kernel) + except RuntimeError as error: # already registered by the vendor + logger.debug("Meta kernel for %s already present: %s", full_name, error) + return False + return True + + +def _register_host_memory_ops() -> None: + """Register fake kernels for host-pinning ops the offload paths call. + + ``aten::_pin_memory`` has no Meta kernel and pinning needs a live device + runtime anyway. Logically a pinned mirror is the same host bytes — pinning + only changes DMA speed — so the fake kernel returns an ordinary CPU clone. + """ + try: + torch.library.register_fake("aten::_pin_memory")( + lambda self, device=None: torch.empty_like(self) + ) + except RuntimeError as error: + logger.debug("aten::_pin_memory fake kernel already present: %s", error) + + +def register_builtin_dryrun_ops() -> None: + """Register Meta kernels for vendor forward/backward ops missing them.""" + global _BUILTINS_REGISTERED # pylint: disable=global-statement + if _BUILTINS_REGISTERED: + return + _BUILTINS_REGISTERED = True + _register_host_memory_ops() + if getattr(torch.ops, "custom", None) is None: + return + + def _sparse_attention_grad( + query, key, value, sparse_indices, d_out, out, softmax_max, softmax_sum, + scale_value, sparse_block_size, query_rope=None, key_rope=None, + actual_seq_qlen=None, actual_seq_kvlen=None, layout="BSND", + sparse_mode=3, pre_tokens=9223372036854775807, + next_tokens=9223372036854775807, attention_mode=0, deterministic=False, + ): + """Gradients of query/key/value/rope mirror the forward inputs.""" + del (sparse_indices, d_out, out, softmax_max, softmax_sum, scale_value, + sparse_block_size, actual_seq_qlen, actual_seq_kvlen, layout, + sparse_mode, pre_tokens, next_tokens, attention_mode, deterministic) + grad_query_rope = ( + torch.empty_like(query_rope) if query_rope is not None else torch.empty_like(query) + ) + grad_key_rope = ( + torch.empty_like(key_rope) if key_rope is not None else torch.empty_like(key) + ) + return ( + torch.empty_like(query), + torch.empty_like(key), + torch.empty_like(value), + grad_query_rope, + grad_key_rope, + ) + + def _indexer_kl_loss_grad( + query, key, query_index, key_index, weights, sparse_indices, + softmax_max, softmax_sum, scale_value, query_rope=None, key_rope=None, + actual_seq_qlen=None, actual_seq_klen=None, layout="BSND", + sparse_mode=3, pre_tokens=2147483647, next_tokens=2147483647, + deterministic=False, sparse_block_size=1, + ): + """Indexer gradients mirror their weights; the inner loss is scalar.""" + del (query, key, sparse_indices, softmax_max, softmax_sum, scale_value, + query_rope, key_rope, actual_seq_qlen, actual_seq_klen, layout, + sparse_mode, pre_tokens, next_tokens, deterministic, sparse_block_size) + return ( + torch.empty_like(query_index), + torch.empty_like(key_index), + torch.empty_like(weights), + torch.empty((), dtype=torch.float32, device=weights.device), + ) + + def _aggregate_hidden_grad(grad_output, input, weight, mask=None): + """Gradients mirror the differentiated forward inputs.""" + del grad_output, mask + return torch.empty_like(input), torch.empty_like(weight) + + def _mhc_pre_grad( + x, phi, alpha, h_in_grad, h_post_grad, h_comb_before_grad, inv_rms, + mm_res, h_pre, h_post, gamma=None, hc_eps=1.0e-6, grad_x_post=None, + ): + """Gradients follow (x, phi, alpha, beta, gamma) of the forward.""" + del (h_in_grad, h_post_grad, h_comb_before_grad, inv_rms, mm_res, h_pre, + h_post, hc_eps, grad_x_post) + grad_gamma = ( + torch.empty_like(gamma) if gamma is not None else torch.empty_like(alpha) + ) + return ( + torch.empty_like(x), + torch.empty_like(phi), + torch.empty_like(alpha), + torch.empty_like(alpha), + grad_gamma, + ) + + def _flash_attention_score_enhance( + query, key, value, head_num, input_layout, pse=None, + padding_mask=None, atten_mask=None, sink_tensor=None, query_rope=None, + key_rope=None, scale=1.0, keep_prob=1.0, pre_tokens=2147483647, + next_tokens=2147483647, inner_precise=0, prefix=None, + actual_seq_qlen=None, actual_seq_kvlen=None, sparse_mode=0, sink_num=0, + pse_type=1, softmaxOutLayout="", q_start_idx=None, kv_start_idx=None, + ): + """Attention output matches the query; LSE carries per-head scalars.""" + del (key, value, pse, padding_mask, atten_mask, sink_tensor, query_rope, + key_rope, scale, keep_prob, pre_tokens, next_tokens, inner_precise, + prefix, actual_seq_qlen, actual_seq_kvlen, sparse_mode, sink_num, + pse_type, softmaxOutLayout, q_start_idx, kv_start_idx) + tokens = query.shape[0] if input_layout == "TND" else query.shape[1] + lse_shape = (head_num, int(tokens), 8) + device = query.device + return ( + torch.empty_like(query), + torch.empty(lse_shape, dtype=torch.float32, device=device), + torch.empty(lse_shape, dtype=torch.float32, device=device), + ) + + for name, kernel in ( + ("npu_sparse_flash_attention_grad_enhance", _sparse_attention_grad), + ("npu_sparse_lightning_indexer_grad_kl_loss_enhance", _indexer_kl_loss_grad), + ("npu_aggregate_hidden_grad", _aggregate_hidden_grad), + ("npu_manifold_constrained_hyper_connection_pre_grad", _mhc_pre_grad), + ("npu_flash_attention_score_enhance", _flash_attention_score_enhance), + ): + _register(name, kernel) + + +def unsupported_operator_hint(error: BaseException) -> str | None: + """Return an actionable extension hint for an unsupported FakeTensor op.""" + message = str(error) + if "UnsupportedOperatorException" not in message and "unsupported operator" not in message.lower(): + return None + return ( + f"{message}\n" + "Dry-run could not derive this custom operator under FakeTensor.\n" + "Register a shape-only kernel before running dry-run, for example:\n" + " from hyper_parallel.tools.dryrun import register_fake_op\n" + " @register_fake_op(\"custom::my_operator\")\n" + " def my_operator_fake(...):\n" + " return torch.empty_like(input_tensor)\n" + "Import the defining module from the model integration code so registration runs.\n" + "Registered ops: " + f"{', '.join(_REGISTRY.registered_ops) or ''}." + ) + + +__all__ = [ + "DryRunOpRegistry", + "get_dryrun_op_registry", + "register_builtin_dryrun_ops", + "register_fake_op", + "unsupported_operator_hint", +] diff --git a/hyper_parallel/tools/dryrun/runner.py b/hyper_parallel/tools/dryrun/runner.py new file mode 100644 index 0000000000..4847282437 --- /dev/null +++ b/hyper_parallel/tools/dryrun/runner.py @@ -0,0 +1,1008 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Standalone configurable FakeTensor dry-run for the HyperModels LLM Trainer.""" +# pylint: disable=no-member,unsupported-binary-operation + +import logging +import os +import re +from copy import copy +from collections.abc import Mapping +from contextlib import contextmanager +from dataclasses import dataclass, fields, is_dataclass +from pathlib import Path +from typing import Any, Iterator, Optional + +import torch # pylint: disable=forbidden-backend-import +import torch.distributed as dist # pylint: disable=forbidden-backend-import +from torch import nn # pylint: disable=forbidden-backend-import +from torch._subclasses.fake_tensor import FakeTensorMode # pylint: disable=forbidden-backend-import +from torch._subclasses.fake_tensor import unset_fake_temporarily # pylint: disable=forbidden-backend-import + +from hyper_parallel.data.batching.build_dataloader import calculate_num_micro_batches +from hyper_parallel.distributed.mesh import DistributedSetup, MeshContext +from hyper_parallel.models.build_options import get_device_type +from hyper_parallel.trainer.base import BaseTrainer +from hyper_parallel.trainer.config import TrainerConfig, normalize_distributed_setup_overrides +from hyper_parallel.trainer.runtime import fsdp as fsdp_runtime +from hyper_parallel.trainer.runtime import memory_profiler as memory_profiler_module +from hyper_parallel.trainer.runtime.device import simulate_device_runtime +from hyper_parallel.trainer.runtime.distributed import destroy_process_group as destroy_distributed_runtime +from hyper_parallel.trainer.runtime.random import set_seed +from hyper_parallel.trainer.text_trainer import TextTrainer +from hyper_parallel.core.dtensor.device_mesh import DeviceMesh +from hyper_parallel.core.fully_shard import api as fully_shard_api +from hyper_parallel.tools.dryrun import runtime as torch_dry_run +from hyper_parallel.tools.dryrun.compatibility import ( + install_dryrun_compatibility, + model_build_context, +) +from hyper_parallel.tools.dryrun.config import DryRunConfig, DryRunSnapshotConfig +from hyper_parallel.tools.dryrun.data import DryRunDataProbe, PreparedDryRunBatch +from hyper_parallel.tools.dryrun.memory import LogicalMemoryTracker, write_memory_snapshot +from hyper_parallel.tools.dryrun.operator_support import ( + register_builtin_dryrun_ops, + unsupported_operator_hint, +) + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class _DryRunTrainingBatch: + """Value-free normal-training inputs and their CPU-derived statistics.""" + + model_inputs: dict[str, Any] + loss_inputs: dict[str, Any] + token_counts: dict[str, int] + valid_token_count: int + target_tokens_per_rank: tuple[int, ...] + + +class HyperModelsDryRunRunner: + """Execute one shape-accurate fake LLM training step and report memory.""" + + def __init__( + self, + config: TrainerConfig, + runtime: Optional[torch_dry_run.DryRunRuntime] = None, + ) -> None: + """Store configuration without changing distributed or device state.""" + self.config = config + self._runtime = runtime + self._stage = "validation" + self._target_device = "cpu" + self._simulation_device = "cpu" + self._resolved_dtype = torch.get_default_dtype() + install_dryrun_compatibility() + + @staticmethod + def _check_torch_version() -> None: + """Require the FakeTensor collective tracking available in PyTorch 2.7.""" + match = re.match(r"^(\d+)\.(\d+)", torch.__version__) + version = tuple(int(part) for part in match.groups()) if match else (0, 0) + if version < (2, 7): + raise RuntimeError( + "HyperModelsDryRunRunner requires PyTorch >= 2.7, found " + f"{torch.__version__}" + ) + + def _get_runtime(self) -> torch_dry_run.DryRunRuntime: + """Return the validated torchrun identity.""" + if self._runtime is None: + dry_run = self.config.dry_run + virtual_size = dry_run.virtual_world_size if isinstance(dry_run, DryRunConfig) else None + torchrun_env = {"RANK", "WORLD_SIZE", "LOCAL_RANK"} + if torchrun_env <= os.environ.keys(): + self._runtime = torch_dry_run.DryRunRuntime.from_torchrun_env() + if virtual_size is not None and virtual_size != self._runtime.world_size: + raise ValueError( + "dry_run.virtual_world_size must match torchrun WORLD_SIZE: " + f"{virtual_size} versus {self._runtime.world_size}" + ) + else: + self._runtime = torch_dry_run.DryRunRuntime( + rank=0, + world_size=virtual_size or 1, + local_rank=0, + ) + return self._runtime + + def _validate_config(self) -> DryRunConfig: + """Validate static Dry-run and unsupported-feature boundaries.""" + self._check_torch_version() + raw_dry_run = self.config.dry_run + if raw_dry_run is None: + raise ValueError("TrainerConfig.dry_run must be configured") + if isinstance(raw_dry_run, Mapping): + try: + raw_dry_run = DryRunConfig.from_mapping(raw_dry_run) + except TypeError as error: + raise ValueError(f"Invalid dry_run configuration: {error}") from error + if not isinstance(raw_dry_run, DryRunConfig): + raise TypeError("TrainerConfig.dry_run must resolve to a DryRunConfig") + dry_run = raw_dry_run + self.config.dry_run = dry_run + if not dry_run.enabled: + raise ValueError("TrainerConfig.dry_run.enabled must be true") + if not isinstance(dry_run.snapshot, DryRunSnapshotConfig): + raise TypeError("dry_run.snapshot must resolve to DryRunSnapshotConfig") + if dry_run.snapshot.enabled and ( + not isinstance(dry_run.snapshot.path, str) or not dry_run.snapshot.path.strip() + ): + raise ValueError("dry_run.snapshot.path must be a non-empty string") + if not isinstance(dry_run.snapshot.capture_stacks, bool): + raise TypeError("dry_run.snapshot.capture_stacks must be a boolean") + if ( + isinstance(dry_run.snapshot.stack_depth, bool) + or not isinstance(dry_run.snapshot.stack_depth, int) + or not 1 <= dry_run.snapshot.stack_depth <= 256 + ): + raise ValueError("dry_run.snapshot.stack_depth must be an integer in [1, 256]") + if dry_run.ranks is not None: + if ( + not isinstance(dry_run.ranks, list) + or not dry_run.ranks + or any( + isinstance(rank, bool) or not isinstance(rank, int) or rank < 0 + for rank in dry_run.ranks + ) + or len(set(dry_run.ranks)) != len(dry_run.ranks) + ): + raise ValueError("dry_run.ranks must be a non-empty list of unique non-negative integers") + if dry_run.virtual_world_size is not None and ( + isinstance(dry_run.virtual_world_size, bool) + or not isinstance(dry_run.virtual_world_size, int) + or dry_run.virtual_world_size < 1 + ): + raise ValueError("dry_run.virtual_world_size must be a positive integer") + if self.config.training.micro_batch_size < 1: + raise ValueError("training.micro_batch_size must be >= 1") + activation_checkpoint = self.config.activation_checkpoint.mode + if activation_checkpoint not in ("off", "none", "full", "selective"): + raise NotImplementedError( + "LLM Dry-run does not support activation checkpoint " + f"mode {activation_checkpoint!r}" + ) + if getattr( + self.config.fsdp_config, + "activation_checkpointing", + None, + ) not in (False, None, "off", "none"): + raise NotImplementedError("LLM Dry-run does not support FSDP activation checkpointing") + if self.config.peft is not None: + raise NotImplementedError("LLM Dry-run does not support PEFT model mutation") + if self.config.compile.enabled: + raise NotImplementedError("LLM Dry-run does not support torch.compile") + torch_dry_run._DryRunValueProfile(dry_run) # pylint: disable=protected-access + self._validate_topology() + if self.config.accelerator.pp_size != 1: + raise NotImplementedError("Dry-run does not support pipeline parallelism") + return dry_run + + def _validate_topology(self) -> None: + """Validate the configured topology against the launcher world size.""" + runtime = self._get_runtime() + if self.config.dry_run.ranks is not None and any( + rank >= runtime.world_size for rank in self.config.dry_run.ranks + ): + raise ValueError( + f"dry_run.ranks must be smaller than virtual world size {runtime.world_size}" + ) + accelerator = self.config.accelerator + topology_sizes = { + "tp_size": accelerator.tp_size, + "cp_size": accelerator.cp_size, + "ep_size": accelerator.ep_size, + "pp_size": accelerator.pp_size, + "dp_shard_size": self.config.fsdp_config.dp_shard_size, + "edp_shard_size": self.config.fsdp_config.edp_shard_size, + } + invalid_sizes = { + name: size + for name, size in topology_sizes.items() + if not isinstance(size, int) or isinstance(size, bool) or size < 1 + } + if invalid_sizes: + raise ValueError( + "Dry-run parallel sizes must be positive integers, got " + f"{invalid_sizes}" + ) + non_dp_size = accelerator.tp_size * accelerator.cp_size * accelerator.pp_size + if runtime.world_size % non_dp_size: + raise ValueError( + f"WORLD_SIZE {runtime.world_size} is not divisible by TP*CP*PP " + f"size {non_dp_size}" + ) + dp_size = runtime.world_size // non_dp_size + fsdp_domain_size = dp_size * max(1, accelerator.cp_size) + if fsdp_domain_size % self.config.fsdp_config.dp_shard_size: + raise ValueError( + f"DP+CP size {fsdp_domain_size} is not divisible by " + f"dp_shard_size {self.config.fsdp_config.dp_shard_size}" + ) + expert_domain_size = dp_size * accelerator.cp_size * accelerator.tp_size + if expert_domain_size % accelerator.ep_size: + raise ValueError( + f"expert domain size {expert_domain_size} is not divisible by " + f"ep_size {accelerator.ep_size}" + ) + expert_dp_size = expert_domain_size // accelerator.ep_size + if expert_dp_size % self.config.fsdp_config.edp_shard_size: + raise ValueError( + f"expert DP size {expert_dp_size} is not divisible by " + f"edp_shard_size {self.config.fsdp_config.edp_shard_size}" + ) + if self.config.dry_run.snapshot.enabled: + self._snapshot_path(self.config.dry_run, runtime) + + @staticmethod + def _snapshot_path( + dry_run: DryRunConfig, + runtime: torch_dry_run.DryRunRuntime, + ) -> Optional[str]: + """Create the rank directory and fixed pickle name under the output path.""" + if not dry_run.snapshot.enabled: + return None + return str( + Path(dry_run.snapshot.path).expanduser() + / f"rank_{runtime.rank}" + / "memory.pickle" + ) + + + @staticmethod + def _prewarm_standard_meshes(mesh_context: MeshContext) -> None: + """Materialize child meshes used by the non-pipeline training path.""" + meshes = [mesh_context.device_mesh, mesh_context.fsdp_non_moe_mesh] + for axis in ("dp", "cp", "tp"): + meshes.append(mesh_context.device_mesh[axis]) + active_names = tuple( + axis + for axis, size in (("cp", mesh_context.cp_size), ("tp", mesh_context.tp_size)) + if size > 1 + ) + if active_names: + active_mesh = mesh_context.device_mesh[active_names] + meshes.append(active_mesh) + meshes.extend(active_mesh[axis] for axis in active_names) + if mesh_context.dp_cp_mesh is not None: + meshes.append(mesh_context.dp_cp_mesh) + dense_selector: str | tuple[str, str] = "fsdp_shard" + if mesh_context.dp_replicate_size > 1: + dense_selector = ("fsdp_replicate", "fsdp_shard") + meshes.append(mesh_context.fsdp_non_moe_mesh[dense_selector]) + if mesh_context.fsdp_moe_mesh is not None: + meshes.extend((mesh_context.fsdp_moe_mesh, mesh_context.fsdp_moe_mesh["ep"])) + expert_selector: str | tuple[str, str] = "edp_shard" + if "edp_replicate" in mesh_context.fsdp_moe_mesh.mesh_dim_names: + expert_selector = ("edp_replicate", "edp_shard") + meshes.append(mesh_context.fsdp_moe_mesh[expert_selector]) + for mesh in meshes: + for mesh_dim in range(mesh.ndim): + mesh.get_group(mesh_dim) + + def _build_distributed_setup( + self, + runtime: torch_dry_run.DryRunRuntime, + device_type: str, + ) -> DistributedSetup: + """Build the standard non-pipeline mesh and sharding setup.""" + accelerator = self.config.accelerator + tp_size = accelerator.tp_size + cp_size = accelerator.cp_size + dp_size = runtime.world_size // (tp_size * cp_size) + dp_shard_size = self.config.fsdp_config.dp_shard_size + dp_replicate_size = dp_size * cp_size // dp_shard_size + mesh_context = MeshContext( + dp_size=dp_size, + dp_replicate_size=dp_replicate_size, + dp_shard_size=dp_shard_size, + edp_shard_size=self.config.fsdp_config.edp_shard_size, + tp_size=tp_size, + cp_size=cp_size, + pp_size=1, + ep_size=accelerator.ep_size, + sequence_parallel=bool(accelerator.sequence_parallel), + loss_parallel=bool(accelerator.loss_parallel), + ) + mesh_context.build_meshs(device_type, runtime.world_size) + mesh_context.dp_rank = mesh_context.device_mesh.get_local_rank("dp") + mesh_context.tp_rank = mesh_context.device_mesh.get_local_rank("tp") + mesh_context.cp_rank = mesh_context.device_mesh.get_local_rank("cp") + mesh_context.ep_rank = ( + mesh_context.fsdp_moe_mesh.get_local_rank("ep") + if mesh_context.fsdp_moe_mesh is not None + else 0 + ) + mesh_context.pp_rank = 0 + self._prewarm_standard_meshes(mesh_context) + fsdp_enabled = ( + dp_shard_size > 1 + or dp_replicate_size > 1 + or self.config.fsdp_config.edp_shard_size > 1 + ) + return DistributedSetup( + mesh_context=mesh_context, + strategy_config=self.config.fsdp_config if fsdp_enabled else None, + plan_overrides=self.config.plan_overrides, + low_precision_config=getattr(self.config.training, "low_precision", None), + fp32_main_params=self.config.optimizer.fp32_main_params, + ) + + def _select_simulation_device(self, target_device: str) -> str: + """Select a registered FakeTensor device without probing real hardware.""" + backend_registered = ( + torch.version.cuda is not None + if target_device == "cuda" + else hasattr(torch, "npu") + ) + if not backend_registered: + logger.info( + "PyTorch has no %s device guard; using CPU FakeTensor simulation", + target_device, + ) + return "cpu" + try: + # A backend with zero visible devices aborts inside its lazy + # device binding the first time any op touches a device-typed + # tensor; simulate on CPU instead of crashing mid-step. + device_module = getattr(torch, target_device) + if int(device_module.device_count()) < 1: + logger.info( + "No visible %s device; using CPU FakeTensor simulation", + target_device, + ) + self._neutralize_device_capability_probes(target_device) + return "cpu" + target = torch.device(target_device) + with FakeTensorMode(): + torch.empty((), device=target) + return target_device + except (AttributeError, RuntimeError): + logger.info( + "FakeTensor device %s is not registered; using CPU simulation", + target_device, + ) + return "cpu" + + @staticmethod + def _neutralize_device_capability_probes(target_device: str) -> None: + """Answer global capability queries without initializing the backend. + + ``torch.utils.checkpoint`` recompute preserves per-backend autocast + state, and ``torch_npu.npu.is_bf16_supported`` unconditionally + lazy-initializes the NPU runtime — which aborts when zero devices are + visible. The answer is a property of the *target* accelerator, not of + this host, so pin it to the simulated target's capability. + """ + if target_device != "npu" or not hasattr(torch, "npu"): + return + torch.npu.is_bf16_supported = ( # type: ignore[attr-defined] + lambda including_emulation=False: True + ) + + def _resolve_target_device(self) -> str: + """Resolve the normal-training accelerator type without binding a device. + + ``dry_run.target_device`` states the accelerator the simulated plan is + for; environment detection is only a fallback so capacity planning can + run on hosts without that accelerator (or with none visible at all). + """ + target_device = getattr(self.config.dry_run, "target_device", None) + if target_device not in ("cuda", "npu"): + target_device = get_device_type() + if target_device not in ("cuda", "npu"): + raise RuntimeError( + "HyperModels Dry-run requires dry_run.target_device or an " + f"available CUDA or NPU runtime; detected {target_device!r}" + ) + self._target_device = target_device + return target_device + + @staticmethod + def _init_cpu_data_process_group(runtime: torch_dry_run.DryRunRuntime) -> None: + """Initialize a temporary Gloo group for the normal data pipeline.""" + if dist.is_initialized(): + raise RuntimeError("Dry-run data probing must start before process-group initialization") + os.environ.setdefault("GLOO_SOCKET_IFNAME", "lo") + kwargs = { + "backend": "gloo", + "world_size": runtime.world_size, + "rank": runtime.rank, + } + if runtime.world_size == 1: + kwargs["store"] = dist.HashStore() + dist.init_process_group(**kwargs) + + @staticmethod + def _init_fake_process_group(runtime: torch_dry_run.DryRunRuntime) -> None: + """Initialize PyTorch's fake backend using the torchrun identity.""" + if dist.is_initialized(): + raise RuntimeError("Dry-run must start before any process group is initialized") + # Import registers the version-matched fake backend factory. + # pylint: disable-next=C0415,unused-import,forbidden-backend-import + from torch.testing._internal.distributed import fake_pg + + dist.init_process_group( + backend=dist.Backend.FAKE, + store=dist.HashStore(), + world_size=runtime.world_size, + rank=runtime.rank, + ) + + def _prepare_data_base( + self, + setup: DistributedSetup, + model: nn.Module, + device: Optional[torch.device] = None, + ) -> BaseTrainer: + """Create the BaseTrainer state consumed by shared text-data setup.""" + runtime = self._get_runtime() + base = BaseTrainer.__new__(BaseTrainer) + base.config = self.config + base.local_rank = runtime.local_rank + base.global_rank = runtime.rank + base.world_size = runtime.world_size + base.device = torch.device("cpu") if device is None else device + base.distributed_setup = setup + base.mesh = setup.mesh_context + base.device_mesh = base.mesh.device_mesh + base.dp_cp_mesh = base.mesh.dp_cp_mesh + base.model_config = model.config + if base.config.training.seed is None: + base.config.training.seed = base.default_seed + set_seed( + base.config.training.seed, + base.config.training.enable_full_determinism, + ) + return base + + def _read_training_batch( + self, + setup: DistributedSetup, + model: nn.Module, + device: Optional[torch.device] = None, + ) -> PreparedDryRunBatch: + """Read one CPU batch through the normal TextTrainer data lifecycle.""" + data_runtime = DryRunDataProbe(self._prepare_data_base(setup, model, device)) + try: + data_runtime.build() + return data_runtime.read_first_batch() + finally: + data_runtime.close() + + def _mock_training_batch( + self, + batch: PreparedDryRunBatch, + fake_mode: FakeTensorMode, + model: nn.Module, + tp_size: int, + ) -> _DryRunTrainingBatch: + """Erase CPU values and retain only the normal batch structure.""" + valid_count, owned = torch_dry_run.derive_tp_target_counts( + batch.loss_inputs, + int(model.config.vocab_size), + tp_size, + ) + mocker = torch_dry_run.DryRunBatchMocker( + fake_mode, + self._simulation_torch_device(), + ) + return _DryRunTrainingBatch( + model_inputs=mocker.mock(batch.model_inputs), + loss_inputs=mocker.mock(batch.loss_inputs), + token_counts=dict(batch.token_counts), + valid_token_count=valid_count, + target_tokens_per_rank=owned, + ) + + @contextmanager + def _dry_run_fsdp_device(self) -> Iterator[None]: + """Temporarily resolve FSDP storage to the simulation device.""" + original = fully_shard_api._get_device_from_mesh # pylint: disable=protected-access + original_concatenate = DeviceMesh.concatenate + simulation_device = self._simulation_torch_device() + + def _resolve_device(mesh: Any) -> torch.device: + del mesh + return simulation_device + + def _concatenate_meshes(meshes: Any) -> DeviceMesh: + # Mesh rank maps are host metadata and must never become FakeTensors. + with unset_fake_temporarily(): + return original_concatenate(meshes) + + fully_shard_api._get_device_from_mesh = _resolve_device # pylint: disable=protected-access + DeviceMesh.concatenate = staticmethod(_concatenate_meshes) + try: + yield + finally: + fully_shard_api._get_device_from_mesh = original # pylint: disable=protected-access + DeviceMesh.concatenate = staticmethod(original_concatenate) + + def _simulation_torch_device(self) -> torch.device: + """Return the rank-local logical device used by FakeTensor execution.""" + if self._simulation_device == "cpu": + return torch.device("cpu") + return torch.device( + self._simulation_device, + self._get_runtime().local_rank, + ) + + def _build_target_model( + self, + setup: DistributedSetup, + ) -> nn.Module: + """Build the configured model through the normal deferred Target path.""" + with model_build_context(): + model = self.config.model.build( + distributed_setup=setup, + peft_config=self.config.peft, + activation_checkpoint=self.config.activation_checkpoint.mode, + activation_checkpoint_selection=self.config.activation_checkpoint.selection, + swap_inputs=getattr(self.config.activation_checkpoint, "swap_inputs", False), + activation_swap=self.config.activation_swap, + compile_config=self.config.compile, + model_init_dtype=self.config.model_init_dtype, + ) + return model + + def _record_model_dtype(self, model: nn.Module) -> None: + """Record the effective floating dtype from the normally built model.""" + self._resolved_dtype = next( + (tensor.dtype for tensor in model.parameters() if tensor.is_floating_point()), + torch.get_default_dtype(), + ) + + def _prepare_base( + self, + setup: DistributedSetup, + model: nn.Module, + ) -> BaseTrainer: + """Build the real Trainer state for one fake step via its own stages. + + Every stage below is the unmodified BaseTrainer/TextTrainer code; only + the environment preparation that ``_setup()`` performs (process group, + device binding) is owned by dry-run, which has already installed the + fake process group and enabled the simulated device runtime. Driving + the real initialization keeps dry-run's execution identical to a + normal step by construction instead of by imitation. + """ + runtime = self._get_runtime() + base = BaseTrainer.__new__(BaseTrainer) + base.config = self.config + base.local_rank = runtime.local_rank + base.global_rank = runtime.rank + base.world_size = runtime.world_size + base.device = self._simulation_torch_device() + base.distributed_setup = setup + base.mesh = setup.mesh_context + base.device_mesh = base.mesh.device_mesh + base.dp_cp_mesh = base.mesh.dp_cp_mesh + # The model was already built and parallelized by dry-run's deferred + # path; adopt it and run the real _build_model tail (parts, HSDP list, + # integration session, diagnostics) through the same statements. + base.model = model + base.model_config = model.config + model_parts = getattr(model, "parts", None) + base.model_parts = list(model_parts) if model_parts is not None else [model] + base.hsdp_model_parts = [ + module for module in model.modules() + if hasattr(module, "hsdp_scheduler") + ] + from hyper_parallel.trainer.runtime import ( # pylint: disable=C0415 + model_integration as model_integration_runtime, + ) + base.model_integration = model_integration_runtime.build_model_integration_session( + base.model, base.config, base.global_rank, base.mesh, + ) + base.fsdp_runtime_diagnostics = ( + fsdp_runtime.build_fsdp_runtime_diagnostics(base.hsdp_model_parts) + if getattr(self.config.debug, "check_fsdp_runtime", False) + else None + ) + base._build_loss() # pylint: disable=protected-access + base._build_optimizer() # pylint: disable=protected-access + base.model_integration.attach_optimizer(base.optimizer) + base.num_micro_batches = calculate_num_micro_batches( + global_batch_size=self.config.training.global_batch_size, + micro_batch_size=self.config.training.micro_batch_size, + dp_world_size=base.mesh.dp_size, + ) + base._compute_train_iters() # pylint: disable=protected-access + if self.config.lr_scheduler is not None: + base._build_lr_scheduler() # pylint: disable=protected-access + else: + base.lr_scheduler = None + base._build_training_context() # pylint: disable=protected-access + base.graph_compiler = None + base._init_callbacks() # pylint: disable=protected-access + return base + + def _prepare_text_trainer(self, base: BaseTrainer, batch: _DryRunTrainingBatch) -> TextTrainer: + """Attach the real TextTrainer facade and the replayed data runtime.""" + trainer = TextTrainer.__new__(TextTrainer) + trainer.base = base + base.train_dataloader = None + base.current_token_counts = {} + + def replay_get_batch(iterator: Any) -> tuple: + """Serve the mocked micro-batch through the get_batch contract.""" + return next(iterator) + + base.get_batch = replay_get_batch + base.model_integration.attach_data_pipeline( + runtime_adapter=None, + get_batch=base.get_batch, + model_assets=(base.model_config,), + ) + del batch + return trainer + + def _materialize_fake_model(self, model: nn.Module) -> nn.Module: + """Materialize meta state as FakeTensors without assigning values.""" + fake_parameter = next(( + parameter + for parameter in model.parameters() + if hasattr(parameter, "fake_mode") + ), None) + if fake_parameter is None: + # FSDP may move every parameter into its internal state. + model.to_empty(device=self._simulation_torch_device()) + return model + converter = fake_parameter.fake_mode.fake_tensor_converter.meta_converter + tensor_memo = converter.tensor_memo + tensor_memo.clear() + converter.tensor_memo = {} + try: + model.to_empty(device=self._simulation_torch_device()) + finally: + converter.tensor_memo = tensor_memo + return model + + @staticmethod + def _initialize_flat_buffers(model: nn.Module) -> None: + """Materialize enabled zero-copy FSDP flat shards before tracking.""" + visited_states = set() + for module in model.modules(): + scheduler = getattr(module, "hsdp_scheduler", None) + state = getattr(scheduler, "hsdp_state", None) + if state is None or id(state) in visited_states: + continue + visited_states.add(id(state)) + # Match the real forward-pre-hook order: dtype attributes must be + # initialized before communication buckets choose their dtype. + state.lazy_init() + param_group = getattr(state, "param_group", None) + if param_group is not None and param_group.enable_zero_copy: + param_group._init_all_gather_buckets() # pylint: disable=protected-access + for bucket in param_group.all_gather_buckets: + bucket.init_flat_param_buffer(param_group.device) + + def _execute_step( + self, + base: BaseTrainer, + batch: _DryRunTrainingBatch, + profile: Any, + value_dependencies: Any, + ) -> dict[str, Any]: + """Drive one unmodified TextTrainer.train_step over the fake state.""" + self._initialize_flat_buffers(base.model) + tracker = LogicalMemoryTracker( + self._simulation_device, + self._get_runtime().local_rank, + capture_stacks=self.config.dry_run.snapshot.capture_stacks, + stack_depth=self.config.dry_run.snapshot.stack_depth, + ) + trainer = self._prepare_text_trainer(base, batch) + iterator = iter(torch_dry_run.DryRunBatchIterator( + batch.model_inputs, + batch.loss_inputs, + )) + scalar_shim = torch_dry_run.FakeHostScalarShim() + # The probe observed the real token counts; pin them so the + # gradient-normalization denominator reads its true host value. + for name, count in batch.token_counts.items(): + tensor = batch.loss_inputs.get(name) + if tensor is not None and hasattr(tensor, "fake_mode"): + scalar_shim.register(tensor, count) + num_micro_batches = base.num_micro_batches + with ( + tracker, + value_dependencies.fake_step_context(base), + scalar_shim, + ): + tracker.track(self._existing_tensors(base, batch)) + self._register_token_count_values(base, batch, scalar_shim) + trainer.train_step(iterator) + snapshot = tracker.snapshot() + snapshot["metadata"] = self._metadata(base, batch, profile, num_micro_batches=num_micro_batches) + return snapshot + + @staticmethod + def _register_token_count_values( + base: BaseTrainer, + batch: _DryRunTrainingBatch, + scalar_shim: Any, + ) -> None: + """Pin probe-derived token counts onto the counts the step will read. + + ``count_loss_token`` recomputes token counts from the mocked labels, + producing fake scalar tensors. Wrap it so every count tensor it emits + is registered with the probe-known true value before ``.item()`` runs. + """ + from hyper_parallel.trainer.runtime import loss_aggregation # pylint: disable=C0415 + from hyper_parallel.trainer import text_trainer as text_trainer_module # pylint: disable=C0415 + original = loss_aggregation.count_loss_token + known = dict(batch.token_counts) + + def counted(batches: Any) -> dict: + """Count tokens normally, then pin probe-known true values.""" + result = original(batches) + for name, tensor in result.items(): + if hasattr(tensor, "fake_mode") and name in known: + scalar_shim.register(tensor, known[name]) + return result + + # The step loop reads count_loss_token through the modules that + # imported it; patch both import sites for the step's duration. The + # fake step is single-shot per process, so no restore is needed before + # process exit, but keep the originals on the base for debuggability. + text_trainer_module.count_loss_token = counted + import hyper_parallel.trainer.base as base_module # pylint: disable=C0415 + base_module.count_loss_token = counted + base._dryrun_original_count_loss_token = original # pylint: disable=protected-access + + @classmethod + def _existing_tensors(cls, base: BaseTrainer, batch: _DryRunTrainingBatch) -> Iterator[torch.Tensor]: + """Seed storage allocated before the tracked step, without assigning roles.""" + yield from base.model.parameters() + yield from base.model.buffers() + yield from cls._training_batch_tensors(batch) + optimizers = getattr(base.optimizer, "chained_optimizers", None) + if optimizers is None: + optimizers = base.optimizer if isinstance(base.optimizer, list) else [base.optimizer] + for optimizer in optimizers: + for group in optimizer.param_groups: + yield from group["params"] + yield from cls._tensor_leaves(optimizer.state) + + def _metadata( + self, + base: BaseTrainer, + batch: _DryRunTrainingBatch, + profile: Any, + num_micro_batches: Optional[int] = None, + ) -> dict[str, Any]: + """Build reproducibility metadata for the memory snapshot.""" + runtime = self._get_runtime() + metadata = { + "model": type(base.model).__name__, + "torch_version": str(torch.__version__), + "rank": runtime.rank, + "world_size": runtime.world_size, + "local_rank": runtime.local_rank, + "target_device": self._target_device, + "simulation_device": self._simulation_device, + "initialization_device": "meta", + "weights_loaded": False, + "model_dtype": str(self._resolved_dtype), + "batch_shapes": { + "model_inputs": self._batch_shape_metadata(batch.model_inputs), + "loss_inputs": self._batch_shape_metadata(batch.loss_inputs), + }, + "token_counts": dict(batch.token_counts), + "parallel": { + "dp": base.mesh.dp_size, + "dp_replicate": base.mesh.dp_replicate_size, + "dp_shard": base.mesh.dp_shard_size, + "tp": base.mesh.tp_size, + "cp": base.mesh.cp_size, + "ep": base.mesh.ep_size, + "pp": base.mesh.pp_size, + }, + "reshard_after_forward": self.config.fsdp_config.reshard_after_forward, + "comm_fusion": self.config.fsdp_config.comm_fusion, + "value_dependency_replay": profile.metadata(), + } + if num_micro_batches is not None: + metadata["num_micro_batches"] = num_micro_batches + return metadata + + @classmethod + def _training_batch_tensors(cls, batch: _DryRunTrainingBatch) -> tuple[torch.Tensor, ...]: + """Return deduplicated tensor leaves from model and loss inputs.""" + tensors = [] + seen = set() + for value in (batch.model_inputs, batch.loss_inputs): + for tensor in cls._tensor_leaves(value): + if id(tensor) not in seen: + seen.add(id(tensor)) + tensors.append(tensor) + return tuple(tensors) + + @classmethod + def _tensor_leaves(cls, value: Any) -> tuple[torch.Tensor, ...]: + """Recursively collect tensor leaves from a mocked batch value.""" + if isinstance(value, torch.Tensor): + return (value,) + if isinstance(value, Mapping): + return tuple( + tensor + for item in value.values() + for tensor in cls._tensor_leaves(item) + ) + if isinstance(value, (list, tuple)): + return tuple( + tensor + for item in value + for tensor in cls._tensor_leaves(item) + ) + if is_dataclass(value) and not isinstance(value, type): + return tuple( + tensor + for field in fields(value) + for tensor in cls._tensor_leaves(getattr(value, field.name)) + ) + return () + + @classmethod + def _batch_shape_metadata(cls, value: Any) -> Any: + """Serialize nested batch tensor shapes without retaining values.""" + if isinstance(value, torch.Tensor): + return { + "shape": [int(size) for size in value.shape], + "dtype": str(value.dtype), + "layout": str(value.layout), + } + if isinstance(value, Mapping): + return {name: cls._batch_shape_metadata(item) for name, item in value.items()} + if isinstance(value, (list, tuple)): + return [cls._batch_shape_metadata(item) for item in value] + if is_dataclass(value) and not isinstance(value, type): + return { + field.name: cls._batch_shape_metadata(getattr(value, field.name)) + for field in fields(value) + } + return value + + def _probe_training_batch( + self, + runtime: torch_dry_run.DryRunRuntime, + fake_mode: FakeTensorMode, + ) -> _DryRunTrainingBatch: + """Read one CPU batch and erase its tensor values for simulation.""" + self._stage = "data_probe" + data_group_initialized = False + cpu_setup = None + try: + if self.config.dry_run.virtual_world_size is None: + self._init_cpu_data_process_group(runtime) + else: + self._init_fake_process_group(runtime) + data_group_initialized = True + cpu_setup = self._build_distributed_setup(runtime, "cpu") + normalize_distributed_setup_overrides(cpu_setup, self.config) + probe_setup = copy(cpu_setup) + probe_setup.strategy_config = None + probe_model = self._build_target_model(probe_setup) + cpu_batch = self._read_training_batch(cpu_setup, probe_model) + batch = self._mock_training_batch( + cpu_batch, + fake_mode, + probe_model, + cpu_setup.mesh_context.tp_size, + ) + del cpu_batch + del probe_model + return batch + finally: + cpu_setup = None + if data_group_initialized: + destroy_distributed_runtime() + + def _run_simulation( + self, + runtime: torch_dry_run.DryRunRuntime, + dry_run: DryRunConfig, + batch: _DryRunTrainingBatch, + fake_mode: FakeTensorMode, + snapshot_path: Optional[str], + ) -> dict[str, Any]: + """Execute one fake distributed step and persist its memory snapshot.""" + initialized_here = False + try: + self._stage = "distributed_setup" + self._init_fake_process_group(runtime) + initialized_here = True + setup = self._build_distributed_setup(runtime, self._simulation_device) + normalize_distributed_setup_overrides(setup, self.config) + + self._stage = "model_build" + profile = torch_dry_run._DryRunValueProfile(dry_run) # pylint: disable=protected-access + with fake_mode: + with self._dry_run_fsdp_device(): + model = self._build_target_model(setup) + self._record_model_dtype(model) + profile.bind_model(model) + value_dependencies = torch_dry_run.ValueDependencyManager( + profile, model, runtime, + ) + self._stage = "materialize" + model = self._materialize_fake_model(model) + profile.configure_tp_cross_entropy_counts( + batch.valid_token_count, + batch.target_tokens_per_rank, + ) + base = self._prepare_base(setup, model) + self._stage = "fake_step" + snapshot = self._execute_step(base, batch, profile, value_dependencies) + + if snapshot_path is not None: + self._stage = "snapshot_generation" + write_memory_snapshot(snapshot, snapshot_path) + logger.info("HyperModels Dry-run memory snapshot: %s", snapshot_path) + return snapshot + except Exception as error: + plugin_hint = unsupported_operator_hint(error) + if plugin_hint is not None: + raise RuntimeError( + f"HyperModels Dry-run failed during {self._stage}: {plugin_hint}" + ) from error + raise RuntimeError( + f"HyperModels Dry-run failed during {self._stage}: {error}" + ) from error + finally: + if initialized_here: + destroy_distributed_runtime() + + def run(self) -> dict[str, Any]: + """Execute one fake step, write the rank-local pickle, and return its snapshot.""" + dry_run = self._validate_config() + runtime = self._get_runtime() + memory_profiler_module.memory_profiler.reset( + None, global_rank=runtime.rank, tp_rank=0, dp_rank=0, + ) + # Register shape-only Meta kernels for Ascend vendor operators whose + # backward passes ship without them, so the fused kernels stay on the + # execution path instead of falling back to reference implementations. + register_builtin_dryrun_ops() + snapshot_path = self._snapshot_path(dry_run, runtime) + target_device = self._resolve_target_device() + self._simulation_device = self._select_simulation_device(target_device) + # The real Trainer step loop runs over FakeTensors: silence every + # device-side effect (set_device/synchronize/empty_cache, allocator + # history, swap streams and events) so simulation never binds or + # initializes an accelerator. Components that stage collectives + # consult the lied accelerator so their buffers land where real + # runs put them. + simulate_device_runtime( + True, + accelerator=None if self._simulation_device == "cpu" else self._simulation_device, + ) + # The FakeTensor dispatch cache pins op-output storages inside its + # keys. Harmless for shape checking, but this tracker equates storage + # lifetime with PyObject lifetime, so a cached entry would keep a + # released activation "live" (observed with activation swap: resized + # storages stayed pinned by _DispatchCacheKey). Simulation disables + # the cache; correctness of lifetimes outranks dispatch speed here. + torch._dynamo.config.fake_tensor_cache_enabled = False # pylint: disable=protected-access + fake_mode = FakeTensorMode(allow_non_fake_inputs=True) + batch = self._probe_training_batch(runtime, fake_mode) + return self._run_simulation(runtime, dry_run, batch, fake_mode, snapshot_path) + + +__all__ = ["HyperModelsDryRunRunner"] diff --git a/hyper_parallel/tools/dryrun/runtime.py b/hyper_parallel/tools/dryrun/runtime.py new file mode 100644 index 0000000000..184e3c9128 --- /dev/null +++ b/hyper_parallel/tools/dryrun/runtime.py @@ -0,0 +1,1575 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Configurable FakeTensor LLM dry-run support for the HyperModels Trainer.""" +# pylint: disable=protected-access +import copy +import os +import sys +from collections.abc import Mapping +from contextlib import AbstractContextManager, ExitStack, nullcontext +from dataclasses import dataclass, fields, is_dataclass, replace +from enum import Enum +from fnmatch import fnmatchcase +from types import MethodType, SimpleNamespace +from typing import Any, Callable, ContextManager, Dict, Optional + +from hyper_parallel.data.constants import IGNORE_INDEX + + +@dataclass(frozen=True) +class DryRunRuntime: + """Distributed identity supplied by ``torchrun`` for one dry-run worker.""" + + rank: int + world_size: int + local_rank: int + + @classmethod + def from_torchrun_env(cls) -> "DryRunRuntime": + """Build and validate the current worker identity. + + Returns: + The validated dry-run runtime identity. + + Raises: + RuntimeError: If the process was not launched by ``torchrun``. + ValueError: If a launcher variable is outside its valid range. + """ + required_names = ("RANK", "WORLD_SIZE", "LOCAL_RANK") + missing_names = [name for name in required_names if name not in os.environ] + if missing_names: + raise RuntimeError( + "Multi-rank HyperDryRun must be launched with torchrun; " + f"missing environment variables: {missing_names}" + ) + rank = int(os.environ["RANK"]) + world_size = int(os.environ["WORLD_SIZE"]) + local_rank = int(os.environ["LOCAL_RANK"]) + if world_size < 1: + raise ValueError(f"WORLD_SIZE must be >= 1, got {world_size}") + if rank < 0 or rank >= world_size: + raise ValueError( + f"RANK must be in [0, {world_size}), got {rank}" + ) + if local_rank < 0: + raise ValueError(f"LOCAL_RANK must be >= 0, got {local_rank}") + return cls(rank=rank, world_size=world_size, local_rank=local_rank) + + +class DryRunBatchMocker: + """Erase one CPU training batch while preserving its complete structure.""" + + def __init__(self, fake_mode: Any, device: Any) -> None: + """Store the FakeTensor mode and logical simulation device.""" + self.fake_mode = fake_mode + self.device = device + + def mock(self, value: Any) -> Any: + """Recursively replace tensor leaves with value-free FakeTensors.""" + import torch # pylint: disable=C0415,forbidden-backend-import + + if isinstance(value, torch.Tensor): + with self.fake_mode: + return torch.empty_like( + value, + device=self.device, + memory_format=torch.preserve_format, + ) + if isinstance(value, Mapping): + return self._mock_mapping(value) + if isinstance(value, (list, tuple)): + return self._mock_sequence(value) + if is_dataclass(value) and not isinstance(value, type): + return self._mock_dataclass(value) + if hasattr(value, "__dict__") and self._contains_tensor(vars(value)): + raise ValueError( + "Dry-run cannot mock tensor-bearing batch carrier " + f"{type(value).__module__}.{type(value).__qualname__}" + ) + return value + + def _mock_mapping(self, value: Mapping[Any, Any]) -> Mapping[Any, Any]: + """Mock mapping values while preserving the mapping carrier type.""" + mocked_items = [(key, self.mock(item)) for key, item in value.items()] + if type(value) is dict: # pylint: disable=unidiomatic-typecheck + return dict(mocked_items) + try: + mocked_mapping = copy.copy(value) + mocked_mapping.clear() + mocked_mapping.update(mocked_items) + return mocked_mapping + except (AttributeError, TypeError): + try: + return type(value)(mocked_items) + except TypeError as exc: + raise ValueError( + "Dry-run cannot preserve batch mapping carrier " + f"{type(value).__module__}.{type(value).__qualname__}" + ) from exc + + def _mock_sequence(self, value: Any) -> Any: + """Mock list and tuple values while preserving named tuples.""" + mocked = tuple(self.mock(item) for item in value) + if isinstance(value, list): + return list(mocked) + if hasattr(value, "_fields"): + return type(value)(*mocked) + return mocked + + def _mock_dataclass(self, value: Any) -> Any: + """Mock initialized and deferred dataclass fields.""" + updates = { + field.name: self.mock(getattr(value, field.name)) + for field in fields(value) + if field.init + } + mocked_dataclass = replace(value, **updates) + for field in fields(value): + if not field.init: + object.__setattr__( + mocked_dataclass, + field.name, + self.mock(getattr(value, field.name)), + ) + return mocked_dataclass + + def _contains_tensor(self, value: Any) -> bool: + """Return whether an unsupported object recursively owns a tensor.""" + import torch # pylint: disable=C0415,forbidden-backend-import + + if isinstance(value, torch.Tensor): + return True + if isinstance(value, Mapping): + return any(self._contains_tensor(item) for item in value.values()) + if isinstance(value, (list, tuple)): + return any(self._contains_tensor(item) for item in value) + return False + + +def derive_tp_target_counts( + loss_inputs: Mapping[str, Any], + vocab_size: int, + tp_size: int, +) -> tuple[int, tuple[int, ...]]: + """Derive valid-label ownership using the real loss-parallel vocab split.""" + import torch # pylint: disable=C0415,forbidden-backend-import + + labels = loss_inputs.get("shift_labels") + if labels is None: + labels = loss_inputs.get("labels") + if labels is None: + raise ValueError("Dry-run loss_inputs must contain labels") + labels = labels[..., 1:] + if not isinstance(labels, torch.Tensor): + raise ValueError("Dry-run labels must be a torch.Tensor") + valid_mask = labels.ne(IGNORE_INDEX) + loss_mask = loss_inputs.get("loss_mask") + if isinstance(loss_mask, torch.Tensor): + if loss_mask.shape != labels.shape: + loss_mask = loss_mask[..., -labels.shape[-1]:] + valid_mask = valid_mask & loss_mask.reshape_as(labels).bool() + valid_labels = labels[valid_mask] + chunk_size = (vocab_size + tp_size - 1) // tp_size + owned = tuple( + int(((valid_labels >= rank * chunk_size) & + (valid_labels < min((rank + 1) * chunk_size, vocab_size))).sum().item()) + for rank in range(tp_size) + ) + valid_count = int(valid_labels.numel()) + if sum(owned) != valid_count: + raise ValueError("Dry-run labels contain token IDs outside the configured vocabulary") + return valid_count, owned + + +@dataclass(frozen=True) +class _DryRunMoERoutingPlan: + """Aggregate routed-token load for one MoE dry-run invocation.""" + + source_expert_loads: tuple[tuple[int, ...], ...] + local_expert_loads: tuple[int, ...] + received_expert_loads: tuple[int, ...] + input_split_sizes: tuple[int, ...] + output_split_sizes: tuple[int, ...] + outgoing_tokens: int + + +@dataclass(frozen=True) +class _DryRunMoEDispatchContext: + """Shape-only state retained between dry-run dispatch and combine.""" + + rank_major_shape: tuple[int, ...] + permuted_indices: Any + input_split_sizes: tuple[int, ...] + output_split_sizes: tuple[int, ...] + + +@dataclass(frozen=True) +class _ValueDependencyRule: + """One validated but not yet expanded value-dependency rule.""" + + index: int + match: str + handler: str + path: str + inputs: Dict[str, Any] + optional: bool + + +class ValueDependencyStage(Enum): + """Lifecycle stages at which a value-dependency handler may install state.""" + + FAKE_STEP = "fake_step" + + +class ValueDependencyHandler: + """Base interface for configured dry-run value-dependency handlers.""" + + name = "" + paths: tuple[str, ...] = () + default_path = "" + + def discover(self, model: Any, context: Any) -> list[str]: + """Return logical targets supported by this handler.""" + del model, context + return [] + + def recognizes(self, target: str, module: Any) -> bool: + """Return whether this handler can explain an unconfigured target.""" + del target, module + return False + + def validate(self, target: str, path: str, inputs: Dict[str, Any], context: Any) -> None: + """Validate one expanded rule.""" + del target, inputs, context + if self.paths and path not in self.paths: + raise ValueError( + f"value dependency handler {self.name!r} path must be one of " + f"{self.paths}, got {path!r}" + ) + + def install( + self, stage: ValueDependencyStage, target: str, path: str, + inputs: Dict[str, Any], context: Any, + ) -> ContextManager[Any]: + """Return scoped runtime state for one expanded rule.""" + del stage, target, path, inputs, context + return nullcontext() + + def resolved_metadata(self) -> Dict[str, Any]: + """Return handler-specific resolved report metadata.""" + return {} + + +_VALUE_DEPENDENCY_HANDLER_FACTORIES: Dict[str, Callable[[], ValueDependencyHandler]] = {} + + +def register_value_dependency_handler( + name: str, factory: Callable[[], ValueDependencyHandler], +) -> None: + """Register one dry-run value-dependency handler factory. + + Args: + name: Unique handler name used by YAML rules. + factory: Zero-argument factory returning a fresh handler instance. + + Raises: + ValueError: If the name is empty, already registered, or inconsistent + with the constructed handler. + """ + if not isinstance(name, str) or not name.strip(): + raise ValueError("value dependency handler name must be a non-empty string") + if name in _VALUE_DEPENDENCY_HANDLER_FACTORIES: + raise ValueError(f"value dependency handler {name!r} is already registered") + handler = factory() + if not isinstance(handler, ValueDependencyHandler) or handler.name != name: + raise ValueError( + f"value dependency handler factory for {name!r} must return " + f"ValueDependencyHandler(name={name!r})" + ) + _VALUE_DEPENDENCY_HANDLER_FACTORIES[name] = factory + + +def _is_supported_moe_module(module: Any) -> bool: + """Return whether a module exposes a supported routed-expert layout.""" + experts = getattr(module, "experts", None) + if experts is None or ( + getattr(module, "gate", None) is None + and getattr(module, "router", None) is None + ): + return False + layouts = ( + ("w1", "w2", "w3"), + ("gate_up_proj", "down_proj"), + ("gate_proj", "up_proj", "down_proj"), + ) + return any(all(hasattr(experts, name) for name in layout) for layout in layouts) + + +class MoERoutingValueDependencyHandler(ValueDependencyHandler): + """Validate and describe configured routed-expert replay.""" + + name = "moe_routing" + paths = ("balanced", "hotspot", "explicit") + default_path = "balanced" + + def discover(self, model: Any, context: Any) -> list[str]: + """Return supported routed-expert module FQNs.""" + del context + return [name for name, module in model.named_modules() if name and _is_supported_moe_module(module)] + + def recognizes(self, target: str, module: Any) -> bool: + """Recognize supported routed-expert modules.""" + del target + return _is_supported_moe_module(module) + + def validate(self, target: str, path: str, inputs: Dict[str, Any], context: Any) -> None: + """Validate path-specific MoE rule inputs.""" + super().validate(target, path, inputs, context) + allowed = { + "balanced": set(), + "hotspot": {"hotspot_expert"}, + "explicit": {"source_expert_loads"}, + }[path] + unknown = sorted(set(inputs) - allowed) + if unknown: + raise ValueError(f"MoE rule for {target!r} contains unsupported inputs: {unknown}") + if path == "hotspot" and ( + not isinstance(inputs.get("hotspot_expert"), int) + or isinstance(inputs.get("hotspot_expert"), bool) + or inputs["hotspot_expert"] < 0 + ): + raise ValueError(f"MoE hotspot rule for {target!r} requires non-negative hotspot_expert") + if path == "explicit" and not isinstance(inputs.get("source_expert_loads"), list): + raise ValueError(f"MoE explicit rule for {target!r} requires source_expert_loads") + + def install( + self, stage: ValueDependencyStage, target: str, path: str, + inputs: Dict[str, Any], context: Any, + ) -> ContextManager[Any]: + """Install routed-expert replay once for the configured fake step.""" + del target, path, inputs + if stage is ValueDependencyStage.FAKE_STEP: + return _MoERoutingValueDependencyRuntime(context.profile, context.base) + return nullcontext() + + +_VALUE_DEPENDENCY_HANDLER_FACTORIES[MoERoutingValueDependencyHandler.name] = MoERoutingValueDependencyHandler + + +class _DryRunValueProfile: + """Parse, expand, and report unified value-dependency rules.""" + + def __init__(self, config: Any) -> None: + """Parse optional internal rules, or use handler defaults.""" + self._config = config + self._rules = self._parse_rules(getattr(config, "value_dependencies", {"rules": []})) + self._handlers = { + name: factory() + for name, factory in _VALUE_DEPENDENCY_HANDLER_FACTORIES.items() + } + self._resolved: Dict[str, Dict[str, _ValueDependencyRule]] = {} + self._resolved_moe_plans: Dict[str, Dict[str, Any]] = {} + self._tp_cross_entropy_counts: Optional[tuple[int, tuple[int, ...]]] = None + + @staticmethod + def _parse_rules(value_dependencies: Any) -> tuple[_ValueDependencyRule, ...]: + """Validate the common rule envelope before model discovery.""" + if not isinstance(value_dependencies, dict): + raise ValueError("dry_run.value_dependencies must be a mapping") + unknown = sorted(set(value_dependencies) - {"rules"}) + if unknown: + raise ValueError(f"dry_run.value_dependencies contains unknown fields: {unknown}") + raw_rules = value_dependencies.get("rules", []) + if not isinstance(raw_rules, list): + raise ValueError("dry_run.value_dependencies.rules must be a list") + rules = [] + for index, raw_rule in enumerate(raw_rules): + location = f"dry_run.value_dependencies.rules[{index}]" + if not isinstance(raw_rule, dict): + raise ValueError(f"{location} must be a mapping") + unknown_rule_fields = sorted( + set(raw_rule) - {"match", "handler", "path", "inputs", "optional"} + ) + if unknown_rule_fields: + raise ValueError(f"{location} contains unknown fields: {unknown_rule_fields}") + match = raw_rule.get("match") + handler = raw_rule.get("handler") + path = raw_rule.get("path") + inputs = raw_rule.get("inputs", {}) + optional = raw_rule.get("optional", False) + if not isinstance(match, str) or not match.strip(): + raise ValueError(f"{location}.match must be a non-empty string") + if not isinstance(handler, str) or not handler.strip(): + raise ValueError(f"{location}.handler must be a non-empty string") + if not isinstance(path, str) or not path.strip(): + raise ValueError(f"{location}.path must be a non-empty string") + if not isinstance(inputs, dict): + raise ValueError(f"{location}.inputs must be a mapping") + if not isinstance(optional, bool): + raise ValueError(f"{location}.optional must be a boolean") + rules.append(_ValueDependencyRule(index, match, handler, path, inputs, optional)) + return tuple(rules) + + def bind_model(self, model: Any, context: Any = None) -> None: + """Expand all configured rules against one model and logical scopes.""" + candidates: Dict[tuple[str, str], list[tuple[int, _ValueDependencyRule]]] = {} + rules = self._rules + if not rules: + default_rules = [] + for handler in self._handlers.values(): + if not handler.default_path: + continue + for target in handler.discover(model, context): + default_rules.append( + _ValueDependencyRule( + index=len(default_rules), + match=target, + handler=handler.name, + path=handler.default_path, + inputs={}, + optional=True, + ) + ) + rules = tuple(default_rules) + for rule in rules: + handler = self._handlers.get(rule.handler) + if handler is None: + raise ValueError( + f"value dependency rule {rule.index} uses unknown handler {rule.handler!r}; " + f"registered handlers: {sorted(self._handlers)}" + ) + supported_targets = handler.discover(model, context) + # Activation wrappers can expose a name through more than one tree + # position; keep the first occurrence so one target resolves once. + matches = list(dict.fromkeys( + target for target in supported_targets if fnmatchcase(target, rule.match) + )) + if not matches and not rule.optional: + raise ValueError( + f"value dependency rule {rule.index} match {rule.match!r} did not match " + f"any target supported by handler {rule.handler!r}" + ) + exact = int(rule.match in matches) + for target in matches: + candidates.setdefault((rule.handler, target), []).append((exact, rule)) + + resolved: Dict[str, Dict[str, _ValueDependencyRule]] = {} + for (handler_name, target), target_candidates in candidates.items(): + best_priority = max(priority for priority, _ in target_candidates) + best_rules = [rule for priority, rule in target_candidates if priority == best_priority] + if len(best_rules) > 1: + indexes = [rule.index for rule in best_rules] + raise ValueError( + f"value dependency rules {indexes} conflict for target {target!r} " + f"and handler {handler_name!r}" + ) + rule = best_rules[0] + self._handlers[handler_name].validate(target, rule.path, rule.inputs, context) + resolved.setdefault(handler_name, {})[target] = rule + self._resolved = resolved + + def handler(self, name: str) -> ValueDependencyHandler: + """Return one fresh profile-owned handler instance.""" + return self._handlers[name] + + @property + def moe_layers(self) -> dict[str, Any]: + """Return per-layer routing overrides.""" + return { + target: {"routing_policy": rule.path, **rule.inputs} + for target, rule in self._resolved.get("moe_routing", {}).items() + } + + @property + def moe_enabled(self) -> bool: + """Return whether the MoE adapter is enabled.""" + return bool(self._resolved.get("moe_routing")) + + def metadata(self) -> Dict[str, Any]: + """Return a JSON-safe profile summary for the memory report.""" + return { + "rules": [ + { + "index": rule.index, + "match": rule.match, + "handler": handler_name, + "path": rule.path, + "inputs": rule.inputs, + "resolved_target": target, + } + for handler_name, targets in self._resolved.items() + for target, rule in targets.items() + ], + "handlers": { + name: handler.resolved_metadata() + for name, handler in self._handlers.items() + if self._resolved.get(name) + }, + "moe_routing": {"resolved_layers": self._resolved_moe_plans}, + } + + def _moe_routing_matrix( + self, + layer: Dict[str, Any], + ep_size: int, + num_experts: int, + expected: int, + module_name: str, + ) -> tuple[tuple[int, ...], ...]: + """Build the configured or synthetic source-to-expert load matrix.""" + policy = layer["routing_policy"] + if policy == "explicit": + return self._explicit_moe_matrix( + layer.get("source_expert_loads"), + ep_size, + num_experts, + expected, + module_name, + ) + + hotspot = int(layer.get("hotspot_expert", 0)) + if policy == "hotspot" and not 0 <= hotspot < num_experts: + raise ValueError(f"MoE hotspot_expert for {module_name!r} must be in [0, {num_experts})") + if policy == "hotspot": + row = tuple(expected if index == hotspot else 0 for index in range(num_experts)) + else: + base, remainder = divmod(expected, num_experts) + row = tuple(base + int(index < remainder) for index in range(num_experts)) + return tuple(row for _ in range(ep_size)) + + @staticmethod + def _local_expert_start( + module_name: str, + ep_size: int, + ep_rank: int, + num_experts: int, + local_expert_count: int, + ) -> int: + """Validate the expert layout and return this rank's first expert.""" + if local_expert_count < 1: + raise ValueError(f"MoE {module_name!r} has no local experts") + if local_expert_count == num_experts: + return 0 + if num_experts % ep_size or local_expert_count != num_experts // ep_size: + raise ValueError(f"MoE {module_name!r} local expert layout is incompatible with EP={ep_size}") + return ep_rank * local_expert_count + + @staticmethod + def _expert_ranges( + ep_size: int, + num_experts: int, + local_expert_count: int, + ) -> tuple[tuple[int, int], ...]: + """Return the global expert range owned by each destination rank.""" + if local_expert_count == num_experts: + return ((0, num_experts),) + return tuple( + (rank * local_expert_count, (rank + 1) * local_expert_count) + for rank in range(ep_size) + ) + + def moe_routing_plan( + self, module_name: str, module: Any, ep_size: int, ep_rank: int, + local_tokens: int, local_expert_count: int, + ) -> _DryRunMoERoutingPlan: + """Build aggregate EP source-to-expert load for one MoE layer.""" + num_experts = int(module.num_experts) + expected = local_tokens * int(module.top_k) + layer = self.moe_layers.get(module_name) + if layer is None: + raise ValueError(f"MoE module {module_name!r} has no value-dependency rule") + matrix = self._moe_routing_matrix(layer, ep_size, num_experts, expected, module_name) + start = self._local_expert_start( + module_name, + ep_size, + ep_rank, + num_experts, + local_expert_count, + ) + local_loads = tuple( + sum(row[index] for row in matrix) + for index in range(start, start + local_expert_count) + ) + received_loads = tuple( + matrix[source_rank][expert_index] + for source_rank in range(ep_size) + for expert_index in range(start, start + local_expert_count) + ) + expert_ranges = self._expert_ranges(ep_size, num_experts, local_expert_count) + input_splits = tuple( + sum(matrix[ep_rank][start_index:end_index]) + for start_index, end_index in expert_ranges + ) + output_splits = tuple( + sum(matrix[source_rank][start:start + local_expert_count]) + for source_rank in range(ep_size) + ) + plan = _DryRunMoERoutingPlan( + matrix, + local_loads, + received_loads, + input_splits, + output_splits, + sum(matrix[ep_rank]), + ) + self._resolved_moe_plans[module_name] = { + "source_expert_loads": [list(row) for row in plan.source_expert_loads], + "local_expert_loads": list(plan.local_expert_loads), + "received_expert_loads": list(plan.received_expert_loads), + "input_split_sizes": list(plan.input_split_sizes), + "output_split_sizes": list(plan.output_split_sizes), + "outgoing_tokens": plan.outgoing_tokens, + } + return plan + + @staticmethod + def _explicit_moe_matrix( + source_loads: Any, ep_size: int, num_experts: int, + expected: int, module_name: str, + ) -> tuple[tuple[int, ...], ...]: + """Validate the configured aggregate EP traffic matrix.""" + if not isinstance(source_loads, list) or len(source_loads) != ep_size: + raise ValueError(f"MoE explicit source_expert_loads for {module_name!r} must have {ep_size} rows") + matrix = [] + for source_rank, row in enumerate(source_loads): + if not isinstance(row, list) or len(row) != num_experts: + raise ValueError( + f"MoE explicit source_expert_loads row {source_rank} for {module_name!r} " + f"must have {num_experts} entries" + ) + if any( + not isinstance(count, int) or isinstance(count, bool) or count < 0 + for count in row + ): + raise ValueError("MoE explicit source_expert_loads entries must be non-negative integers") + if sum(row) != expected: + raise ValueError( + f"MoE explicit source_expert_loads row {source_rank} for {module_name!r} must sum to " + f"local_tokens * top_k ({expected}), got {sum(row)}" + ) + matrix.append(tuple(row)) + return tuple(matrix) + + def tp_cross_entropy_target_count(self, target_count: int, tp_size: int, tp_rank: int) -> tuple[int, int]: + """Return CPU-derived valid and local-owned target counts for TP CE.""" + counts = self._tp_cross_entropy_counts + if counts is None: + raise ValueError("TP cross-entropy counts were not derived from the training batch") + valid, owned = counts + if len(owned) != tp_size or not 0 <= tp_rank < tp_size: + raise ValueError( + f"CPU-derived TP target counts contain {len(owned)} ranks, expected {tp_size}" + ) + if valid > target_count: + raise ValueError( + f"CPU-derived valid token count {valid} exceeds FakeTensor target size {target_count}" + ) + return valid, owned[tp_rank] + + def configure_tp_cross_entropy_counts( + self, + valid_token_count: int, + target_tokens_per_rank: tuple[int, ...], + ) -> None: + """Install token ownership computed from the real CPU training batch.""" + if valid_token_count < 0 or sum(target_tokens_per_rank) != valid_token_count: + raise ValueError("CPU-derived TP target counts must be non-negative and sum to the valid count") + self._tp_cross_entropy_counts = (valid_token_count, target_tokens_per_rank) + + +class ValueDependencyManager(AbstractContextManager): + """Install configured semantic replay for one fake training step.""" + + def __init__(self, profile: _DryRunValueProfile, model: Any, runtime: DryRunRuntime) -> None: + """Store profile state without changing model or distributed runtime.""" + del model, runtime + self._profile = profile + self._base = None + self._stack = ExitStack() + + def fake_step_context(self, base: Any) -> "ValueDependencyManager": + """Bind the prepared Trainer state used by semantic handlers.""" + self._base = base + return self + + def __enter__(self) -> "ValueDependencyManager": + """Install configured MoE and TP cross-entropy replay.""" + if self._base is None: + raise RuntimeError("ValueDependencyManager.fake_step_context must be configured before entry") + for handler_name, rules in self._profile._resolved.items(): + if not rules: + continue + handler = self._profile.handler(handler_name) + target, rule = next(iter(rules.items())) + self._stack.enter_context(handler.install( + ValueDependencyStage.FAKE_STEP, + target, + rule.path, + rule.inputs, + SimpleNamespace(profile=self._profile, base=self._base), + )) + mesh = getattr(self._base, "mesh", None) + if bool(getattr(mesh, "loss_parallel", False)): + self._stack.enter_context(_TPCrossEntropyValueDependencyRuntime(self._profile)) + return self + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> bool: + """Restore every scoped semantic override.""" + del exc_type, exc_value, traceback + self._stack.close() + return False + + +class _TPCrossEntropyValueDependencyRuntime(AbstractContextManager): + """Scoped TP cross-entropy replay owned by its semantic handler.""" + + def __init__(self, profile: _DryRunValueProfile) -> None: + """Store the resolved ownership profile.""" + self._profile = profile + self._loss_parallel_ops = None + self._original_cross_entropy_function = None + + def __enter__(self) -> "_TPCrossEntropyValueDependencyRuntime": + """Patch the complete CE autograd Function for this dry-run scope.""" + from hyper_parallel.components.losses import ( # pylint: disable=C0415 + _vocab_parallel_cross_entropy as loss_parallel_ops, + ) + import torch # pylint: disable=C0415,forbidden-backend-import + + profile = self._profile + + class DryRunDistributedCrossEntropyFunction(torch.autograd.Function): + """CE Function that preserves tensor paths without inspecting labels.""" + + @staticmethod + def forward(ctx: Any, input_local: Any, target: Any, weight: Any, ignore_index: int, + reduction: str, vocab_size: int, mesh: Any, mesh_dim: int) -> Any: + """Run distributed log-softmax and allocate configured local NLL state.""" + del weight, ignore_index, vocab_size + tp_size = int(mesh.size(mesh_dim)) + tp_rank = int(mesh.get_local_rank(mesh_dim)) + valid_count, local_count = profile.tp_cross_entropy_target_count( + int(target.numel()), tp_size, tp_rank, + ) + log_probs = loss_parallel_ops.distributed_log_softmax( + input_local, dim=-1, mesh=mesh, mesh_dim=mesh_dim, + ) + selected = torch.empty((local_count,), dtype=log_probs.dtype, device=log_probs.device) + anchor = log_probs.sum() * 0.0 + selected.sum() * 0.0 + ctx.save_for_backward(log_probs) + ctx.reduction = reduction + ctx.valid_count = valid_count + ctx.local_count = local_count + if reduction == "none": + return torch.empty_like(target, dtype=log_probs.dtype) + anchor + total = loss_parallel_ops._differentiable_all_reduce( + anchor, op="sum", group=mesh.get_group(mesh_dim), + ) + return total / max(valid_count, 1) if reduction == "mean" else total + + @staticmethod + def backward(ctx: Any, grad_output: Any) -> tuple[Any, ...]: + """Retain local softmax-gradient allocations without label indexing.""" + (log_probs,) = ctx.saved_tensors + if ctx.reduction == "none": + grad_scale = grad_output.reshape(-1, 1) + elif ctx.reduction == "mean": + grad_scale = grad_output / max(ctx.valid_count, 1) + else: + grad_scale = grad_output + selected_grad = torch.empty( + (ctx.local_count,), dtype=log_probs.dtype, device=log_probs.device, + ) + grad_input = log_probs.exp() * grad_scale + selected_grad.sum() * 0.0 + return grad_input, None, None, None, None, None, None, None + + self._loss_parallel_ops = loss_parallel_ops + self._original_cross_entropy_function = loss_parallel_ops.DistributedCrossEntropyFunction + loss_parallel_ops.DistributedCrossEntropyFunction = DryRunDistributedCrossEntropyFunction + return self + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> bool: + """Restore the real TP cross-entropy autograd Function.""" + del exc_type, exc_value, traceback + if self._loss_parallel_ops is not None: + self._loss_parallel_ops.DistributedCrossEntropyFunction = self._original_cross_entropy_function + self._loss_parallel_ops = None + self._original_cross_entropy_function = None + return False + + +class _MoERoutingValueDependencyRuntimeBase: + """Scoped routed-expert replay used by the MoE semantic handler.""" + + def __init__(self, profile: _DryRunValueProfile, base: Any) -> None: + """Create a scope for one prepared trainer.""" + self._profile = profile + self._base = base + self._patched_moe_modules = [] + self._moe_module_names: Dict[int, str] = {} + self._ep_compute_modules: list[Any] = [] + self._original_ep_compute = None + + def __enter__(self) -> "_MoERoutingValueDependencyRuntime": + """Install configured MoE replay.""" + try: + if self._profile.moe_enabled: + self._patch_moe() + except Exception: + self._restore_moe() + raise + return self + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> bool: + """Restore every patched symbol even if the simulated step failed.""" + del exc_type, exc_value, traceback + self._restore_moe() + return False + + def _patch_moe(self) -> None: + """Install model-specific MoE replay in a concrete runtime.""" + raise NotImplementedError + + def _restore_moe(self) -> None: + """Restore model-specific MoE symbols in a concrete runtime.""" + raise NotImplementedError + + +class _MoERoutingValueDependencyRuntime(_MoERoutingValueDependencyRuntimeBase): + """Complete shape and memory replay for configured routed-expert modules.""" + + def _patch_moe(self) -> None: + """Patch supported MoE execution modules without changing their classes.""" + module_names = set() + configured_names = set(self._profile.moe_layers) + for module_name, module in self._base.model.named_modules(): + if module_name in configured_names and self._is_supported_moe_module(module): + module_names.add(module_name) + self._moe_module_names[id(module)] = module_name + if self._base.mesh.ep_size > 1: + continue + had_forward = "forward" in module.__dict__ + original_forward = module.__dict__.get("forward") + + def dry_run_forward( + current_module: Any, + x: Any, + *args: Any, + name: str = module_name, + **kwargs: Any, + ) -> Any: + """Execute the configured value-independent MoE path.""" + del args, kwargs + return self._run_moe_forward(name, current_module, x) + + module.forward = MethodType(dry_run_forward, module) + self._patched_moe_modules.append((module, had_forward, original_forward)) + if self._base.mesh.ep_size > 1 and module_names: + from hyper_parallel.distributed.expert_parallel import experts as ep_experts # pylint: disable=C0415 + + self._original_ep_compute = ep_experts.ep_routed_forward + + def dry_run_ep_compute( + module: Any, + hidden_states: Any, + *, + router_fn: Any, + ep_group: Any, + ) -> Any: + """Execute configured EP shapes inside the existing local region.""" + return self._run_hf_native_ep_compute( + module, + hidden_states, + router_fn=router_fn, + ep_group=ep_group, + ) + + # ``ep_routed_forward`` is re-exported through several modules + # (``recipes``, model adapters) that each bind the symbol into + # their own namespace at import time. Patch every live binding so + # the configured replay runs regardless of the entry point. + for loaded_module in list(sys.modules.values()): + if loaded_module is None: + continue + # Inspect the module ``__dict__`` directly: ``getattr`` would + # trigger PEP 562 lazy-import hooks (e.g. ``transformers``) that + # pull optional imaging dependencies this path never uses. + if ( + loaded_module.__dict__.get("ep_routed_forward") + is self._original_ep_compute + ): + loaded_module.ep_routed_forward = dry_run_ep_compute + self._ep_compute_modules.append(loaded_module) + unknown_names = configured_names - module_names + if unknown_names: + self._restore_moe() + raise ValueError( + "moe_routing rules contain unsupported module FQNs: " + f"{sorted(unknown_names)}; supported MoE module FQNs discovered " + f"in the parallelized model: {sorted(module_names)}" + ) + + @staticmethod + def _is_supported_moe_module(module: Any) -> bool: + """Return whether a module exposes one of the supported expert layouts.""" + return _is_supported_moe_module(module) + + @staticmethod + def _moe_dimensions(module: Any) -> tuple[int, int]: + """Return global expert count and top-k for a supported MoE module.""" + experts = module.experts + config = getattr(module, "config", None) + num_experts = ( + getattr(experts, "num_experts", None) + or getattr(module, "num_experts", None) + or getattr(config, "num_experts", None) + or getattr(config, "n_routed_experts", None) + ) + top_k = ( + getattr(module, "top_k", None) + or getattr(getattr(module, "gate", None), "top_k", None) + or getattr(config, "num_experts_per_tok", None) + or getattr(config, "top_k", None) + ) + if num_experts is None or top_k is None: + raise ValueError( + f"Cannot resolve expert count/top-k for {type(module).__name__}" + ) + return int(num_experts), int(top_k) + + @staticmethod + def _local_weight(weight: Any) -> Any: + """Return a local expert-weight shard while preserving FakeTensor state.""" + return weight.to_local() if hasattr(weight, "to_local") else weight + + def _expert_weight_layout(self, module: Any) -> tuple[str, tuple[Any, ...]]: + """Extract local expert weights from a supported MoE module. + + The framework ``MoE`` stores independent SwiGLU matrices ``w1/w2/w3``. + Qwen3.5 packs the gate and up matrices together in ``gate_up_proj``. + Both layouts retain the EP-sharded leading expert dimension. + """ + experts = module.experts + if all(hasattr(experts, name) for name in ("w1", "w2", "w3")): + return "three_projection", tuple( + self._local_weight(getattr(experts, name)) for name in ("w1", "w2", "w3") + ) + if all(hasattr(experts, name) for name in ("gate_up_proj", "down_proj")): + return "packed_gate_up", ( + self._local_weight(experts.gate_up_proj), + self._local_weight(experts.down_proj), + ) + if all( + hasattr(experts, name) + for name in ("gate_proj", "up_proj", "down_proj") + ): + return "three_projection", ( + self._local_weight(experts.gate_proj), + self._local_weight(experts.down_proj), + self._local_weight(experts.up_proj), + ) + raise ValueError( + f"MoE module {type(module).__name__!r} does not expose a supported expert-weight layout" + ) + + def _run_moe_forward(self, module_name: str, module: Any, x: Any) -> Any: + """Run a value-independent analogue of the real routed MoE path.""" + import torch # pylint: disable=C0415,forbidden-backend-import + + batch_size, sequence_length, hidden_size = x.shape + ep_size, ep_rank = self._ep_group_identity() + layout, weights = self._expert_weight_layout(module) + num_experts, top_k = self._moe_dimensions(module) + routing_module = SimpleNamespace(num_experts=num_experts, top_k=top_k) + plan = self._profile.moe_routing_plan( + module_name, routing_module, ep_size, ep_rank, + int(batch_size * sequence_length), int(weights[0].shape[0]), + ) + x_flat = x.view(-1, hidden_size) + shared_output = None + if getattr(module, "shared_expert", None) is not None: + shared_output = module.shared_expert(x_flat) + routed_input, top_weights, permutation, inverse_permutation, token_counts = self._dry_run_route( + module_name, module, x_flat, plan, num_experts, top_k, + ) + expert_input, dispatch_context = self._dry_run_dispatch( + routed_input, token_counts, plan, ep_size, int(weights[0].shape[0]), + ) + expert_output = self._run_grouped_experts( + layout, weights, expert_input, plan.local_expert_loads, + ) + combined = self._dry_run_combine(expert_output, dispatch_context) + routed_output = self._dry_run_weight_and_unpermute( + module, x_flat, combined, top_weights, permutation, inverse_permutation, top_k, + ) + output = routed_output + if shared_output is not None: + shared_gate = torch.sigmoid(module.shared_expert_gate(x_flat)) + output = output + shared_gate * shared_output + if hasattr(module, "last_aux_loss"): + module.last_aux_loss = None + return output.view(batch_size, sequence_length, hidden_size) + + def _run_hf_native_ep_compute( + self, + module: Any, + hidden_states: Any, + *, + router_fn: Any, + ep_group: Any, + ) -> Any: + """Replay the new Trainer's EP lifecycle with configured split sizes.""" + import torch # pylint: disable=C0415,forbidden-backend-import + import torch.distributed as dist # pylint: disable=C0415,forbidden-backend-import + + module_name = self._moe_module_names.get(id(module)) + if module_name is None: + raise ValueError( + f"Dry-run EP compute received an unregistered {type(module).__name__}" + ) + ep_size = int(ep_group.size()) + ep_rank = int(dist.get_rank(group=ep_group)) + num_experts, top_k = self._moe_dimensions(module) + local_expert_count = int(module.experts.local_expert_count) + batch_size, sequence_length, hidden_size = hidden_states.shape + local_tokens = int(batch_size * sequence_length) + routing_module = SimpleNamespace(num_experts=num_experts, top_k=top_k) + plan = self._profile.moe_routing_plan( + module_name, + routing_module, + ep_size, + ep_rank, + local_tokens, + local_expert_count, + ) + + flattened_states = hidden_states.reshape(-1, hidden_size) + topk_indices, topk_weights = router_fn(module, hidden_states) + flattened_weights = topk_weights.reshape(-1).to(flattened_states.dtype) + source_indices = torch.arange( + local_tokens, + device=flattened_states.device, + ).repeat_interleave(top_k) + dispatch_order = topk_indices.reshape(-1).argsort() + dispatched_states = flattened_states[source_indices[dispatch_order]].contiguous() + received_states = self._differentiable_all_to_all( + dispatched_states, + plan.input_split_sizes, + plan.output_split_sizes, + ) + received_expert_indices = torch.empty( + (sum(plan.output_split_sizes),), + dtype=torch.int64, + device=hidden_states.device, + ) + + experts = module.experts + had_forward = "forward" in experts.__dict__ + original_forward = experts.__dict__.get("forward") + + def grouped_forward( + current_experts: Any, + dispatched: Any, + local_indices: Any, + ) -> Any: + """Execute grouped experts with the simulated token distribution. + + Args: + current_experts: Local expert module collection. + dispatched: Tokens received by the local experts. + local_indices: Runtime expert indices, unused by the simulation. + + Returns: + The grouped expert outputs. + """ + del local_indices + layout, weights = self._expert_weight_layout( + SimpleNamespace(experts=current_experts) + ) + if not getattr(current_experts, "_ep_use_grouped_gemm", False): + return self._run_eager_experts( + current_experts, + layout, + weights, + dispatched, + plan.local_expert_loads, + ) + return self._run_grouped_experts( + layout, + weights, + dispatched, + plan.local_expert_loads, + ) + + experts.forward = MethodType(grouped_forward, experts) + try: + local_outputs = experts(received_states, received_expert_indices) + finally: + if had_forward: + experts.forward = original_forward + else: + del experts.forward + + combined = self._differentiable_all_to_all( + local_outputs.contiguous(), + plan.output_split_sizes, + plan.input_split_sizes, + ) + from hyper_parallel.distributed.expert_parallel.experts import ( # pylint: disable=C0415 + _aggregate_ep_outputs, + ) + output = _aggregate_ep_outputs( + combined, + flattened_weights, + source_indices, + dispatch_order, + (batch_size, sequence_length, hidden_size), + ) + # ep_routed_forward owns routed experts only. Model adapters execute + # their shared-expert branch after this function returns. + return output + + @staticmethod + def _dry_run_route( + module_name: str, module: Any, x_flat: Any, + plan: _DryRunMoERoutingPlan, + num_experts: int, + top_k: int, + ) -> tuple[Any, Any, Any, Any, Any]: + """Replay value-independent router, sort, and histogram operators.""" + import torch # pylint: disable=C0415,forbidden-backend-import + from torch.nn import functional # pylint: disable=C0415,forbidden-backend-import + + gate = getattr(module, "gate", None) + gate_weight = getattr(gate, "weight", None) + if gate_weight is None: + raise ValueError( + f"MoE module {module_name!r} must expose gate.weight for configured dry-run routing" + ) + router_logits = x_flat @ gate_weight.transpose(0, 1) + router_probs = functional.softmax(router_logits, dim=-1, dtype=torch.float32) + top_weights, top_indices = torch.topk( + router_probs, top_k, dim=-1, + ) + top_weights = top_weights / top_weights.sum(dim=-1, keepdim=True) + top_weights = top_weights.to(x_flat.dtype) + module.router_logits = router_logits + + token_indices = torch.arange( + x_flat.shape[0], device=x_flat.device, + ).unsqueeze(1).expand(-1, top_k).reshape(-1) + expert_indices = top_indices.reshape(-1) + permutation = torch.argsort(expert_indices, stable=True) + inverse_permutation = torch.empty_like(permutation) + inverse_permutation[permutation] = torch.arange( + plan.outgoing_tokens, device=x_flat.device, + ) + routed_input = x_flat[token_indices[permutation]] + histc_input = expert_indices.float() if x_flat.device.type == "cpu" else expert_indices.int() + token_counts = torch.histc( + histc_input, + bins=num_experts, + min=0, + max=num_experts - 1, + ).to(torch.int64) + return routed_input, top_weights, permutation, inverse_permutation, token_counts + + def _dry_run_dispatch( + self, routed_input: Any, token_counts: Any, + plan: _DryRunMoERoutingPlan, + ep_size: int, local_expert_count: int, + ) -> tuple[Any, _DryRunMoEDispatchContext]: + """Replay counts exchange, token dispatch, and expert permutation.""" + counts_output = self._dry_run_exchange_counts( + token_counts, ep_size, + ) + dispatched = self._differentiable_all_to_all( + routed_input, + plan.input_split_sizes, + plan.output_split_sizes, + ) + rank_major_shape = tuple(dispatched.shape) + permutation = self._dry_run_rank_to_expert_indices( + counts_output, + sum(plan.output_split_sizes), + ep_size, + local_expert_count, + ) + expert_input = dispatched[permutation] + context = _DryRunMoEDispatchContext( + rank_major_shape=rank_major_shape, + permuted_indices=permutation, + input_split_sizes=plan.input_split_sizes, + output_split_sizes=plan.output_split_sizes, + ) + return expert_input, context + + def _dry_run_combine( + self, expert_output: Any, + context: _DryRunMoEDispatchContext, + ) -> Any: + """Replay expert unpermutation and reverse token all-to-all.""" + rank_major_output = expert_output.new_zeros(*context.rank_major_shape) + rank_major_output[context.permuted_indices] = expert_output + return self._differentiable_all_to_all( + rank_major_output, + context.output_split_sizes, + context.input_split_sizes, + ) + + @staticmethod + def _dry_run_weight_and_unpermute( + module: Any, x_flat: Any, combined: Any, top_weights: Any, + permutation: Any, inverse_permutation: Any, + top_k: int, + ) -> Any: + """Apply routing weights and restore the original token order.""" + import torch # pylint: disable=C0415,forbidden-backend-import + + sorted_weights = top_weights.reshape(-1)[permutation] + use_fp32_combine = ( + getattr(module, "_hp_moe_tp_enabled", False) + or getattr(module, "_hp_moe_ep_fp32_routing", False) + ) + if use_fp32_combine: + weighted = ( + combined.to(torch.float32) * sorted_weights.to(torch.float32).unsqueeze(-1) + ).to(combined.dtype) + unsorted = weighted[inverse_permutation] + return unsorted.view( + x_flat.shape[0], top_k, x_flat.shape[-1], + ).sum(dim=1, dtype=torch.float32).to(x_flat.dtype) + weighted = combined * sorted_weights.unsqueeze(-1) + unsorted = weighted[inverse_permutation] + return unsorted.view( + x_flat.shape[0], top_k, x_flat.shape[-1], + ).sum(dim=1).to(x_flat.dtype) + + def _dry_run_exchange_counts( + self, counts_input: Any, ep_size: int, + ) -> Any: + """Replay the non-differentiable count all-to-all at a fixed shape.""" + from hyper_parallel.core.context_parallel.utils import ( # pylint: disable=C0415 + all_to_all_single, + ) + + if ep_size == 1: + return counts_input.clone() + try: + ep_group = self._ep_mesh().get_group() + except (KeyError, TypeError, AttributeError, ValueError) as error: + raise ValueError("Configured MoE dry-run requires an accessible EP process group") from error + counts_output, handle = all_to_all_single( + counts_input, + output_shape=[counts_input.shape[0]], + group=ep_group, + async_op=True, + ) + if handle is not None: + handle.wait() + return counts_output + + @staticmethod + def _dry_run_rank_to_expert_indices( + counts: Any, total_tokens: int, + ep_size: int, local_expert_count: int, + ) -> Any: + """Replay rank-major to expert-major index construction at fixed shape.""" + import torch # pylint: disable=C0415,forbidden-backend-import + + counts_2d = counts.view(ep_size, local_expert_count) + source_offsets = counts.cumsum(0) - counts + expert_major_offsets = source_offsets.view( + ep_size, local_expert_count, + ).transpose(0, 1).contiguous().view(-1) + expert_major_counts = counts_2d.transpose(0, 1).contiguous().view(-1) + block_source_starts = torch.repeat_interleave( + expert_major_offsets, + expert_major_counts, + output_size=total_tokens, + ) + destination_offsets = expert_major_counts.cumsum(0) - expert_major_counts + destination_starts = torch.repeat_interleave( + destination_offsets, + expert_major_counts, + output_size=total_tokens, + ) + intra_block_offsets = torch.arange( + total_tokens, device=counts.device, + ) - destination_starts + return (block_source_starts + intra_block_offsets).long() + + @staticmethod + def _dry_run_grouped_matmul( + input_tensor: Any, weight: Any, + expert_loads: tuple[int, ...], + ) -> Any: + """Replay grouped matmul events with one output and one weight gradient.""" + import torch # pylint: disable=C0415,forbidden-backend-import + + class _DryRunGroupedMatmul(torch.autograd.Function): + """Grouped matmul analogue with preallocated forward/backward buffers.""" + + @staticmethod + def forward( + ctx: Any, value: Any, grouped_weight: Any, + loads: tuple[int, ...], + ) -> Any: + """Compute packed expert outputs into one logical allocation.""" + ctx.save_for_backward(value, grouped_weight) + ctx.expert_loads = loads + output = torch.zeros( + value.shape[0], grouped_weight.shape[-1], + dtype=value.dtype, device=value.device, + ) + start = 0 + for expert_index, load in enumerate(loads): + end = start + load + if load: + torch.mm( + value[start:end], grouped_weight[expert_index], + out=output[start:end], + ) + start = end + return output + + @staticmethod + def backward(ctx: Any, grad_output: Any) -> tuple[Any, Any, None]: + """Compute packed input and expert-weight gradients.""" + value, grouped_weight = ctx.saved_tensors + grad_input = torch.zeros_like(value) + grad_weight = torch.zeros_like(grouped_weight) + start = 0 + for expert_index, load in enumerate(ctx.expert_loads): + end = start + load + if load: + torch.mm( + grad_output[start:end], grouped_weight[expert_index].transpose(0, 1), + out=grad_input[start:end], + ) + torch.mm( + value[start:end].transpose(0, 1), grad_output[start:end], + out=grad_weight[expert_index], + ) + start = end + return grad_input, grad_weight, None + + if len(expert_loads) != int(weight.shape[0]): + raise ValueError( + "MoE dry-run expert load count must match the local expert weight count" + ) + if sum(expert_loads) != int(input_tensor.shape[0]): + raise ValueError( + "MoE dry-run local expert loads must sum to the dispatched token count" + ) + return _DryRunGroupedMatmul.apply(input_tensor, weight, expert_loads) + + def _run_grouped_experts( + self, layout: str, weights: tuple[Any, ...], expert_input: Any, + expert_loads: tuple[int, ...], + ) -> Any: + """Execute the supported expert layout without per-expert slicing.""" + from torch.nn import functional # pylint: disable=C0415,forbidden-backend-import + + if layout == "three_projection": + w1, w2, w3 = weights + gate = self._dry_run_grouped_matmul( + expert_input, w1.transpose(-2, -1), expert_loads, + ) + up = self._dry_run_grouped_matmul( + expert_input, w3.transpose(-2, -1), expert_loads, + ) + hidden = functional.silu(gate) * up + return self._dry_run_grouped_matmul( + hidden, w2.transpose(-2, -1), expert_loads, + ) + gate_up, down = weights + gate_up_output = self._dry_run_grouped_matmul( + expert_input, gate_up.transpose(-2, -1), expert_loads, + ) + gate, up = gate_up_output.chunk(2, dim=-1) + hidden = functional.silu(gate) * up + return self._dry_run_grouped_matmul( + hidden, down.transpose(-2, -1), expert_loads, + ) + + @staticmethod + def _run_eager_experts( + experts: Any, + layout: str, + weights: tuple[Any, ...], + expert_input: Any, + expert_loads: tuple[int, ...], + ) -> Any: + """Replay the real per-expert eager path and its saved activations.""" + from torch import cat # pylint: disable=C0415,forbidden-backend-import + from torch.nn import functional # pylint: disable=C0415,forbidden-backend-import + + outputs = [] + start = 0 + for expert_index, load in enumerate(expert_loads): + end = start + load + states = expert_input[start:end] + if layout == "three_projection": + w1, w2, w3 = weights + gate = functional.linear(states, w1[expert_index]) + up = functional.linear(states, w3[expert_index]) + hidden = functional.silu(gate) * up + output = functional.linear(hidden, w2[expert_index]) + else: + gate_up, down = weights + gate_up_states = functional.linear(states, gate_up[expert_index]) + apply_gate = getattr(experts, "_ep_apply_gate", None) + if apply_gate is None: + gate, up = gate_up_states.chunk(2, dim=-1) + hidden = functional.silu(gate) * up + else: + hidden = apply_gate(gate_up_states) + output = functional.linear(hidden, down[expert_index]) + outputs.append(output) + start = end + return cat(outputs, dim=0) + + def _differentiable_all_to_all( + self, input_tensor: Any, + input_splits: tuple[int, ...], output_splits: tuple[int, ...], + ) -> Any: + """Execute the real differentiable EP collective with configured splits.""" + from hyper_parallel.core.utils.communication import ( # pylint: disable=C0415 + differentiable_all_to_all_single, + ) + + try: + ep_group = self._ep_mesh().get_group() + except (KeyError, TypeError, AttributeError, ValueError) as error: + if len(input_splits) == 1 and input_splits == output_splits: + return input_tensor.clone() + raise ValueError( + "Configured MoE dry-run requires an accessible EP process group" + ) from error + return differentiable_all_to_all_single( + input_tensor, + list(input_splits), + list(output_splits), + group=ep_group, + ) + + def _ep_group_identity(self) -> tuple[int, int]: + """Return EP group size/rank, falling back to the configured topology.""" + try: + ep_mesh = self._ep_mesh() + return int(ep_mesh.size()), int(ep_mesh.get_local_rank()) + except (KeyError, TypeError, AttributeError, ValueError): + return int(self._base.mesh.ep_size), int(self._base.mesh.ep_rank) + + def _ep_mesh(self) -> Any: + """Return the expert-parallel child mesh from the HyperModels mesh context.""" + expert_mesh = getattr(self._base.mesh, "fsdp_moe_mesh", None) + if expert_mesh is None: + raise ValueError("Configured MoE dry-run requires an expert mesh") + return expert_mesh["ep"] + + def _restore_moe(self) -> None: + """Restore every MoE instance forward replaced by this scope.""" + if self._original_ep_compute is not None: + while self._ep_compute_modules: + patched_module = self._ep_compute_modules.pop() + patched_module.ep_routed_forward = self._original_ep_compute + self._original_ep_compute = None + self._moe_module_names.clear() + while self._patched_moe_modules: + module, had_forward, original_forward = self._patched_moe_modules.pop() + if had_forward: + module.forward = original_forward + else: + del module.forward + + +class FakeHostScalarShim(AbstractContextManager): + """Resolve host-scalar reads on FakeTensors with probe-derived values. + + The real Trainer step loop reads a handful of host scalars (``loss.item()`` + for logging accumulators, token counts for gradient normalization, + ``grad_norm.item()`` for diagnostics). FakeTensors have no values, so these + raise ``DataDependentOutputException``. The CPU data probe, however, has + already observed the only value-bearing quantities the step loop needs + (token counts); every other scalar only feeds logging. This scope resolves + ``Tensor.item`` on FakeTensors: zero-filled by default, so loss metrics + read as 0.0 without touching tensor shapes, while probe-known token counts + keep gradient-normalization denominators real. Real tensors pass through. + """ + + def __init__(self) -> None: + """Create the scope without installing it.""" + import torch # pylint: disable=C0415,forbidden-backend-import + from torch.overrides import TorchFunctionMode # pylint: disable=C0415,forbidden-backend-import + + outer = self + + class _Mode(TorchFunctionMode): + """Intercept host-scalar reads before FakeTensor rejects them.""" + + def __torch_function__(self, func, types, args=(), kwargs=None): + kwargs = kwargs or {} + if args and hasattr(args[0], "fake_mode"): + name = getattr(func, "__name__", "") + if func is torch.Tensor.item or name == "__float__": + return outer._resolve(args[0]) + if name == "__int__" or name == "__index__": + return int(outer._resolve(args[0])) + if name == "__bool__": + return bool(outer._resolve(args[0])) + return func(*args, **kwargs) + + self._mode = _Mode() + self._known: Dict[int, float] = {} + + def register(self, tensor: Any, value: float) -> None: + """Pin a probe-derived host value onto one fake tensor identity.""" + self._known[id(tensor)] = float(value) + + def _resolve(self, tensor: Any) -> float: + """Return the pinned value for this tensor, zero otherwise.""" + return self._known.get(id(tensor), 0.0) + + def __enter__(self) -> "FakeHostScalarShim": + self._mode.__enter__() + return self + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> bool: + self._mode.__exit__(exc_type, exc_value, traceback) + return False + + +class DryRunBatchIterator: + """Replay one mocked micro-batch as the step's data iterator. + + The real ``get_batch`` runtime consumed the probe batch on CPU; the fake + step replays its mocked twin for every micro-step so ``train_step`` can + drive the unmodified data-path contract (an iterator whose items reach + ``base.get_batch`` and come back as ``(model_inputs, loss_inputs)``). + """ + + def __init__(self, model_inputs: Mapping[str, Any], loss_inputs: Mapping[str, Any]) -> None: + """Store the single replayed micro-batch.""" + self._model_inputs = dict(model_inputs) + self._loss_inputs = dict(loss_inputs) + + def __iter__(self) -> "DryRunBatchIterator": + return self + + def __next__(self) -> tuple: + """Yield the same prepared micro-batch indefinitely.""" + return dict(self._model_inputs), dict(self._loss_inputs) diff --git a/hyper_parallel/trainer/callbacks/environ_meter_callback.py b/hyper_parallel/trainer/callbacks/environ_meter_callback.py index 88712eaebc..9b2c7e226e 100644 --- a/hyper_parallel/trainer/callbacks/environ_meter_callback.py +++ b/hyper_parallel/trainer/callbacks/environ_meter_callback.py @@ -24,7 +24,11 @@ from hyper_parallel.trainer.runtime.distributed import all_reduce from hyper_parallel.trainer.runtime.flops import estimate_step_flops from hyper_parallel.data.constants import IGNORE_INDEX -from hyper_parallel.trainer.runtime.device import get_device_type, get_torch_device +from hyper_parallel.trainer.runtime.device import ( + get_device_type, + get_torch_device, + is_device_runtime_simulated, +) from .base import Callback, TrainerState @@ -295,7 +299,9 @@ def _current_lr(self) -> float: def _memory_metrics(self) -> dict[str, float]: """Collect maximum accelerator memory metrics, if available.""" - if get_device_type() == "cpu": + if get_device_type() == "cpu" or is_device_runtime_simulated(): + # Logical simulation never touches the accelerator runtime; + # querying allocator statistics would initialize it. return {} device = get_torch_device() allocated = self._reduce(device.max_memory_allocated(), op="max") diff --git a/hyper_parallel/trainer/config/data.py b/hyper_parallel/trainer/config/data.py index e1c4d71291..de6f83dda1 100644 --- a/hyper_parallel/trainer/config/data.py +++ b/hyper_parallel/trainer/config/data.py @@ -49,7 +49,13 @@ def build(self, **runtime_kwargs: Any) -> Any: def __getattr__(self, name: str) -> Any: """Expose configured Dataset options through the wrapped target.""" - return getattr(self.target, name) + # __getattr__ runs for *missing* attributes. During copy/pickle + # reconstruction the instance dict is still empty, so probing + # "target" through normal attribute lookup would recurse forever. + target = self.__dict__.get("target") + if target is None: + raise AttributeError(name) + return getattr(target, name) def to_dict(self) -> dict[str, Any]: """Serialize Dataset components in their compact nested YAML shape.""" @@ -86,7 +92,12 @@ def build(self, **runtime_kwargs: Any) -> Any: def __getattr__(self, name: str) -> Any: """Expose configured DataLoader options through the wrapped target.""" - return getattr(self.target, name) + # See DatasetConfig.__getattr__: guard the not-yet-initialized state + # reached during copy/pickle reconstruction. + target = self.__dict__.get("target") + if target is None: + raise AttributeError(name) + return getattr(target, name) def to_dict(self) -> dict[str, Any]: """Serialize components in their compact nested YAML shape.""" diff --git a/hyper_parallel/trainer/config/trainer.py b/hyper_parallel/trainer/config/trainer.py index 0db0dcb340..237f4308a0 100644 --- a/hyper_parallel/trainer/config/trainer.py +++ b/hyper_parallel/trainer/config/trainer.py @@ -85,6 +85,7 @@ class TrainerConfig: wandb: WandbConfig = field(default_factory=WandbConfig) profiler: ProfilingConfig = field(default_factory=ProfilingConfig) stress_detect: StressDetectConfig = field(default_factory=StressDetectConfig) + dry_run: Optional[Any] = None magi: Optional[Any] = None peft: Optional[Any] = None diff --git a/hyper_parallel/trainer/runtime/device.py b/hyper_parallel/trainer/runtime/device.py index 11c2a24796..5ac0f6bd5b 100644 --- a/hyper_parallel/trainer/runtime/device.py +++ b/hyper_parallel/trainer/runtime/device.py @@ -37,6 +37,8 @@ "stream_synchronize", "empty_cache", "set_device", + "simulate_device_runtime", + "is_device_runtime_simulated", "get_device_rng_state", "set_device_rng_state", "is_nccl_backend", @@ -51,6 +53,17 @@ import torch +# Logical-simulation mode: when enabled (by the dry-run tool before it drives +# real Trainer code over FakeTensors), every device-side effect in this module +# becomes a no-op so the process never binds an accelerator, initializes its +# runtime, or appears on device tooling. Pure queries (device type, flags) +# stay live — simulation intentionally reports the real accelerator topology. +# The switch itself is owned by core so core components (activation swap, +# optimizer swap, FSDP offload) consult the same process-wide state. +from hyper_parallel.core.utils.simulation import ( + is_device_runtime_simulated, + simulate_device_runtime, +) from hyper_parallel.models.build_options import ( IS_CUDA_AVAILABLE, IS_NPU_AVAILABLE, @@ -79,11 +92,15 @@ def get_dist_comm_backend() -> str: def synchronize() -> None: """Execute torch synchronize operation.""" + if is_device_runtime_simulated(): + return get_torch_device().synchronize() def stream_synchronize() -> None: """Execute device stream synchronize operation.""" + if is_device_runtime_simulated(): + return if IS_CUDA_AVAILABLE: torch.cuda.current_stream().synchronize() elif IS_NPU_AVAILABLE: @@ -94,11 +111,15 @@ def stream_synchronize() -> None: def empty_cache() -> None: """Execute torch empty cache operation.""" + if is_device_runtime_simulated(): + return get_torch_device().empty_cache() def set_device(device: torch.types.Device) -> None: """Execute set device operation.""" + if is_device_runtime_simulated(): + return get_torch_device().set_device(device) diff --git a/setup.py b/setup.py index 5c1f2b479a..141673e9fb 100644 --- a/setup.py +++ b/setup.py @@ -260,6 +260,8 @@ def has_ext_modules(self) -> bool: 'generated/*.json', 'generated/*.diff', ], 'hyper_parallel.models.deepseek_v41.tools': ['*.sh', '*.yaml', '*.md'], + 'hyper_parallel.tools.dryrun': ['README.md', 'DESIGN.md'], + 'hyper_parallel.tools.dryrun.examples': ['*.yaml'], 'hyper_parallel.auto_parallel': ['op_profiles/*.yaml', 'op_records.yaml'], 'hyper_parallel.models.deepseek_v3': ['recipes/*.yaml'], 'hyper_parallel.auto_parallel.sapp_nd.memory_estimation': [ diff --git a/tests/torch/tools/__init__.py b/tests/torch/tools/__init__.py new file mode 100644 index 0000000000..33ef3b5685 --- /dev/null +++ b/tests/torch/tools/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Tool-level torch STs (dry-run memory reconciliation).""" diff --git a/tests/torch/tools/dryrun_sync_common.py b/tests/torch/tools/dryrun_sync_common.py new file mode 100644 index 0000000000..7969912cd9 --- /dev/null +++ b/tests/torch/tools/dryrun_sync_common.py @@ -0,0 +1,217 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Dry-run vs real-training memory reconciliation (no test_ prefix: helpers). + +The reconciliation compares the dry-run logical snapshot against the real +allocator snapshot of the same recipe: + +1. Peak ratio: the logical peak must stay within a tolerance band of the real + trace-level peak (allocator reserve excluded by replaying alloc/free events). +2. Site census: every large allocation call site observed in the real trace + must appear in the dry-run trace with the same block sizes. This is the + sensitive check — a divergence in trainer-loop replay (for example a missed + reduce-scatter scheduling branch) shows up here as a named ``file:line`` + before it is visible in the peak. +""" +from __future__ import annotations + +import glob +import pickle +from collections import Counter +from typing import Any + +#: Blocks below this size are noise for divergence hunting. +LARGE_BLOCK_BYTES = 32 * 2**20 + +#: Logical peak must stay within this band of the real trace peak. Measured +#: anchors on the validation crop: CP2 0.94 (value-dependent CSA2 interleave), +#: everything else 0.98–1.02. The band keeps modest headroom without +#: accepting the silent 10–20% skew the previous (0.80, 1.10) band allowed. +PEAK_RATIO_BAND = (0.90, 1.05) + +#: Value-dependency replay owns these modules: their tensor sizes depend on +#: runtime values (document boundaries, routing loads) that dry-run replays +#: with balanced/synthetic equivalents, so per-site size census is expected to +#: differ there. The peak-band check still covers their aggregate. Everything +#: outside this list — FSDP, communication, optimizer — must census-match: +#: a mismatch there is a trainer-loop replay divergence. +VALUE_REPLAY_WAIVED_MODULES = ( + "components/modules/shared_compressed_dsa_attention.py", + "components/modules/engram.py", + "components/functional/", +) + + +def load_snapshot(path: str) -> dict[str, Any]: + """Load one allocator-style snapshot pickle.""" + with open(path, "rb") as snapshot_file: + return pickle.load(snapshot_file) + + +def latest_snapshot(directory: str) -> dict[str, Any]: + """Load the newest snapshot pickle in a directory.""" + files = sorted(glob.glob(f"{directory}/*.pickle")) + if not files: + raise FileNotFoundError(f"no snapshot pickle under {directory}") + return load_snapshot(files[-1]) + + +def trace_peak_bytes(snapshot: dict[str, Any]) -> int: + """Replay alloc/free events and return the trace-level peak.""" + peak = 0 + current = 0 + for trace in snapshot["device_traces"]: + for event in trace: + if event["action"] == "alloc": + current += event["size"] + peak = max(peak, current) + elif event["action"] == "free_completed": + current -= event["size"] + return peak + + +def _allocation_site(event: dict[str, Any]) -> str: + """Return the first project-owned python frame of one allocation. + + Line numbers are bucketed (//8) because the native allocator and the + logical tracker capture a multi-line expression on slightly different + lines; the bucket keeps the site discriminating without false negatives. + """ + for frame in event.get("frames", ()): + filename = frame.get("filename", "") + if not filename.endswith(".py"): + continue + if "hyper_parallel" not in filename: + continue + if any(skip in filename for skip in ("tools/dryrun", "trainer/")): + continue + line_bucket = int(frame.get("line", 0)) // 8 + return f"{filename.split('hyper_parallel/')[-1]}:~{line_bucket * 8}" + return "(anonymous)" + + +def large_allocation_census(snapshot: dict[str, Any]) -> Counter: + """Count large allocations keyed by (site, size bucket). + + Sizes are bucketed to 1 MiB because the native allocator pads requests + (for example 32 KiB NPU alignment) while the logical tracker records the + exact storage bytes; the bucket absorbs padding without hiding a missing + buffer class. + """ + census: Counter = Counter() + for trace in snapshot["device_traces"]: + for event in trace: + if event["action"] == "alloc" and event["size"] >= LARGE_BLOCK_BYTES: + size_bucket = round(event["size"] / 2**20) + census[(_allocation_site(event), size_bucket)] += 1 + return census + + +def reconcile(dry_snapshot: dict[str, Any], real_snapshot: dict[str, Any]) -> list[str]: + """Return a list of human-readable divergences (empty when reconciled). + + The real allocator snapshot typically spans several optimizer steps + (``memory.start_step``..``end_step``) while dry-run simulates exactly one, + so call-site counts are compared per step: the real census is divided by + the smallest ratio that makes the most common sites match. A genuine replay + divergence breaks per-site proportionality and still fails. + """ + problems: list[str] = [] + + dry_peak = int(dry_snapshot["summary"]["peak_bytes"]) + real_peak = trace_peak_bytes(real_snapshot) + ratio = dry_peak / max(real_peak, 1) + low, high = PEAK_RATIO_BAND + if not low <= ratio <= high: + problems.append( + f"peak ratio {ratio:.3f} outside [{low}, {high}]: " + f"dry={dry_peak / 2**30:.3f} GiB real={real_peak / 2**30:.3f} GiB" + ) + + dry_census = large_allocation_census(dry_snapshot) + real_census = large_allocation_census(real_snapshot) + steps = _estimate_real_steps(dry_census, real_census) + for (site, size_bucket), count in sorted(real_census.items(), key=lambda item: -item[0][1]): + if site == "(anonymous)": + continue + if any(waived in site for waived in VALUE_REPLAY_WAIVED_MODULES): + continue + per_step = count / steps + dry_count = dry_census.get((site, size_bucket), 0) + if dry_count < per_step: + problems.append( + f"real trace has {per_step:g}x/step ~{size_bucket} MiB at {site}, " + f"dry-run only {dry_count}x — trainer-loop replay divergence" + ) + return problems + + +def _estimate_real_steps(dry_census: Counter, real_census: Counter) -> int: + """Estimate how many optimizer steps the real snapshot window spans. + + Uses the modal real/dry count ratio over sites both traces share. Falls + back to 1 (strictest comparison) when no site is shared. + """ + ratios: Counter = Counter() + for key, real_count in real_census.items(): + dry_count = dry_census.get(key, 0) + if dry_count > 0 and real_count % dry_count == 0: + ratios[real_count // dry_count] += 1 + if not ratios: + return 1 + return max(ratios.most_common(1)[0][0], 1) + + +#: Host-ledger tolerance: the simulated host bytes must cover at least this +#: fraction of the expected host footprint derived from the scenario's device +#: saving (host mirrors are what the device bytes were traded for). +HOST_COVERAGE_MIN = 0.5 + + +def check_offload_trade( + scenario_snapshot: dict[str, Any], + baseline_snapshot: dict[str, Any], +) -> list[str]: + """Assert the device/host trade an offload or swap scenario must show. + + The reconciliation census proves the device story matches the real run; + this check covers the other half of capacity planning, which has no real + counterpart trace: the host ledger must actually carry the bytes the + device saved (within :data:`HOST_COVERAGE_MIN`, since padding, dtype + casts, and transient staging legitimately skew the two sides). + """ + problems: list[str] = [] + scenario = scenario_snapshot["summary"] + baseline = baseline_snapshot["summary"] + host_peak = int(scenario.get("host_peak_bytes", 0)) + device_saving = int(baseline["peak_bytes"]) - int(scenario["peak_bytes"]) + if host_peak <= 0: + problems.append( + "offload/swap scenario reports an empty host ledger — host" + " mirrors were misclassified onto the device account" + ) + if device_saving < 0 and scenario["peak_bytes"] > baseline["peak_bytes"] * 1.02: + problems.append( + f"offload/swap scenario device peak {scenario['peak_bytes']}" + f" exceeds baseline {baseline['peak_bytes']} — the feature did" + " not reduce (or hold) the device footprint" + ) + if device_saving > 0 and host_peak < device_saving * HOST_COVERAGE_MIN: + problems.append( + f"host ledger {host_peak} covers less than" + f" {HOST_COVERAGE_MIN:.0%} of the device saving {device_saving}" + " — host accounting is undercounting the trade" + ) + return problems diff --git a/tests/torch/tools/dryrun_sync_recipe.yaml b/tests/torch/tools/dryrun_sync_recipe.yaml new file mode 100644 index 0000000000..7af1abb469 --- /dev/null +++ b/tests/torch/tools/dryrun_sync_recipe.yaml @@ -0,0 +1,203 @@ +# DeepSeek-V4.1 validation-crop recipe for the dry-run memory sync ST. +# @MODEL_PATH@/@ENGRAM_PATH@/@DATA_PATH@/@OUTPUT_ROOT@ are substituted by the +# launcher from HP_DRYRUN_ST_* environment variables (assets are local-only). + +model: + _target_: hyper_parallel.tools.dryrun.examples.deepseek_v41.build_text_model + config_path: @MODEL_PATH@ + engram_assets_path: @ENGRAM_PATH@ + torch_dtype: bfloat16 + num_hidden_layers: 8 + text_parameter_divisor: 8 + num_routed_experts: 16 + loss_chunks: 8 + recompute_tier: none + validate_placement: false +model_init_dtype: float32 + +training: + train_iters: 3 # memory profiler dumps after step 2 + global_batch_size: 8 + micro_batch_size: 1 + backend: hccl + init_device: meta + loss_aggregation: token_weighted + max_grad_norm: 1.0 + seed: 42 + +accelerator: + tp_size: 1 + cp_size: 1 + ep_size: 8 + pp_size: 1 + sequence_parallel: false + loss_parallel: false + +fsdp_config: + dp_shard_size: 8 + edp_shard_size: 1 + mix_precision: {param_dtype: bfloat16, reduce_dtype: float32, cast_forward_inputs: false} + reshard_after_forward: true + reshard_after_backward: true + requires_grad_sync: true + forward_prefetch_depth: 0 + backward_prefetch_depth: 0 + comm_fusion: false + +plan_overrides: + - match: "model.layers.*.engram" + replace: + kind: module + implementation: + _target_: hyper_parallel.components.modules.engram.EngramModule + - match: "model.layers.*.self_attn" + replace: + kind: module + implementation: + _target_: >- + hyper_parallel.components.modules.shared_compressed_dsa_attention.SharedCompressedDSAAttention + use_fused_ops: false + - match: "*.self_attn" + tp_divide_attrs: [num_groups] + - match: "*.self_attn.forward" + replace: + kind: method + implementation: + _target_: >- + hyper_parallel.models.deepseek_v41.adapter.distributed.shared_attention_context_parallel.deepseek_v41_shared_attention_parallel_wrapper + - match: "model.layers.*.*_hc" + region_dispatch: false + params: + fn: {tp: replicate} + base: {tp: replicate} + scale: {tp: replicate} + in_src: + hidden_streams: {tp: replicate, cp: "shard(1)", ep: replicate} + in_dst: + hidden_streams: {tp: replicate, cp: "shard(1)", ep: replicate} + out_src: + pre: {tp: replicate, cp: "shard(1)", ep: replicate} + post: {tp: replicate, cp: "shard(1)", ep: replicate} + residual: {tp: replicate, cp: "shard(1)", ep: replicate} + out_dst: + pre: {tp: replicate, cp: "shard(1)", ep: replicate} + post: {tp: replicate, cp: "shard(1)", ep: replicate} + residual: {tp: replicate, cp: "shard(1)", ep: replicate} + - match: "model.layers.*.*_hc" + when: sequence_parallel + in_src: + hidden_streams: {tp: "shard(1)", cp: "shard(1)", ep: replicate} + in_dst: + hidden_streams: {tp: "shard(1)", cp: "shard(1)", ep: replicate} + out_src: + pre: {tp: "shard(1)", cp: "shard(1)", ep: replicate} + post: {tp: "shard(1)", cp: "shard(1)", ep: replicate} + residual: {tp: "shard(1)", cp: "shard(1)", ep: replicate} + out_dst: + pre: {tp: "shard(1)", cp: "shard(1)", ep: replicate} + post: {tp: "shard(1)", cp: "shard(1)", ep: replicate} + residual: {tp: "shard(1)", cp: "shard(1)", ep: replicate} + - match: "model.layers.*.engram" + params: + q_weight: {tp: replicate} + k_weight: {tp: replicate} + embed.weight: {ep: "shard(0)"} + wkv.weight: {tp: replicate} + - match: "model.layers.*.engram.forward" + when: ep + replace: + kind: method + implementation: + _target_: >- + hyper_parallel.models.deepseek_v41.adapter.distributed.moe_engram_expert_parallel.deepseek_v41_engram_compute_fn + - match: "model.layers.*.mlp.forward" + when: ep + replace: + kind: method + implementation: + _target_: >- + hyper_parallel.models.deepseek_v41.adapter.distributed.moe_engram_expert_parallel.deepseek_v41_ep_compute_fn + use_grouped_gemm: false + fix_router: true + +activation_checkpoint: + mode: full + +dataset: + _target_: hyper_parallel.data.text.build_dataset.build_online_text_mapping_dataset + data_path: @DATA_PATH@ + data_config: {create_attention_mask_in_dataloader: true, labels_are_shifted: true} + model_assets: + chat_template: null + tokenizer: + _target_: hyper_parallel.data.text.build_tokenizer.AutoTokenizer.from_pretrained + pretrained_model_name_or_path: @MODEL_PATH@ + local_files_only: true + use_fast: true + trust_remote_code: false + data_transform: + _target_: hyper_parallel.data.text.text_transform.build_text_transform + data_type: plaintext + text_keys: text + max_seq_len: 4096 + drop_ragged_tail: true + +dataloader: + _target_: hyper_parallel.data.batching.TokenBatchLoader + min_buffered_samples: 1 + collate_fn: + _target_: hyper_parallel.data.batching.build_online_text_collate_fn + pad_to_length: 4096 + get_batch: + _target_: hyper_parallel.data.batching.TextParallelBatch + source_type: online + attention_mode: compressed + runtime_input_adapter: + _target_: hyper_parallel.models.deepseek_v41.adapter.data.runtime.DeepseekV41Runtime + include_position_ids: false + include_image_sequence_start: false + sampler_type: single + data_sharding: false + drop_last: true + use_background_prefetcher: false + num_workers: 0 + pin_memory: false + +optimizer: + _target_: hyper_parallel.components.optim.Muon + fp32_main_params: true + no_decay_params: [bias, norm, ln_] + muon_config: {muon_lr: 3.0e-05, muon_weight_decay: 0.0, muon_momentum: 0.95, + muon_nesterov: true, muon_ns_steps: 5, muon_ns_variant: asym5} + adamw_config: {adamw_lr: 1.0e-05, adamw_weight_decay: 0.01, + adamw_betas: [0.9, 0.999], adamw_eps: 1.0e-08} + +lr_scheduler: + _target_: hyper_parallel.components.optim.MultiLRScheduler + lr_decay_style: cosine + lr_config: + lr: 3.0e-5 + lr_warmup_steps: 1 + min_lr: 0.0 + +checkpoint: {save_ckpt: false, checkpoint_dir: @OUTPUT_ROOT@/ckpt_unused} + +# Real-run allocator snapshots; inert for dryrun. +memory: + enable: true + start_step: 1 + end_step: 1 # single-step window: counts compare 1:1 with the one-step simulation + save_path: @OUTPUT_ROOT@/real_snapshots + dump_ranks: [0] + stacks: all + +dry_run: + enabled: true + target_device: npu + ranks: [0] + virtual_world_size: 8 + snapshot: + enabled: true + path: @OUTPUT_ROOT@/dryrun_snapshots + capture_stacks: true + stack_depth: 64 diff --git a/tests/torch/tools/test_dryrun_memory_sync.py b/tests/torch/tools/test_dryrun_memory_sync.py new file mode 100644 index 0000000000..7ea4263fe5 --- /dev/null +++ b/tests/torch/tools/test_dryrun_memory_sync.py @@ -0,0 +1,159 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Dry-run vs real-training memory reconciliation ST (launcher, torch-free). + +Runs the same DeepSeek-V4.1 validation-crop recipe twice — once through the +dry-run simulator (single process, FakeTensors, no device) and once through +torchrun on 8 cards with allocator snapshots — then reconciles the two memory +traces: the logical peak must land inside a tolerance band of the real trace +peak, and every large real allocation call site must be present in the +dry-run trace (see ``dryrun_sync_common.reconcile``). + +Model/tokenizer/data assets are environment-supplied (the released config is +not redistributable). Set: + + HP_DRYRUN_ST_MODEL_PATH DeepSeek-V4.1-Flash config/tokenizer directory + HP_DRYRUN_ST_ENGRAM_PATH engram_depth_preserving_d8.json + HP_DRYRUN_ST_DATA_PATH online_4k.jsonl + +The test skips when any is unset or missing. +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +import pytest + +from tests.common.mark_utils import arg_mark + +_ASSET_ENV = { + "model": "HP_DRYRUN_ST_MODEL_PATH", + "engram": "HP_DRYRUN_ST_ENGRAM_PATH", + "data": "HP_DRYRUN_ST_DATA_PATH", +} + +#: Scenario matrix: (name, extra CLI overrides). The base recipe is EP8/FSDP8. +#: The offload/swap scenarios exercise the host-ordered storage lifecycle +#: (resize_(0) on wait_offload, resize_ + copy on load) through the same +#: census: a missed release or restore shows up as a device-side count skew. +#: The baseline runs first so offload/swap scenarios can additionally assert +#: the device/host trade against it (host ledger has no real counterpart +#: trace, so it is checked for coverage of the device saving instead). +_BASELINE = ("baseline", ()) +_SCENARIOS = ( + _BASELINE, + ("cp2_ep4_fsdp4", ( + "--accelerator.cp_size=2", + "--accelerator.ep_size=4", + "--fsdp_config.dp_shard_size=4", + )), + ("fsdp_cpu_offload", ( + "--fsdp_config.enable_offload=true", + )), + ("attention_swap", ( + "--activation_swap=attention", + "--activation_checkpoint.mode=off", + )), +) +#: Scenarios whose device/host trade is asserted against the baseline. +_OFFLOAD_SCENARIOS = ("fsdp_cpu_offload", "attention_swap") + + +def _assets() -> dict: + """Resolve env-supplied assets or skip the ST.""" + resolved = {} + for key, env_name in _ASSET_ENV.items(): + value = os.environ.get(env_name) + if not value or not Path(value).exists(): + pytest.skip(f"{env_name} is unset or missing; dry-run sync ST needs local V4.1 assets") + resolved[key] = value + return resolved + + +def _write_recipe(directory: Path, assets: dict) -> Path: + """Materialize the reconciliation recipe with env-supplied asset paths.""" + template = Path(__file__).with_name("dryrun_sync_recipe.yaml").read_text(encoding="utf-8") + recipe = ( + template + .replace("@MODEL_PATH@", assets["model"]) + .replace("@ENGRAM_PATH@", assets["engram"]) + .replace("@DATA_PATH@", assets["data"]) + .replace("@OUTPUT_ROOT@", str(directory)) + ) + path = directory / "recipe.yaml" + path.write_text(recipe, encoding="utf-8") + return path + + +def _run(command: list, **kwargs) -> None: + """Run one child process, surfacing its tail on failure.""" + result = subprocess.run(command, capture_output=True, text=True, check=False, **kwargs) + if result.returncode != 0: + tail = (result.stdout + result.stderr)[-4000:] + raise AssertionError(f"{' '.join(command[:6])}... failed:\n{tail}") + + +@arg_mark(plat_marks=["platform_ascend910b"], level_mark="level1", card_mark="allcards", essential_mark="essential") +def test_dryrun_memory_reconciles_with_real_training(): + """ + Feature: Dry-run logical memory vs real allocator traces. + Description: Run the V4.1 validation crop through the dry-run simulator and + through an 8-card real training step with allocator snapshots; compare + peak memory and per-call-site large allocations. + Expectation: Peak ratio inside the tolerance band; every large real + allocation site is present in the dry-run trace. + """ + assets = _assets() + with tempfile.TemporaryDirectory() as raw_directory: + directory = Path(raw_directory) + recipe = _write_recipe(directory, assets) + for name, overrides in _SCENARIOS: + dry_dir = directory / f"dry_{name}" + real_dir = directory / f"real_{name}" + _run([ + sys.executable, "-m", "hyper_parallel.tools.dryrun", str(recipe), + *overrides, + f"--dry_run.snapshot.path={dry_dir}", + ]) + _run([ + sys.executable, "-m", "torch.distributed.run", + "--standalone", "--nproc_per_node=8", + "--module", "examples.training_demo.train_text", str(recipe), + *overrides, + f"--memory.save_path={real_dir}", + ]) + report = subprocess.run( + [sys.executable, "-c", ( + "import sys, json; sys.path.insert(0, sys.argv[1]);" + "import dryrun_sync_common as c;" + "dry = c.load_snapshot(sys.argv[2] + '/rank_0/memory.pickle');" + "real = c.latest_snapshot(sys.argv[3]);" + "problems = c.reconcile(dry, real);" + "baseline_dir = sys.argv[4];" + "problems += c.check_offload_trade(dry," + " c.load_snapshot(baseline_dir + '/rank_0/memory.pickle'))" + " if baseline_dir else [];" + "print(json.dumps(problems))" + ), str(Path(__file__).parent), str(dry_dir), str(real_dir), + str(directory / f"dry_{_BASELINE[0]}") if name in _OFFLOAD_SCENARIOS else ""], + capture_output=True, text=True, check=True, + ) + problems = json.loads(report.stdout.strip().splitlines()[-1]) + assert not problems, f"scenario {name} diverged:\n" + "\n".join(problems) diff --git a/tests/ut/core/activation_memory/test_pinned_memory_pool.py b/tests/ut/core/activation_memory/test_pinned_memory_pool.py index 94746419ec..ce4048ee39 100644 --- a/tests/ut/core/activation_memory/test_pinned_memory_pool.py +++ b/tests/ut/core/activation_memory/test_pinned_memory_pool.py @@ -42,20 +42,14 @@ def synchronize(self) -> None: class _SyntheticStorage: - """Storage exposing a caller-controlled identity.""" - - def __init__(self, key: int) -> None: - self.key = key - - def data_ptr(self) -> int: - return self.key + """Storage stand-in; the pool keys blocks by this object's identity.""" class _SyntheticView: """Tensor-like object used to emulate framework storage views.""" - def __init__(self, key: int) -> None: - self.storage = _SyntheticStorage(key) + def __init__(self) -> None: + self.storage = _SyntheticStorage() def untyped_storage(self) -> _SyntheticStorage: return self.storage @@ -64,13 +58,16 @@ def untyped_storage(self) -> _SyntheticStorage: class _SyntheticBuffer(_SyntheticView): """Buffer whose slices expose identities different from the base buffer.""" - def __init__(self, key: int, view_keys) -> None: - super().__init__(key) - self.view_keys = iter(view_keys) + def __init__(self, view_count: int = 0) -> None: + super().__init__() + self._remaining_views = view_count def __getitem__(self, index) -> _SyntheticView: del index - return _SyntheticView(next(self.view_keys)) + if self._remaining_views <= 0: + raise AssertionError("unexpected view checkout") + self._remaining_views -= 1 + return _SyntheticView() class TestPinnedMemoryPool(unittest.TestCase): @@ -119,7 +116,7 @@ def test_alignment_policy_and_properties(self) -> None: self.assertEqual(pool.align_limit, 1500) def test_exact_capacity_checkout_returns_base_buffer(self) -> None: - buffer = _SyntheticBuffer(100, view_keys=()) + buffer = _SyntheticBuffer() self.mock_platform.alloc_tensor_buffer.side_effect = [buffer] pool = PinnedMemoryPool(1024) @@ -222,16 +219,17 @@ def test_typed_view_release_and_invalid_releases(self) -> None: def test_tracks_storage_identity_of_each_returned_view(self) -> None: """Replace stale storage identities when returning a new view.""" self.mock_platform.alloc_tensor_buffer.side_effect = [ - _SyntheticBuffer(100, view_keys=(101, 102)) + _SyntheticBuffer(view_count=2) ] pool = PinnedMemoryPool(1024) first = pool.acquire(600) + first_key = id(first.untyped_storage()) pool.release(first) second = pool.acquire(600) - self.assertNotIn(101, pool._blocks) # pylint: disable=protected-access - self.assertIn(102, pool._blocks) # pylint: disable=protected-access + self.assertNotIn(first_key, pool._blocks) # pylint: disable=protected-access + self.assertIn(id(second.untyped_storage()), pool._blocks) # pylint: disable=protected-access pool.release(second) def test_allocation_failure_rolls_back_reserved_capacity(self) -> None: diff --git a/tests/ut/core/activation_memory/test_swap.py b/tests/ut/core/activation_memory/test_swap.py index df3bee350b..495a2db5c9 100644 --- a/tests/ut/core/activation_memory/test_swap.py +++ b/tests/ut/core/activation_memory/test_swap.py @@ -29,7 +29,7 @@ SwapManager, SwapTensor, Storage, - _collect_device_storage_ptrs, + _collect_device_storage_ids, _get_cpu_pinned_buf, _return_cpu_pinned_buf, _teardown_wired_swap_layers, @@ -151,17 +151,17 @@ def test_return_cpu_pinned_buf_adds_to_pool(self): self.assertIn("torch.float32", _CPU_PINNED_POOL) self.assertIn(buf, _CPU_PINNED_POOL["torch.float32"]) - def test_collect_device_storage_ptrs_device_tensor(self): - """Test _collect_device_storage_ptrs collects storage ptrs from device tensors.""" + def test_collect_device_storage_ids_device_tensor(self): + """Test _collect_device_storage_ids collects storage identities from device tensors.""" with _swap_plat_patch() as mp: mp.Tensor = torch.Tensor mp.tree_map = _build_mock_platform().tree_map t = torch.empty(10, device="meta") - result = _collect_device_storage_ptrs(t) + result = _collect_device_storage_ids(t) self.assertEqual(len(result), 1) - self.assertIn(t.untyped_storage().data_ptr(), result) + self.assertIn(id(t.untyped_storage()), result) - def test_collect_device_storage_ptrs_unwraps_dtensor(self): + def test_collect_device_storage_ids_unwraps_dtensor(self): """Test storage collection reads the local shard of a DTensor-like object.""" local_tensor = torch.empty(3, device="meta") @@ -176,29 +176,29 @@ def to_local() -> torch.Tensor: with _swap_plat_patch() as mp, patch.object(swap_module, "DTensor", FakeDTensor): mp.Tensor = torch.Tensor mp.tree_map = _build_mock_platform().tree_map - result = _collect_device_storage_ptrs(FakeDTensor()) + result = _collect_device_storage_ids(FakeDTensor()) - self.assertEqual(result, {local_tensor.untyped_storage().data_ptr()}) + self.assertEqual(result, {id(local_tensor.untyped_storage())}) - def test_collect_device_storage_ptrs_nested(self): - """Test _collect_device_storage_ptrs traverses nested structures.""" + def test_collect_device_storage_ids_nested(self): + """Test _collect_device_storage_ids traverses nested structures.""" with _swap_plat_patch() as mp: mp.Tensor = torch.Tensor mp.tree_map = _build_mock_platform().tree_map t1 = torch.empty(10, device="meta") t2 = torch.empty(20, device="meta") nested = {"a": t1, "b": [t2, 1.0]} - result = _collect_device_storage_ptrs(nested) - # Meta tensors share data_ptr=0 so set deduplicates; verify at least 1 found - self.assertGreaterEqual(len(result), 1) + result = _collect_device_storage_ids(nested) + # Storage identity distinguishes meta tensors that data_ptr=0 collapsed. + self.assertEqual(len(result), 2) - def test_collect_device_storage_ptrs_cpu_tensor_ignored(self): - """Test _collect_device_storage_ptrs ignores CPU tensors.""" + def test_collect_device_storage_ids_cpu_tensor_ignored(self): + """Test _collect_device_storage_ids ignores CPU tensors.""" with _swap_plat_patch() as mp: mp.Tensor = torch.Tensor mp.tree_map = _build_mock_platform().tree_map t_cpu = torch.empty(10, device="cpu") - result = _collect_device_storage_ptrs(t_cpu) + result = _collect_device_storage_ids(t_cpu) self.assertEqual(len(result), 0) @@ -293,8 +293,8 @@ def test_protect_if_aliases_match(self): mp.get_element_size = lambda t: t.element_size() t = self._make_tensor(3) st = SwapTensor(t, "test_func") - alias_ptrs = {t.untyped_storage().data_ptr()} - st.protect_if_aliases(alias_ptrs) + alias_ids = {id(t.untyped_storage())} + st.protect_if_aliases(alias_ids) self.assertTrue(st._keep_on_device) def test_protect_if_aliases_no_match(self): @@ -832,23 +832,23 @@ def test_mark_duplicate_swaps_with_duplicate(self): self.assertEqual(count, 1) self.assertTrue(st2._duplicate_swap) - def test_protect_alias_storage_ptrs_no_aliases(self): - """Test protect_alias_storage_ptrs with empty set is no-op.""" + def test_protect_alias_storage_ids_no_aliases(self): + """Test protect_alias_storage_ids with empty set is no-op.""" t = torch.empty(3, device="meta") st = SwapTensor(t, "f1") s = Storage() s["key1"].append(st) - s.protect_alias_storage_ptrs(set()) + s.protect_alias_storage_ids(set()) self.assertFalse(st._keep_on_device) - def test_protect_alias_storage_ptrs_with_alias(self): - """Test protect_alias_storage_ptrs protects aliased tensors.""" + def test_protect_alias_storage_ids_with_alias(self): + """Test protect_alias_storage_ids protects aliased tensors.""" t = torch.empty(3, device="meta") st = SwapTensor(t, "f1") s = Storage() s["key1"].append(st) - alias_ptrs = {t.untyped_storage().data_ptr()} - s.protect_alias_storage_ptrs(alias_ptrs) + alias_ids = {id(t.untyped_storage())} + s.protect_alias_storage_ids(alias_ids) self.assertTrue(st._keep_on_device) def test_launch_load(self): @@ -992,12 +992,12 @@ def test_protect_alias_tensors_keeps_all_output_aliases(self): with patch.object( swap_module, - "_collect_device_storage_ptrs", + "_collect_device_storage_ids", return_value={1, 2, 3}, ): group.protect_alias_tensors(sentinel.output) - storage.protect_alias_storage_ptrs.assert_called_once_with({1, 2, 3}) + storage.protect_alias_storage_ids.assert_called_once_with({1, 2, 3}) def test_collect_packable_tensors_empty(self): """Test _collect_packable_tensors returns 0 when no tensors are packable.""" diff --git a/tests/ut/core/dtensor/test_dtensor_base.py b/tests/ut/core/dtensor/test_dtensor_base.py index ffc9ae4665..add1017e40 100644 --- a/tests/ut/core/dtensor/test_dtensor_base.py +++ b/tests/ut/core/dtensor/test_dtensor_base.py @@ -15,10 +15,17 @@ """Unit tests for :mod:`hyper_parallel.core.dtensor.dtensor_base` — the ``data`` descriptor.""" import unittest +from contextlib import nullcontext from types import SimpleNamespace from unittest.mock import patch +import torch +from torch._subclasses.fake_tensor import FakeTensorMode + +from hyper_parallel.core.dtensor.dtensor import DTensor, SkipDTensorDispatch from hyper_parallel.core.dtensor.dtensor_base import DTensorBase +from hyper_parallel.core.dtensor.layout import Layout +from hyper_parallel.core.dtensor.placement_types import Shard _MODULE = "hyper_parallel.core.dtensor.dtensor_base" @@ -77,5 +84,106 @@ def to_local(self): ) +class TestDTensorMetadata(unittest.TestCase): + """All public numel entry points retain logical global tensor semantics.""" + + def test_global_numel_matches_for_real_and_fake_parameters(self): + """A shard's local capacity must not leak into global metadata queries.""" + layout = Layout((2,), ("dp",), init_backend=False) + layout.set_placements((Shard(0),)) + layout.placement_to_tensor_map(2) + for fake in (False, True): + with self.subTest(fake=fake), FakeTensorMode() if fake else nullcontext(): + local = torch.empty(4, 3) + parameter = torch.nn.Parameter(DTensor.from_local_with_layout(local, layout, shape=(8, 3))) + for skip in (False, True): + with self.subTest(skip=skip), SkipDTensorDispatch() if skip else nullcontext(): + self.assertEqual(parameter.shape, (8, 3)) + self.assertEqual(parameter.size(), (8, 3)) + self.assertEqual(parameter.numel(), 24) + self.assertEqual(torch.numel(parameter), 24) + self.assertEqual(torch.numel(input=parameter), 24) + self.assertEqual(torch.Tensor.numel(parameter), 24) + self.assertEqual(parameter.to_local().numel(), 12) + + +class TestDTensorFakeLocalSupport(unittest.TestCase): + """DTensor wraps FakeTensor locals natively via a storage-less meta shell.""" + + @staticmethod + def _make_layout(): + layout = Layout((2,), ("dp",), init_backend=False) + layout.set_placements((Shard(0),)) + layout.placement_to_tensor_map(2) + return layout + + def test_meta_shell_construction_and_storage_forwarding(self): + """A FakeTensor local yields a meta-shell wrapper answering for the local.""" + layout = self._make_layout() + with FakeTensorMode(): + local = torch.empty(4, 3) + dtensor = DTensor.from_local_with_layout(local, layout, shape=(8, 3)) + self.assertTrue(dtensor.is_meta_shell()) + self.assertIs(dtensor._local_tensor, local) + self.assertIs(dtensor.untyped_storage(), local.untyped_storage()) + self.assertEqual(dtensor.device, local.device) + detached = dtensor.detach() + self.assertIsInstance(detached, DTensor) + self.assertTrue(detached.is_meta_shell()) + parameter = torch.nn.Parameter(dtensor, requires_grad=True) + self.assertIsInstance(parameter, DTensor) + + def test_real_local_is_not_meta_shell(self): + """Real locals keep aliasing the wrapper storage; no shell is involved.""" + layout = self._make_layout() + local = torch.empty(4, 3) + dtensor = DTensor.from_local_with_layout(local, layout, shape=(8, 3)) + self.assertFalse(dtensor.is_meta_shell()) + self.assertIs(dtensor.untyped_storage(), local.untyped_storage()) + + def test_backward_populates_local_grad_through_dispatcher(self): + """Autograd over fake locals lands gradients on the local leaf.""" + layout = self._make_layout() + mode = FakeTensorMode() + with mode: + local = torch.empty(4, 3, requires_grad=True) + dtensor = DTensor.from_local_with_layout(local, layout, shape=(8, 3)) + with mode: + (dtensor * 2).sum().backward() + self.assertIsNotNone(local.grad) + self.assertIsNotNone(dtensor.grad) + + def test_meta_shell_dtensor_grad_assignment_round_trips(self): + """Assigning a meta-shell DTensor gradient stores the local and returns the wrapper.""" + layout = self._make_layout() + with FakeTensorMode(): + local = torch.empty(4, 3, requires_grad=True) + reduced = torch.empty(4, 3) + parameter = DTensor.from_local_with_layout(local, layout, shape=(8, 3)) + grad_dtensor = DTensor.from_local_with_layout(reduced, layout, shape=(8, 3)) + parameter.grad = grad_dtensor + self.assertIs(parameter.grad, grad_dtensor) + self.assertIs(local.grad, reduced) + parameter.grad = None + self.assertIsNone(parameter.grad) + self.assertIsNone(local.grad) + + def test_data_accessors_use_local_payload_for_meta_shell(self): + """``.data`` reads and writes go to the local tensor on meta shells.""" + layout = self._make_layout() + mode = FakeTensorMode() + with mode: + local = torch.empty(4, 3) + replacement = torch.empty(4, 3) + dtensor = DTensor.from_local_with_layout(local, layout, shape=(8, 3)) + self.assertIs( + dtensor.data.untyped_storage(), dtensor._local_tensor.untyped_storage() + ) + dtensor.data = replacement + self.assertIs( + dtensor._local_tensor.untyped_storage(), replacement.untyped_storage() + ) + + if __name__ == "__main__": unittest.main() diff --git a/tests/ut/core/optimizer/test_muon.py b/tests/ut/core/optimizer/test_muon.py index bdead598ee..905928e1f0 100644 --- a/tests/ut/core/optimizer/test_muon.py +++ b/tests/ut/core/optimizer/test_muon.py @@ -15,13 +15,21 @@ """Unit tests for the Muon optimizer.""" +from contextlib import nullcontext from unittest.mock import MagicMock, patch import pytest import torch +from torch._subclasses.fake_tensor import FakeTensorMode +from hyper_parallel.core.dtensor.layout import Layout +from hyper_parallel.core.dtensor.placement_types import Shard, StridedShard +from hyper_parallel.core.optimizer import muon_shard from hyper_parallel.core.optimizer.muon import Muon +from hyper_parallel.core.optimizer.muon_shard import chunk_update_by_layout from hyper_parallel.core.optimizer.optimizer import BaseDistributedOptimizer +from hyper_parallel.core.optimizer.sharding_category import HSDPCommGroup, ParamLayoutSpec, ParamRecord +from hyper_parallel.tools.dryrun.compatibility import install_dryrun_compatibility def _plain_ns_group(**overrides): @@ -151,3 +159,80 @@ def test_async_hierarchical_broadcast_waits_between_mesh_dimensions(): first_work.wait.assert_called_once_with() final_work.wait.assert_not_called() assert handles == [final_work] + + +@pytest.mark.parametrize("fsdp_rank", [0, 20]) +@pytest.mark.parametrize("fake", [False, True]) +def test_muon_metadata_handles_empty_fsdp_shards_before_tp(fsdp_rank, fake): + """FSDP gathers only TP-local rows, including ranks with empty shards.""" + install_dryrun_compatibility() + layout = Layout((64, 2), ("fsdp", "tp"), init_backend=False) + layout.set_placements((StridedShard(0, split_factor=2, uneven_shard=True), Shard(0))) + layout.placement_to_tensor_map(2) + mesh = MagicMock() + mesh.get_coordinate.return_value = (fsdp_rank, 0) + mesh.size.side_effect = lambda dim: (64, 2)[dim] + spec = ParamLayoutSpec(tensor_ndim=2, shard_axes=((0, 0), (1, 0)), replicate_mesh_dims=()) + groups = (MagicMock(), MagicMock()) + with FakeTensorMode() if fake else nullcontext(): + local = torch.empty((1 if fsdp_rank < 16 else 0, 4)) + parameter = MagicMock(data=local, shape=(32, 4)) + parameter.layout = layout + parameter.device_mesh = mesh + parameter._global_shape = (32, 4) + group = HSDPCommGroup(comm_key=None, layout_spec=spec, shard_pgs=groups, + records=[ParamRecord(index=0, param=parameter)]) + with patch("hyper_parallel.core.optimizer.muon_shard.dist.get_world_size", side_effect=[64, 2]), \ + patch("hyper_parallel.core.optimizer.muon_shard.dist.get_rank", side_effect=[fsdp_rank, 0]): + meta = muon_shard.build_param_shard_metadata_for_group(group)[parameter] + assert meta.stage_metas[0].split_sizes == (1,) * 16 + (0,) * 48 + assert meta.stage_metas[1].split_sizes == (16, 16) + local_update = chunk_update_by_layout(torch.empty(32, 4), parameter, spec, meta) + assert local_update.shape == local.shape + + +@pytest.mark.parametrize("fake", [False, True]) +def test_muon_reshape_uses_untyped_storage_identity(fake): + """Views share storage while clones remain distinguishable under FakeTensor.""" + optimizer = Muon.__new__(Muon) + with FakeTensorMode() if fake else nullcontext(): + parameter = torch.nn.Parameter(torch.empty(4, 4)) + parameter.model_name = "weight" + reshape_input = torch.empty(4, 4) + + optimizer.reshape_fn = lambda _, value: [value[:2]] + working, reshaped = optimizer._reshape_ns_input(parameter, reshape_input) + assert reshaped[0].untyped_storage() is working.untyped_storage() + + optimizer.reshape_fn = lambda _, value: [value.clone()] + with pytest.raises(ValueError, match="share storage"): + optimizer._reshape_ns_input(parameter, reshape_input) + + +def test_fake_subgroup_creation_uses_local_rank_membership(): + """Fake process groups create only the subgroup owned by this virtual rank.""" + install_dryrun_compatibility() + optimizer = object.__new__(BaseDistributedOptimizer) + optimizer._split_sub_pg_cache = {} + parent_group = MagicMock() + subgroup = MagicMock() + with patch( + "hyper_parallel.core.optimizer.optimizer.dist.get_process_group_ranks", + return_value=[0, 1, 2, 3], + ), patch( + "hyper_parallel.core.optimizer.optimizer.dist.get_backend", + return_value="fake", + ), patch( + "hyper_parallel.core.optimizer.optimizer.dist.get_rank", + return_value=1, + ), patch( + "hyper_parallel.core.optimizer.optimizer.dist.new_group", + return_value=subgroup, + ) as new_group, patch( + "hyper_parallel.core.optimizer.optimizer.dist.all_gather_into_tensor", + ) as all_gather: + result = optimizer._get_or_create_sub_pg(parent_group, 2) + + assert result is subgroup + new_group.assert_called_once_with(ranks=[0, 1], use_local_synchronization=True) + all_gather.assert_not_called() diff --git a/tests/ut/core/utils/test_simulation.py b/tests/ut/core/utils/test_simulation.py new file mode 100644 index 0000000000..85021f58a5 --- /dev/null +++ b/tests/ut/core/utils/test_simulation.py @@ -0,0 +1,123 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Unit tests for the device-runtime simulation switch and stand-ins.""" +import unittest +import weakref + +import torch + +from hyper_parallel.core.utils.simulation import ( + SimulatedEvent, + SimulatedStream, + hold_for_simulated_stream, + is_device_runtime_simulated, + release_simulated_stream_holds, + simulate_device_runtime, + simulated_accelerator, + simulated_device_handle, +) + + +class TestSimulationSwitch(unittest.TestCase): + """The process-wide switch toggles and reads back.""" + + def tearDown(self): + simulate_device_runtime(False) + + def test_switch_round_trip(self): + """Enabling and disabling the switch is observable.""" + self.assertFalse(is_device_runtime_simulated()) + simulate_device_runtime(True) + self.assertTrue(is_device_runtime_simulated()) + simulate_device_runtime(False) + self.assertFalse(is_device_runtime_simulated()) + + def test_accelerator_lie_round_trip(self): + """The lied accelerator is readable while enabled and cleared after.""" + simulate_device_runtime(True, accelerator="npu") + self.assertEqual(simulated_accelerator(), "npu") + simulate_device_runtime(False) + self.assertIsNone(simulated_accelerator()) + + def test_stream_hold_keeps_tensor_until_wait(self): + """record_stream stand-in retains the staging buffer until a wait. + + A real caching allocator keeps a record_stream'd buffer alive past + the last Python reference until the stream's pending work completes; + the conservative stand-in mirrors that by holding the reference until + the next simulated synchronization point. + """ + simulate_device_runtime(True) + try: + tensor = torch.empty(16) + ref = weakref.ref(tensor) + hold_for_simulated_stream(tensor) + del tensor + self.assertIsNotNone(ref(), "hold must outlive the last reference") + SimulatedEvent().wait() + self.assertIsNone(ref(), "wait must release the hold") + finally: + simulate_device_runtime(False) + + def test_disable_clears_pending_holds(self): + """Turning simulation off never leaks held references across runs.""" + simulate_device_runtime(True) + tensor = torch.empty(8) + ref = weakref.ref(tensor) + hold_for_simulated_stream(tensor) + del tensor + simulate_device_runtime(False) + self.assertIsNone(ref()) + release_simulated_stream_holds() + + +class TestSimulatedPrimitives(unittest.TestCase): + """Stand-ins answer the full stream/event protocol without a device.""" + + def test_event_protocol(self): + """Record/wait/query/synchronize complete immediately.""" + event = SimulatedEvent() + self.assertIs(event.record(), event) + event.record(SimulatedStream()) + event.wait() + event.wait(SimulatedStream()) + self.assertTrue(event.query()) + event.synchronize() + + def test_stream_protocol(self): + """Streams drain immediately and mint elapsed events.""" + stream = SimulatedStream() + stream.synchronize() + self.assertTrue(stream.query()) + stream.wait_event(SimulatedEvent()) + stream.wait_stream(SimulatedStream()) + self.assertTrue(stream.record_event().query()) + + def test_device_handle_namespace(self): + """The handle mirrors the torch. surface swap code touches.""" + handle = simulated_device_handle() + self.assertIsInstance(handle.Stream(), SimulatedStream) + self.assertIsInstance(handle.Event(), SimulatedEvent) + self.assertIs(handle.current_stream(), handle.current_stream()) + with handle.stream(handle.Stream()): + pass + handle.synchronize() + handle.synchronize(0) + with self.assertRaises(AttributeError): + handle.get_device_name() # pylint: disable=no-member + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ut/tools/dryrun/test_compatibility.py b/tests/ut/tools/dryrun/test_compatibility.py new file mode 100644 index 0000000000..e4473db603 --- /dev/null +++ b/tests/ut/tools/dryrun/test_compatibility.py @@ -0,0 +1,77 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Tests for the model-specific environment-patch registry.""" +import unittest + +from hyper_parallel.tools.dryrun import compatibility + + +class TestEnvironmentPatchRegistry(unittest.TestCase): + """Model-specific patches register from plugins, not generic code.""" + + def setUp(self): + self._saved_patches = dict(compatibility._ENVIRONMENT_PATCHES) + self._saved_installed = set(compatibility._ENVIRONMENT_PATCHES_INSTALLED) + + def tearDown(self): + compatibility._ENVIRONMENT_PATCHES.clear() + compatibility._ENVIRONMENT_PATCHES.update(self._saved_patches) + compatibility._ENVIRONMENT_PATCHES_INSTALLED.clear() + compatibility._ENVIRONMENT_PATCHES_INSTALLED.update(self._saved_installed) + + def test_rejects_non_callable_installer(self): + """A patch must be a zero-argument callable.""" + with self.assertRaisesRegex(TypeError, "callable"): + compatibility.register_environment_patch("bad.patch", object()) + + def test_patch_registered_before_install_runs_once_on_install(self): + """install_dryrun_compatibility applies registered patches exactly once.""" + calls = [] + compatibility.register_environment_patch("test.before", lambda: calls.append(1)) + if compatibility._INSTALLED: + # Install already happened in this process: registration itself + # must have applied the patch (late-plugin contract). + self.assertEqual(calls, [1]) + else: + self.assertEqual(calls, []) + compatibility.install_dryrun_compatibility() + self.assertEqual(calls, [1]) + compatibility.install_dryrun_compatibility() + self.assertEqual(calls, [1]) + + def test_patch_registered_after_install_applies_immediately(self): + """A plugin imported after install still takes effect for the process.""" + compatibility.install_dryrun_compatibility() + calls = [] + compatibility.register_environment_patch("test.after", lambda: calls.append(1)) + self.assertEqual(calls, [1]) + compatibility.register_environment_patch("test.after", lambda: calls.append(2)) + # Same name: already-installed patches are not re-applied. + self.assertEqual(calls, [1]) + + def test_deepseek_example_registers_csa2_patch(self): + """The CSA2 patch lives in the DeepSeek example, not generic code.""" + import importlib + source = importlib.import_module( + "hyper_parallel.tools.dryrun.compatibility" + ).__file__ + with open(source, encoding="utf-8") as handle: + self.assertNotIn("shared_compressed_dsa_attention", handle.read()) + import hyper_parallel.tools.dryrun.examples.deepseek_v41 # noqa: F401 pylint: disable=unused-import + self.assertIn("deepseek_v41.csa2", compatibility._ENVIRONMENT_PATCHES) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ut/tools/dryrun/test_memory.py b/tests/ut/tools/dryrun/test_memory.py new file mode 100644 index 0000000000..443e7942c0 --- /dev/null +++ b/tests/ut/tools/dryrun/test_memory.py @@ -0,0 +1,255 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Tests for logical storage accounting and native snapshot compatibility.""" + +import gc +import pickle +import pickletools +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace + +import torch +from torch._subclasses.fake_tensor import FakeTensorMode +from torch.cuda import _memory_viz + +from hyper_parallel.core.dtensor.dtensor import DTensor +from hyper_parallel.core.dtensor.layout import Layout +from hyper_parallel.core.dtensor.placement_types import Shard +from hyper_parallel.tools.dryrun.compatibility import install_dryrun_compatibility +from hyper_parallel.tools.dryrun.memory import LogicalMemoryTracker, write_memory_snapshot + + +class TestLogicalMemoryTracker(unittest.TestCase): + """Validate allocation lifetimes independently of model structure.""" + + def _assert_event_totals(self, snapshot: dict) -> None: + """Replay native events and compare their totals with the logical summary.""" + for device, trace in enumerate(snapshot["device_traces"]): + allocated = {} + current_bytes = 0 + peak_bytes = 0 + for event in trace: + if event["action"] == "alloc": + self.assertNotIn(event["addr"], allocated) + allocated[event["addr"]] = event["size"] + current_bytes += event["size"] + peak_bytes = max(peak_bytes, current_bytes) + elif event["action"] == "free_completed": + self.assertEqual(allocated.pop(event["addr"]), event["size"]) + current_bytes -= event["size"] + totals = snapshot["summary"]["devices"].get(device, {"current_bytes": 0, "peak_bytes": 0}) + self.assertEqual(totals, {"current_bytes": current_bytes, "peak_bytes": peak_bytes}) + self.assertEqual(current_bytes, sum( + segment["allocated_size"] for segment in snapshot["segments"] if segment["device"] == device + )) + + def test_stack_capture_can_be_disabled(self): + """Large simulations can skip Python stack collection.""" + with LogicalMemoryTracker("cpu", capture_stacks=False) as tracker: + tensor = torch.empty(8) + snapshot = tracker.snapshot() + self.assertEqual(snapshot["device_traces"][0][0]["frames"], []) + self.assertEqual(snapshot["segments"][0]["blocks"][0]["frames"], []) + del tensor + + def test_aliases_share_capacity_and_last_reference_releases_storage(self): + """Count an entire storage once, even when only a small view survives.""" + with FakeTensorMode(), LogicalMemoryTracker("cpu") as tracker: + tensor = torch.empty(8) + alias = tensor[:2] + self.assertEqual(tracker.snapshot()["summary"]["current_bytes"], 32) + del tensor + gc.collect() + self.assertEqual(tracker.snapshot()["summary"]["current_bytes"], 32) + del alias + gc.collect() + snapshot = tracker.snapshot() + + self.assertEqual(snapshot["summary"]["current_bytes"], 0) + self.assertEqual(snapshot["summary"]["peak_bytes"], 32) + self._assert_event_totals(snapshot) + self.assertEqual(snapshot["segments"], []) + self.assertEqual([event["action"] for event in snapshot["device_traces"][0]], [ + "segment_alloc", "alloc", "free_requested", "free_completed", "segment_free", + ]) + + def test_resize_to_zero_and_regrow_emit_balanced_lifetimes(self): + """Track FSDP-style capacity release and reuse, including zero-sized seeds.""" + previous_resize = torch.UntypedStorage.resize_ + with FakeTensorMode(), LogicalMemoryTracker("cpu") as tracker: + tensor = torch.empty(8) + storage = tensor.untyped_storage() + storage.resize_(0) + self.assertEqual(tracker.snapshot()["summary"]["current_bytes"], 0) + storage.resize_(64) + self.assertEqual(tracker.snapshot()["summary"]["current_bytes"], 64) + storage.resize_(16) + snapshot = tracker.snapshot() + empty = torch.empty(0) + empty.untyped_storage().resize_(12) + self.assertEqual(tracker.snapshot()["summary"]["current_bytes"], 28) + + self.assertIs(torch.UntypedStorage.resize_, previous_resize) + self.assertEqual(snapshot["summary"]["current_bytes"], 16) + self.assertEqual(snapshot["summary"]["peak_bytes"], 64) + self._assert_event_totals(snapshot) + trace = snapshot["device_traces"][0] + self.assertEqual([event["size"] for event in trace if event["action"] == "alloc"], [32, 64, 16]) + self.assertEqual([event["size"] for event in trace if event["action"] == "free_completed"], [32, 64]) + + def test_seed_inputs_and_pickle_are_compatible_with_native_visualizer(self): + """Preserve existing state and frames in an independently loadable snapshot.""" + with FakeTensorMode(): + tensor = torch.empty(4) + with LogicalMemoryTracker("cpu", device_index=2) as tracker: + tracker.track([tensor, tensor.view(2, 2)]) + snapshot = tracker.snapshot() + temporary = tensor.clone() + del temporary + self.assertEqual(len(snapshot["device_traces"][2]), 2) + + self.assertEqual(snapshot["summary"]["current_bytes"], 16) + self.assertEqual(len(snapshot["segments"]), 1) + self.assertEqual(snapshot["segments"][0]["device"], 2) + self.assertTrue(snapshot["segments"][0]["frames"]) + with tempfile.TemporaryDirectory() as directory: + destination = write_memory_snapshot(snapshot, str(Path(directory) / "memory.pickle")) + with open(destination, "rb") as snapshot_file: + payload = snapshot_file.read() + self.assertEqual(payload[:2], b"\x80\x04") + object_opcodes = {"GLOBAL", "STACK_GLOBAL", "REDUCE", "NEWOBJ", "NEWOBJ_EX", "BUILD"} + self.assertFalse(object_opcodes & {opcode.name for opcode, _, _ in pickletools.genops(payload)}) + snapshot_file.seek(0) + restored = pickle.load(snapshot_file) + self.assertEqual(restored, snapshot) + self._assert_event_totals(restored) + self.assertIn("alloc", _memory_viz.trace(restored)) + self.assertIn(" Tensor") + + +def test_decorator_registers_shape_only_kernel(): + """A user decorator can register a shape-only implementation.""" + registry = DryRunOpRegistry() + + @registry.register_fake(_OP_NAME) + def identity_fake(input_tensor): + return torch.empty_like(input_tensor) + + with FakeTensorMode(): + fake = torch.empty(2, 3) + output = torch.ops.hp_dryrun_test.identity.default(fake) + assert output.shape == fake.shape + assert registry.registered_ops == (_OP_NAME,) + + +def test_unsupported_operator_hint_is_actionable(): + """Unsupported-op diagnostics explain immediate registration.""" + hint = unsupported_operator_hint( + RuntimeError("UnsupportedOperatorException: custom::missing"), + ) + assert hint is not None + assert "register_fake_op" in hint + assert "Import the defining module" in hint + + +def test_public_decorator_targets_global_registry(): + """The package-level shortcut delegates to the process registry.""" + assert callable(register_fake_op) diff --git a/tests/ut/tools/dryrun/test_runner.py b/tests/ut/tools/dryrun/test_runner.py new file mode 100644 index 0000000000..ad25272c32 --- /dev/null +++ b/tests/ut/tools/dryrun/test_runner.py @@ -0,0 +1,432 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Unit tests for the primary LLM dry-run workflow.""" +# pylint: disable=protected-access + +import unittest +from contextlib import nullcontext +from types import SimpleNamespace +from typing import Any, Optional +from unittest.mock import MagicMock, patch + +import torch +from torch.nn import functional +from torch._subclasses.fake_tensor import FakeTensorMode + +from hyper_parallel.models._transformers import model_builder +from hyper_parallel.tools.dryrun.config import DryRunConfig +from hyper_parallel.tools.dryrun.__main__ import _configured_ranks +from hyper_parallel.tools.dryrun.runtime import ( + DryRunBatchMocker, + DryRunRuntime, + ValueDependencyManager, + _DryRunValueProfile, + _MoERoutingValueDependencyRuntime, + derive_tp_target_counts, +) +from hyper_parallel.trainer.config import ( + OptimizerConfig, + Target, + TrainerConfig, + TrainingConfig, +) +from hyper_parallel.tools.dryrun.data import DryRunDataProbe +from hyper_parallel.tools.dryrun.runner import HyperModelsDryRunRunner, _DryRunTrainingBatch + + +def _target(**kwargs: Any) -> dict[str, Any]: + """Provide a callable placeholder for validation-only configurations.""" + return kwargs + + +def _config(dry_run: Optional[DryRunConfig] = None) -> TrainerConfig: + """Build a minimal Trainer configuration for dry-run tests.""" + return TrainerConfig( + model=Target(_target, target_path="tests.model", path="unused"), + optimizer=OptimizerConfig(Target(_target, target_path="tests.optimizer")), + training=TrainingConfig(global_batch_size=1, micro_batch_size=1, backend="gloo"), + dry_run=dry_run or DryRunConfig(), + ) + + +def _named_model(*modules: tuple[str, Any]) -> Any: + """Expose the named-module surface consumed by value-dependency rules.""" + return SimpleNamespace(named_modules=lambda: [("", SimpleNamespace()), *modules]) + + +def _profile(*rules: dict[str, Any]) -> _DryRunValueProfile: + """Build a value-dependency profile from rule dictionaries.""" + return _DryRunValueProfile(SimpleNamespace(value_dependencies={"rules": list(rules)})) + + +class _TinyModel(torch.nn.Module): + """Minimal differentiable model accepted by the dry-run micro-step.""" + + def __init__(self) -> None: + """Create one parameterized projection.""" + super().__init__() + self.projection = torch.nn.Linear(4, 4) + self.config = SimpleNamespace(vocab_size=4) + + def forward(self, inputs: torch.Tensor, use_cache: bool = False) -> Any: + """Return an object exposing a Trainer-compatible loss.""" + del use_cache + return SimpleNamespace(loss=self.projection(inputs).square().mean()) + + +class _KwargLoss(torch.nn.Module): + """Trainer-compatible loss module reading the model-computed loss.""" + + def forward(self, *, model_output: Any, labels: Any = None) -> torch.Tensor: + """Return the loss the tiny model computed.""" + del labels + return model_output.loss + + +class TestDryRunConfiguration(unittest.TestCase): + """Tests for dry-run validation and Trainer entrypoint routing.""" + + def test_validate_accepts_accelerator_targets_and_rejects_disabled_mode(self): + """Accept supported report targets and require dry-run to be enabled.""" + runtime = DryRunRuntime(rank=0, world_size=1, local_rank=0) + for target_device in ("npu", "cuda"): + with self.subTest(target_device=target_device): + runner = HyperModelsDryRunRunner( + _config(DryRunConfig(target_device=target_device)), + runtime, + ) + self.assertEqual(runner._validate_config().target_device, target_device) + + runner = HyperModelsDryRunRunner( + _config(DryRunConfig(enabled=False)), + runtime, + ) + with self.assertRaisesRegex(ValueError, "dry_run.enabled must be true"): + runner._validate_config() + + def test_validate_rejects_invalid_topology_and_unsupported_mutation(self): + """Reject impossible parallel sizes and unsupported model mutation.""" + runtime = DryRunRuntime(rank=0, world_size=1, local_rank=0) + config = _config() + config.accelerator.ep_size = 2 + with self.assertRaisesRegex(ValueError, "expert domain size"): + HyperModelsDryRunRunner(config, runtime)._validate_config() + + config = _config() + config.peft = SimpleNamespace() + with self.assertRaisesRegex(NotImplementedError, "PEFT"): + HyperModelsDryRunRunner(config, runtime)._validate_config() + + # Offload and swap features simulate: their storage lifecycle is + # host-ordered (resize_(0) on wait_offload, resize_ + copy on load), + # so validation accepts them. + config = _config() + config.activation_swap = "attention" + config.fsdp_config.enable_offload = True + config.optimizer.swap.enabled = True + HyperModelsDryRunRunner(config, runtime)._validate_config() + + config = _config() + config.accelerator.pp_size = 2 + with self.assertRaisesRegex( + NotImplementedError, + "does not support pipeline parallelism", + ): + HyperModelsDryRunRunner( + config, + DryRunRuntime(rank=0, world_size=2, local_rank=0), + )._validate_config() + + def test_validate_normalizes_yaml_mapping(self): + """The trainer config keeps dry-run as a generic mapping until validation.""" + runtime = DryRunRuntime(rank=0, world_size=1, local_rank=0) + config = _config() + config.dry_run = { + "enabled": True, + "target_device": "npu", + "ranks": [0], + "virtual_world_size": 1, + "snapshot": { + "enabled": True, + "path": "outputs/dry_run", + "capture_stacks": False, + "stack_depth": 8, + }, + } + runner = HyperModelsDryRunRunner(config, runtime) + dry_run = runner._validate_config() + self.assertIsInstance(dry_run, DryRunConfig) + self.assertFalse(dry_run.snapshot.capture_stacks) + self.assertEqual(dry_run.snapshot.stack_depth, 8) + self.assertEqual( + runner._snapshot_path(dry_run, runtime), + "outputs/dry_run/rank_0/memory.pickle", + ) + + def test_validate_rejects_rank_outside_virtual_world(self): + """Configured ranks must exist in the logical topology.""" + runtime = DryRunRuntime(rank=0, world_size=2, local_rank=0) + config = _config(DryRunConfig(ranks=[0, 2], virtual_world_size=2)) + with self.assertRaisesRegex(ValueError, "dry_run.ranks"): + HyperModelsDryRunRunner(config, runtime)._validate_config() + + def test_cli_rank_defaults_and_validation(self): + """CLI rank selection defaults to rank zero and rejects invalid entries.""" + self.assertEqual(_configured_ranks(DryRunConfig(), 8), [0]) + self.assertEqual(_configured_ranks(DryRunConfig(ranks=[0, 7]), 8), [0, 7]) + with self.assertRaisesRegex(ValueError, "dry_run.ranks"): + _configured_ranks(DryRunConfig(ranks=[8]), 8) + + +class TestDryRunModelAndData(unittest.TestCase): + """Tests for meta-model creation and normal-data reuse.""" + + def test_dry_run_builds_the_configured_target_with_distributed_setup(self): + """Use arbitrary configured factories instead of reparsing Hugging Face paths.""" + captured = {} + + def from_config_factory(distributed_setup: Any = None, **kwargs: Any) -> torch.nn.Module: + """Capture build options and create a deferred meta model.""" + captured.update(kwargs) + captured["distributed_setup"] = distributed_setup + captured["deferred"] = model_builder.is_model_materialization_deferred() + model = torch.nn.Linear(2, 2, device="meta") + model.config = SimpleNamespace(vocab_size=8) + return model + + config = _config() + config.model = Target( + from_config_factory, + target_path="tests.from_config_factory", + config_path="config-only-model", + ) + setup = SimpleNamespace(marker="setup") + model = HyperModelsDryRunRunner(config, DryRunRuntime(0, 1, 0))._build_target_model(setup) + + self.assertEqual(model.config.vocab_size, 8) + self.assertIs(captured["distributed_setup"], setup) + self.assertEqual(captured["config_path"], "config-only-model") + self.assertTrue(captured["deferred"]) + self.assertFalse(model_builder.is_model_materialization_deferred()) + + def test_data_probe_reuses_trainer_build_and_reads_first_batch(self): + """Read one batch through the normal Trainer data construction sequence.""" + labels = torch.tensor([[0, 1, -100, 3]]) + loss_mask = torch.tensor([[1, 1, 0, 1]]) + base = MagicMock() + base.train_dataloader = [object()] + base.get_batch.return_value = ( + {"input_ids": torch.tensor([[0, 1, 2, 3]])}, + {"labels": labels, "loss_mask": loss_mask}, + ) + probe = DryRunDataProbe(base) + + with ( + patch.object(type(probe._trainer), "_build_model_assets"), + patch.object(type(probe._trainer), "_build_data_transform"), + patch.object(type(probe._trainer), "_build_collate_fn"), + patch.object(type(probe._trainer), "_build_get_batch"), + ): + probe.build() + prepared = probe.read_first_batch() + + self.assertEqual(prepared.token_counts["foundation_tokens"], 2) + self.assertIs(prepared.loss_inputs["labels"], labels) + + def test_batch_mocker_preserves_structure_and_tp_label_statistics(self): + """Erase tensor values while retaining batch structure and label counts.""" + labels = torch.tensor([[0, 1, 6, -100]]) + loss_inputs = { + "labels": labels, + "loss_mask": torch.tensor([[1, 1, 0]]), + } + valid_count, owned = derive_tp_target_counts(loss_inputs, vocab_size=8, tp_size=2) + + with FakeTensorMode() as fake_mode: + mocked = DryRunBatchMocker(fake_mode, torch.device("cpu")).mock({ + "tokens": torch.ones(1, 4, dtype=torch.int64), + "nested": [torch.ones(2, dtype=torch.float32)], + }) + + self.assertEqual(valid_count, 2) + self.assertEqual(owned, (1, 1)) + self.assertEqual(mocked["tokens"].shape, (1, 4)) + self.assertEqual(mocked["tokens"].dtype, torch.int64) + self.assertEqual(mocked["nested"][0].shape, (2,)) + + +class TestDryRunValueDependencies(unittest.TestCase): + """Tests for value-dependent branches used by LLM dry-run.""" + + def test_moe_routing_supports_balanced_and_explicit_loads(self): + """Produce deterministic EP traffic for common routing profiles.""" + experts = SimpleNamespace(w1=None, w2=None, w3=None) + module = SimpleNamespace(num_experts=4, top_k=2, experts=experts, gate=SimpleNamespace()) + model = _named_model(("mlp", module)) + + automatic = _profile() + automatic.bind_model(model) + automatic_plan = automatic.moe_routing_plan("mlp", module, 2, 0, 4, 2) + self.assertEqual(automatic_plan.input_split_sizes, (4, 4)) + + balanced = _profile({"match": "mlp", "handler": "moe_routing", "path": "balanced"}) + balanced.bind_model(model) + balanced_plan = balanced.moe_routing_plan("mlp", module, 2, 0, 4, 2) + self.assertEqual(balanced_plan.input_split_sizes, (4, 4)) + + explicit = _profile({ + "match": "mlp", + "handler": "moe_routing", + "path": "explicit", + "inputs": {"source_expert_loads": [[4, 4, 0, 0], [0, 0, 4, 4]]}, + }) + explicit.bind_model(model) + explicit_plan = explicit.moe_routing_plan("mlp", module, 2, 1, 4, 2) + self.assertEqual(explicit_plan.input_split_sizes, (0, 8)) + self.assertEqual(explicit_plan.output_split_sizes, (0, 8)) + + def test_local_moe_replay_accepts_model_specific_keyword_arguments(self): + """Ignore keyword-only model context that does not change routed shapes.""" + module = torch.nn.Module() + module.experts = SimpleNamespace(w1=None, w2=None, w3=None) + module.gate = SimpleNamespace() + model = torch.nn.Module() + model.mlp = module + profile = _profile({"match": "mlp", "handler": "moe_routing", "path": "balanced"}) + profile.bind_model(model) + base = SimpleNamespace(model=model, mesh=SimpleNamespace(ep_size=1, ep_rank=0)) + runtime = _MoERoutingValueDependencyRuntime(profile, base) + expected = torch.empty(1, 2, 4) + runtime._run_moe_forward = MagicMock(return_value=expected) + + self.assertEqual(runtime._ep_group_identity(), (1, 0)) + with runtime: + actual = module(torch.empty(1, 2, 4), input_ids=torch.ones(1, 2, dtype=torch.long)) + + self.assertIs(actual, expected) + runtime._run_moe_forward.assert_called_once() + + def test_eager_expert_replay_matches_per_expert_packed_swiglu(self): + """Replay the non-grouped EP expert path with independent expert activations.""" + experts = SimpleNamespace( + _ep_apply_gate=lambda gate_up: functional.silu(gate_up.chunk(2, dim=-1)[0]) + * gate_up.chunk(2, dim=-1)[1] + ) + gate_up = torch.randn(2, 6, 4) + down = torch.randn(2, 4, 3) + values = torch.randn(5, 4) + loads = (2, 3) + + actual = _MoERoutingValueDependencyRuntime._run_eager_experts( + experts, + "packed_gate_up", + (gate_up, down), + values, + loads, + ) + expected = [] + start = 0 + for expert, load in enumerate(loads): + rows = values[start:start + load] + packed = functional.linear(rows, gate_up[expert]) + gate, up = packed.chunk(2, dim=-1) + expected.append(functional.linear(functional.silu(gate) * up, down[expert])) + start += load + + torch.testing.assert_close(actual, torch.cat(expected)) + +class TestDryRunExecution(unittest.TestCase): + """Tests for FakeTensor execution, memory accounting, and reporting.""" + + def test_fake_step_runs_forward_backward_optimizer_and_builds_report(self): + """Drive the real train_step over FakeTensors and report memory.""" + from hyper_parallel.trainer.base import BaseTrainer + from hyper_parallel.trainer.runtime.model_integration import ( + DisabledModelIntegrationSession, + ) + + runner = HyperModelsDryRunRunner(_config(), DryRunRuntime(0, 1, 0)) + runner._simulation_device = "cpu" + runner._target_device = "cuda" + profile = _profile() + + with FakeTensorMode(allow_non_fake_inputs=True): + model = _TinyModel() + profile.bind_model(model) + labels = torch.zeros(1, 4, dtype=torch.long) + batch = _DryRunTrainingBatch( + model_inputs={"inputs": torch.empty(1, 4)}, + loss_inputs={"labels": labels}, + token_counts={"foundation_tokens": 4}, + valid_token_count=0, + target_tokens_per_rank=(0,), + ) + base = BaseTrainer.__new__(BaseTrainer) + base.config = runner.config + base.local_rank = 0 + base.global_rank = 0 + base.world_size = 1 + base.device = torch.device("cpu") + base.model = model + base.model_config = model.config + base.model_parts = [model] + base.hsdp_model_parts = [] + base.mesh = SimpleNamespace( + dp_size=1, + dp_replicate_size=1, + dp_shard_size=1, + tp_size=1, + cp_size=1, + ep_size=1, + pp_size=1, + tp_rank=0, + dp_rank=0, + loss_parallel=False, + sequence_parallel=False, + dp_cp_mesh=None, + device_mesh=None, + ) + base.dp_cp_mesh = None + base.device_mesh = None + base.loss_fn = _KwargLoss() + base.optimizer = torch.optim.SGD(model.parameters(), lr=0.1, foreach=False) + base.lr_scheduler = None + base.graph_compiler = None + base.num_micro_batches = 1 + base.model_integration = DisabledModelIntegrationSession() + base.fsdp_runtime_diagnostics = None + base.train_dataloader = None + base._build_training_context() + base._init_callbacks() + dependencies = ValueDependencyManager(profile, model, DryRunRuntime(0, 1, 0)) + report = runner._execute_step(base, batch, profile, dependencies) + + self.assertEqual(report["metadata"]["num_micro_batches"], 1) + self.assertEqual(report["metadata"]["parallel"]["dp"], 1) + self.assertGreater(report["summary"]["peak_bytes"], 0) + self.assertEqual(report["metadata"]["target_device"], "cuda") + self.assertIs(type(report["metadata"]["torch_version"]), str) + self.assertTrue(report["device_traces"][0]) + # Model parameters (64B weight + 16B bias seeds, 32B+16B optimizer + # momentum) survive the step; the real train_step additionally keeps + # its scalar bookkeeping (token counts, loss accumulators) alive. + # Assert the parameter seeds remain and transient activations do not. + self.assertGreaterEqual(report["summary"]["current_bytes"], 96) + self.assertLess(report["summary"]["current_bytes"], 512) + self.assertNotIn("modules", report) + self.assertNotIn("peak_breakdown_bytes", report["summary"]) + +if __name__ == "__main__": + unittest.main()