Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 44 additions & 5 deletions hyper_parallel/core/activation_memory/_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
"get_stream_context",
"get_current_stream",
"new_event",
"record_stream",
"no_grad",
"preserve_version_counter",
"cat",
Expand All @@ -37,26 +38,58 @@
from torch import Tensor

from hyper_parallel.core.utils.communication import get_device_handle
from hyper_parallel.core.utils.simulation import (
hold_for_simulated_stream,
is_device_runtime_simulated,
simulated_device_handle,
)


def _stream_owner():
"""Return the namespace owning streams and events for this process.

Under logical simulation the process never binds an accelerator, so stream
and event primitives come from the simulated handle: every transfer then
completes in host program order, which preserves the storage lifecycle the
swap state machine expresses while dropping only device-side overlap.
"""
if is_device_runtime_simulated():
return simulated_device_handle()
return get_device_handle()


def new_stream():
"""Create a new device stream on the current accelerator."""
return get_device_handle().Stream()
return _stream_owner().Stream()


def get_stream_context():
"""Return the stream context manager (``torch.npu.stream`` / ``torch.cuda.stream``)."""
return get_device_handle().stream
return _stream_owner().stream


def get_current_stream():
"""Return the current device stream."""
return get_device_handle().current_stream()
return _stream_owner().current_stream()


def new_event():
"""Create a new device event on the current accelerator."""
return get_device_handle().Event()
return _stream_owner().Event()


def record_stream(tensor, stream) -> None:
"""Reserve ``tensor``'s storage for ``stream`` until its pending work ends.

Under simulation the reference is held until the next simulated wait —
the same point a real caching allocator could first reclaim it — so the
logical lifetime of staging buffers is conservative, never shorter than
the real one.
"""
if is_device_runtime_simulated():
hold_for_simulated_stream(tensor)
return
tensor.record_stream(stream)


def no_grad():
Expand All @@ -76,6 +109,10 @@ def cat(tensors, dim=0):

def empty_like(tensor, *, dtype=None, device=None, pin_memory=False):
"""Allocate an uninitialized tensor shaped like *tensor*."""
if pin_memory and is_device_runtime_simulated():
# Pinning needs a live device runtime; logically it only affects DMA
# speed, never which host bytes are allocated.
pin_memory = False
return torch.empty_like(tensor, dtype=dtype, device=device, pin_memory=pin_memory)


Expand All @@ -91,8 +128,10 @@ def get_element_size(tensor) -> int:

def alloc_tensor_buffer(numel: int, dtype, device="cpu", pin_memory: bool = False):
"""Allocate an uninitialized 1-D tensor buffer."""
if pin_memory:
if pin_memory and not is_device_runtime_simulated():
return torch.empty(numel, dtype=dtype, device="cpu", pin_memory=True)
if pin_memory:
return torch.empty(numel, dtype=dtype, device="cpu")
return torch.empty(numel, dtype=dtype, device=device)


Expand Down
10 changes: 8 additions & 2 deletions hyper_parallel/core/activation_memory/pinned_memory_pool.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,15 @@ def _valid_positive_int(value: int, name: str) -> None:


def _storage_key(tensor: Any) -> int:
"""Return a stable identity for a tensor's underlying storage."""
"""Return a stable identity for a tensor's underlying storage.

Identity of the cached storage PyObject, not ``data_ptr()``: the pool's
buffers are unpinned CPU tensors under logical simulation and would all
report pointer 0. The pool owns every underlying buffer for its whole
lifetime, so the id cannot be recycled while a key is registered.
"""
try:
return int(tensor.untyped_storage().data_ptr())
return id(tensor.untyped_storage())
except (AttributeError, TypeError, ValueError) as exc:
raise ValueError("release() expects a tensor with an identifiable storage.") from exc

Expand Down
49 changes: 32 additions & 17 deletions hyper_parallel/core/activation_memory/swap.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,18 +76,25 @@ def _return_cpu_pinned_buf(buf):
_CPU_PINNED_POOL[str(buf.dtype)].append(buf)


def _collect_device_storage_ptrs(tensors: Any) -> Set[int]:
"""Collect device storage pointers from a nested tensor structure."""
storage_ptrs = set()
def _collect_device_storage_ids(tensors: Any) -> Set[int]:
"""Collect device storage identities from a nested tensor structure.

Identity of the cached storage PyObject, not ``data_ptr()``: empty tensors
and FakeTensors report pointer 0 for distinct storages, which would make
every such tensor look like an alias of the protected set. The ids stay
valid while the owning tensors are alive, which the caller guarantees for
the duration of one protect pass.
"""
storage_ids = set()

def _collect(x):
local_tensor = x.to_local() if isinstance(x, DTensor) else x
if isinstance(local_tensor, _backend.Tensor) and str(local_tensor.device).lower() != "cpu":
storage_ptrs.add(local_tensor.untyped_storage().data_ptr())
storage_ids.add(id(local_tensor.untyped_storage()))
return x

_backend.tree_map(_collect, tensors)
return storage_ptrs
return storage_ids


class SwapTensor:
Expand Down Expand Up @@ -122,13 +129,21 @@ def __init__(self, val: Any, funcname: str, group_swap: bool = False, cpu_pool=N
self.storage_size = 0

def dedup_key(self):
"""Return a stable identity key for duplicate-swap detection."""
"""Return a stable identity key for duplicate-swap detection.

Keyed by the storage PyObject's identity rather than ``data_ptr()``:
the pointer is 0 for every FakeTensor and empty storage, which would
collapse distinct registrations into false duplicates. The SwapTensor
keeps ``self.val`` (and with it the storage object) alive for the
whole group lifecycle, so the id cannot be recycled while the key is
in a group's seen-set.
"""
if self._state == self.STATE_NON_TENSOR:
return None
val_storage = self.val.untyped_storage()
return (
str(self.val.device),
val_storage.data_ptr(),
id(val_storage),
self.val.storage_offset(),
val_storage.size(),
tuple(self.val.stride()),
Expand All @@ -138,11 +153,11 @@ def mark_duplicate_swap(self) -> None:
"""Mark this wrapper as a duplicate registration in the same swap group."""
self._duplicate_swap = True

def protect_if_aliases(self, alias_storage_ptrs: Set[int]) -> None:
def protect_if_aliases(self, alias_storage_ids: Set[int]) -> None:
"""Keep tensors that alias externally-owned tensors on device."""
if self._state == self.STATE_NON_TENSOR:
return
if self.val.untyped_storage().data_ptr() in alias_storage_ptrs:
if id(self.val.untyped_storage()) in alias_storage_ids:
self._keep_on_device = True

def get_val(self) -> Any:
Expand Down Expand Up @@ -394,14 +409,14 @@ def mark_duplicate_swaps(self, seen_keys) -> int:
seen_keys.add(dedup_key)
return duplicate_count

def protect_alias_storage_ptrs(self, alias_storage_ptrs: Set[int]):
def protect_alias_storage_ids(self, alias_storage_ids: Set[int]):
"""Avoid offloading swap entries that alias externally-owned storage."""
if not alias_storage_ptrs:
if not alias_storage_ids:
return

def _protect_tensor(x):
if isinstance(x, SwapTensor):
x.protect_if_aliases(alias_storage_ptrs)
x.protect_if_aliases(alias_storage_ids)
return x

for storage_list in self.values():
Expand Down Expand Up @@ -501,7 +516,7 @@ def __init__(self, group_name: str):
self._group_device_buf = None # temp device bufs; cleared after each phase
# Persistent dedup set accumulated across add() calls; avoids O(N²) rebuild.
# mark_duplicate_swaps mutates it in-place, so new keys are added automatically.
# Reset at wait_load() so stale data_ptrs don't leak into the next iteration.
# Reset at wait_load() so stale storage ids don't leak into the next iteration.
self._seen_dedup_keys: set = set()
# Per-bucket SwapTensor lists built in _collect_packable_tensors and consumed
# in launch_offload, eliminating a redundant pass over _packed_tensor_info.
Expand All @@ -518,11 +533,11 @@ def add(self, storage):

def protect_alias_tensors(self, tensors: Any):
"""Protect externally-owned tensors from premature offload."""
alias_storage_ptrs = _collect_device_storage_ptrs(tensors)
if not alias_storage_ptrs:
alias_storage_ids = _collect_device_storage_ids(tensors)
if not alias_storage_ids:
return
for storage in self._storages:
storage.protect_alias_storage_ptrs(alias_storage_ptrs)
storage.protect_alias_storage_ids(alias_storage_ids)

@staticmethod
def _bucket_tensor(candidate_buckets, dtype_key, x):
Expand Down Expand Up @@ -658,7 +673,7 @@ def _protect_device_bufs(group_device_bufs, copy_stream):
for buf in group_device_bufs.values():
device = getattr(buf, "device", None)
if device is not None and device.type not in ("cpu", "meta"):
buf.record_stream(copy_stream)
_backend.record_stream(buf, copy_stream)

def _acquire_bucket_cpu_buf(self, bucket_key, bucket):
"""Acquire the pinned CPU buffer that receives one bucket's D2H copy."""
Expand Down
Loading
Loading