Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 22 additions & 3 deletions hyper_parallel/compile/compiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,13 @@ def is_compiled(self) -> bool:
"""Whether a joint graph has already been compiled."""
return self._joint_graph is not None

@property
def execution_backend(self) -> Optional[str]:
"""Actual terminal backend mode; None until a graph is ready."""
if self._joint_graph is None:
return None
return getattr(self._joint_graph.graph_module, "meta", {}).get("inductor_compiled", "fx")

@property
def parameter_layouts(self) -> Optional[dict]:
"""Final parameter-identity layouts, available after lazy compilation.
Expand Down Expand Up @@ -254,13 +261,25 @@ def compile(self, **inputs: Any) -> None:
if ac_scope is not None:
pass_kwargs["ac_scope"] = ac_scope

# Passes mutate ``graph_module`` in place and return it, so the
# transformed graph lives on ``joint_graph`` for ``forward_backward``.
pipeline.run(joint_graph.graph_module, **pass_kwargs)
# A pass may replace the module, e.g. when installing an executable
# backend wrapper. Keep its returned graph as the runtime executable.
runtime_graph = pipeline.run(joint_graph.graph_module, **pass_kwargs)
if getattr(runtime_graph, "meta", {}).get("inductor_compiled") == "unavailable":
raise RuntimeError("The requested joint-graph Inductor backend is unavailable in this torch build")
for name in (
"state_fqns", "state_is_param", "num_state_inputs", "user_inputs_spec",
"loss_dict_keys", "fsdp_sharded_param_names", "meta",
):
if not hasattr(runtime_graph, name) and hasattr(joint_graph.graph_module, name):
setattr(runtime_graph, name, getattr(joint_graph.graph_module, name))
joint_graph.graph_module = runtime_graph

if joint_graph.input_guards is not None:
joint_graph.input_guards.refresh()
if joint_graph.state_guards is not None:
joint_graph.state_guards.refresh(self.model)
self._joint_graph = joint_graph
_LOG.info("Joint graph ready: execution_backend=%s", self.execution_backend)

def forward_backward(self, **inputs: Any) -> Any:
"""
Expand Down
4 changes: 4 additions & 0 deletions hyper_parallel/compile/pass_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -569,6 +569,10 @@ def build_pass_config_from_trainer_config(
compile_config, "fsdp_overlap_grad_reduce", True
),
max_in_flight_gb=getattr(compile_config, "max_in_flight_gb", 0.25),
fsdp_bucketing=getattr(compile_config, "fsdp_bucketing", False),
fsdp_bucket_modules=getattr(compile_config, "fsdp_bucket_modules", None),
inductor_mode=getattr(compile_config, "inductor_mode", "off"),
inductor_configs=getattr(compile_config, "inductor_configs", None),
ac_mode=ac_mode,
ac_layer_count=ac_layer_count,
ac_layer_indices=ac_layer_indices,
Expand Down
7 changes: 6 additions & 1 deletion hyper_parallel/compile/passes/inductor_pass.py
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,7 @@ def run(
"JointGraph.example_inputs) to size compiled regions"
)
if _regional_inductor_available():
self._compile_graph(graph_module, pass_config, example_inputs)
graph_module = self._compile_graph(graph_module, pass_config, example_inputs)
compiled = True
else:
self._warn_unavailable(mode)
Expand Down Expand Up @@ -417,6 +417,11 @@ def _compile_graph(
_propagate_fake_vals(graph_module)
if example_inputs is None:
example_inputs = _placeholder_fake_args(graph_module)
else:
# Partitioning may have changed the state storage since tracing.
state_count = getattr(graph_module, "num_state_inputs", 0)
state_inputs = tuple(_placeholder_fake_args(graph_module)[:state_count])
example_inputs = state_inputs + tuple(example_inputs[state_count:])
_LOG.info(
"InductorPass: compiling graph in %r mode (%d placeholders, %d nodes)",
mode,
Expand Down
9 changes: 9 additions & 0 deletions hyper_parallel/compile/passes/parallel/fsdp_pass.py
Original file line number Diff line number Diff line change
Expand Up @@ -439,6 +439,15 @@ def run(
len(param_nodes),
)

# Backend propagation starts from runtime shard inputs, rather than the
# full logical tensors used to capture the forward/backward operators.
parameters = dict(model.named_parameters())
for node in param_nodes:
value = node.meta.get("val")
parameter = parameters.get(node.meta.get("param_name"))
if isinstance(value, torch.Tensor) and parameter is not None:
node.meta["val"] = value.new_empty(parameter.shape, requires_grad=value.requires_grad)

graph_module.recompile()
return graph_module

Expand Down
50 changes: 48 additions & 2 deletions hyper_parallel/compile/tracer/dynamic_shapes.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,10 +122,11 @@ class InputGuards:
No real input tensors or storage are retained here.
"""

def __init__(self, inputs: list[Any], fake_inputs: tuple[Any, ...], shape_env: ShapeEnv) -> None:
def __init__(self, inputs: list[Any], fake_inputs: tuple[Any, ...], shape_env: ShapeEnv | None) -> None:
self.shape_env = shape_env
self.fake_tensors = [value for value in fake_inputs if isinstance(value, torch.Tensor)]
self.leaves = [self._metadata(value) for value in inputs]
self.static_geometry = None if shape_env is not None else self._geometry(inputs)
self.aliases = self._aliases(inputs)
self.expression = None
self.refresh()
Expand All @@ -144,20 +145,65 @@ def _aliases(inputs: list[Any]) -> list[int]:
for index, value in enumerate(inputs) if isinstance(value, torch.Tensor)
]

@staticmethod
def _geometry(inputs: list[Any]) -> list[tuple]:
return [
(tuple(value.shape), tuple(value.stride()) if value.layout == torch.strided else None,
value.storage_offset() if value.layout == torch.strided else None)
for value in inputs if isinstance(value, torch.Tensor)
]

def refresh(self) -> None:
"""Include constraints introduced by tracing and graph passes."""
self.expression = self.shape_env.produce_guards_expression(self.fake_tensors, ignore_static=False)
if self.shape_env is not None:
self.expression = self.shape_env.produce_guards_expression(self.fake_tensors, ignore_static=False)

def validate(self, inputs: list[Any]) -> None:
"""Reject incompatible inputs before any graph or collective is executed."""
if [self._metadata(value) for value in inputs] != self.leaves:
raise ValueError("Graph input rank, dtype, device, layout, requires_grad or Python constant changed")
if self._aliases(inputs) != self.aliases:
raise ValueError("Graph input tensor aliasing changed since compilation")
if self.static_geometry is not None and self._geometry(inputs) != self.static_geometry:
raise ValueError("Static graph input shape/stride changed; compile again for the new inputs")
tensors = [value for value in inputs if isinstance(value, torch.Tensor)]
if self.expression and not self.shape_env.evaluate_guards_expression(self.expression, tensors):
raise ValueError(
"Graph input shape/stride violates the traced shape guards. Check unmarked static dimensions, "
"related input sizes and shape-dependent branches. Compile with representative dynamic sizes > 1; "
"sizes 0/1 and Python shape branches may specialize the graph."
)


class ModelStateGuards:
"""Guard graph-specialized module modes and storage metadata, allowing value updates."""

def __init__(self, model: torch.nn.Module) -> None:
self.signature = self._signature(model)

@staticmethod
def _signature(model: torch.nn.Module) -> tuple:
"""Snapshot module identities, state roles and storage geometry without retaining values."""
modes = tuple((name, id(module), type(module), module.training) for name, module in model.named_modules())
state = [
(kind, name, value)
for kind, entries in (("parameter", model.named_parameters(remove_duplicate=False)),
("buffer", model.named_buffers(remove_duplicate=False)))
for name, value in entries
]
tensors = tuple(
(kind, name, id(value), tuple(value.shape), value.dtype, value.device, value.layout, value.requires_grad,
tuple(value.stride()) if value.layout == torch.strided else None,
value.storage_offset() if value.layout == torch.strided else None)
for kind, name, value in state
)
return modes, tensors

def refresh(self, model: torch.nn.Module) -> None:
"""Record the final storage contract after graph partitioning."""
self.signature = self._signature(model)

def validate(self, model: torch.nn.Module) -> None:
"""Reject stale captures before running graph operators or collectives."""
if self._signature(model) != self.signature:
raise ValueError("Graph model state or training mode changed; compile again before execution")
11 changes: 7 additions & 4 deletions hyper_parallel/compile/tracer/graph_tracer.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@
from torch.fx.traceback import preserve_node_meta
from torch.nn.utils import stateless

from .dynamic_shapes import InputGuards, build_symbolic_inputs
from .dynamic_shapes import InputGuards, ModelStateGuards, build_symbolic_inputs

# Older torch's make_fx predates record_stack_traces / record_module_stack
# and rejects unknown kwargs; forward only the kwargs this torch supports.
Expand Down Expand Up @@ -180,6 +180,7 @@ class JointGraph:
state_fqns: List[str]
example_inputs: tuple
input_guards: InputGuards | None = None
state_guards: ModelStateGuards | None = None


def extract_module_state(mod: nn.Module) -> Dict[str, torch.Tensor]:
Expand Down Expand Up @@ -647,10 +648,10 @@ def trace_model_graph( # pylint: disable=too-many-locals
num_layers=num_layers,
state_fqns=state_fqns,
example_inputs=fake_args,
input_guards=(
InputGuards(user_inputs_flat, fake_args[num_state_inputs:], fake_mode.shape_env)
if symbolic else None
input_guards=InputGuards(
user_inputs_flat, fake_args[num_state_inputs:], fake_mode.shape_env if symbolic else None
),
state_guards=ModelStateGuards(model),
)


Expand Down Expand Up @@ -686,6 +687,8 @@ def run_traced_graph(
f" Traced: {joint_graph.state_fqns}\n"
f" Got: {list(model_state.keys())}"
)
if joint_graph.state_guards is not None:
joint_graph.state_guards.validate(model)

user_flat, user_spec = torch.utils._pytree.tree_flatten(inputs)
traced_spec = getattr(joint_graph.graph_module, "user_inputs_spec", None)
Expand Down
22 changes: 21 additions & 1 deletion hyper_parallel/models/build_options.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,11 @@ class CompileConfig:
fsdp_allgather_prefetch: bool = True
fsdp_overlap_grad_reduce: bool = True
max_in_flight_gb: float = 0.25
# Joint-graph backend options; decoder torch.compile uses backend/mode/options above.
inductor_mode: Literal["off", "regional", "full"] = "off"
inductor_configs: Optional[dict[str, Any]] = None
fsdp_bucketing: bool = False
fsdp_bucket_modules: Optional[list[str | list[str]]] = None

def selects_graph_compiler(self) -> bool:
"""Return whether this config selects the graph-compiler trainer path."""
Expand All @@ -147,12 +152,13 @@ def __post_init__(self) -> None:
"""Validate values that the YAML resolver cannot express precisely."""
boolean_fields = (
"enabled", "use_joint_graph", "fullgraph", "dynamic", "fsdp_allgather_prefetch",
"fsdp_overlap_grad_reduce",
"fsdp_overlap_grad_reduce", "fsdp_bucketing",
)
for name in boolean_fields:
if not isinstance(getattr(self, name), bool):
raise TypeError(f"compile.{name} must be a bool")
self._validate_max_in_flight_gb()
self._validate_joint_graph_options()
if self.dynamic_arg_dims is not None and not isinstance(self.dynamic_arg_dims, dict):
raise ValueError("compile.dynamic_arg_dims must be a mapping or None")
if not isinstance(self.mode, str) or not self.mode.strip():
Expand Down Expand Up @@ -185,6 +191,20 @@ def _validate_max_in_flight_gb(self) -> None:
):
raise ValueError("compile.max_in_flight_gb must be a finite non-negative number")

def _validate_joint_graph_options(self) -> None:
"""Validate the backend and bucketing options consumed by graph passes."""
if self.inductor_mode not in ("off", "regional", "full"):
raise ValueError("compile.inductor_mode must be 'off', 'regional' or 'full'")
if self.inductor_configs is not None and (
not isinstance(self.inductor_configs, dict)
or any(not isinstance(key, str) for key in self.inductor_configs)
):
raise ValueError("compile.inductor_configs must be a mapping with string keys or None")
if self.inductor_configs and self.inductor_mode == "off":
raise ValueError("compile.inductor_configs requires an enabled compile.inductor_mode")
if self.fsdp_bucket_modules is not None and not isinstance(self.fsdp_bucket_modules, list):
raise ValueError("compile.fsdp_bucket_modules must be a list or None")


@dataclass
class FSDP2MixedPrecisionConfig:
Expand Down
14 changes: 14 additions & 0 deletions tests/ut/auto_models/test_build_options.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,10 @@ def test_compile_config_fields(self):
("fsdp_allgather_prefetch", "True"),
("fsdp_overlap_grad_reduce", "True"),
("max_in_flight_gb", "0.25"),
("inductor_mode", "'off'"),
("inductor_configs", "None"),
("fsdp_bucketing", "False"),
("fsdp_bucket_modules", "None"),
],
)

Expand Down Expand Up @@ -117,6 +121,16 @@ def test_compile_config_validation(self):
CompileConfig(mode="reduce-overhead", options={"triton": True})
with self.assertRaisesRegex(ValueError, "dynamo_cache_size_limit"):
CompileConfig(dynamo_cache_size_limit=0)
for options in (
{"inductor_mode": "unknown"},
{"inductor_configs": {1: True}},
{"inductor_configs": {"max_autotune": True}},
{"fsdp_bucket_modules": "layers"},
):
with self.subTest(options=options), self.assertRaises(ValueError):
CompileConfig(**options)
with self.assertRaisesRegex(TypeError, "fsdp_bucketing"):
CompileConfig(fsdp_bucketing=1)

@arg_mark(plat_marks=["cpu_linux", "cpu_macos"], level_mark="level0",
card_mark="allcards", essential_mark="essential")
Expand Down
4 changes: 4 additions & 0 deletions tests/ut/auto_models/test_public_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,10 @@ def test_compile_config_fields(self):
("fsdp_allgather_prefetch", "True"),
("fsdp_overlap_grad_reduce", "True"),
("max_in_flight_gb", "0.25"),
("inductor_mode", "'off'"),
("inductor_configs", "None"),
("fsdp_bucketing", "False"),
("fsdp_bucket_modules", "None"),
],
)

Expand Down
Loading
Loading