Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 18 additions & 3 deletions hyper_parallel/compile/compiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,13 @@ def is_compiled(self) -> bool:
"""Whether a joint graph has already been compiled."""
return self._joint_graph is not None

@property
def execution_backend(self) -> Optional[str]:
"""Actual terminal backend, or None before the graph is ready."""
if self._joint_graph is None:
return None
return getattr(self._joint_graph.graph_module, "meta", {}).get("inductor_compiled", "fx")

@property
def parameter_layouts(self) -> Optional[dict]:
"""Final parameter-identity layouts, available after storage preparation.
Expand Down Expand Up @@ -317,9 +324,17 @@ def compile(self, **inputs: Any) -> None:
if ac_scope is not None:
pass_kwargs["ac_scope"] = ac_scope

# Passes mutate ``graph_module`` in place and return it, so the
# transformed graph lives on ``joint_graph`` for ``forward_backward``.
pipeline.run(joint_graph.graph_module, **pass_kwargs)
# The terminal backend may return a wrapper around the transformed graph.
runtime_graph = pipeline.run(joint_graph.graph_module, **pass_kwargs)
if getattr(runtime_graph, "meta", {}).get("inductor_compiled") == "unavailable":
raise RuntimeError("The requested joint-graph Inductor backend is unavailable in this torch build")
for name in (
"state_fqns", "state_is_param", "num_state_inputs", "user_inputs_spec",
"loss_dict_keys", "gradient_usage", "fsdp_sharded_param_names", "meta",
):
if not hasattr(runtime_graph, name) and hasattr(joint_graph.graph_module, name):
setattr(runtime_graph, name, getattr(joint_graph.graph_module, name))
joint_graph.graph_module = runtime_graph

if joint_graph.input_guards is not None:
joint_graph.input_guards.refresh()
Expand Down
4 changes: 3 additions & 1 deletion hyper_parallel/compile/pass_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -497,7 +497,7 @@ def build_pass_config_from_trainer_config(
enable_overlap: bool = False,
mesh_context: Optional[Any] = None,
) -> PassConfig:
"""Project trainer topology intent onto graph-mode pass config.
"""Project trainer topology and joint-backend intent onto graph-mode passes.

This helper stays in the pass-config module because it computes the
graph-pass surface from the trainer topology without altering runtime
Expand Down Expand Up @@ -578,6 +578,8 @@ def build_pass_config_from_trainer_config(
compile_config, "fsdp_overlap_grad_reduce", True
),
max_in_flight_gb=getattr(compile_config, "max_in_flight_gb", 0.25),
inductor_mode=getattr(compile_config, "inductor_mode", "off"),
inductor_configs=getattr(compile_config, "inductor_configs", None),
ac_mode=ac_mode,
ac_layer_count=ac_layer_count,
ac_layer_indices=ac_layer_indices,
Expand Down
9 changes: 7 additions & 2 deletions hyper_parallel/compile/passes/inductor_pass.py
Original file line number Diff line number Diff line change
Expand Up @@ -311,7 +311,7 @@ def run(
placeholder metas.

Returns:
The same graph module with compiled regions spliced in and the
The runtime module with compiled regions spliced in and the
positional calling convention restored. On torch builds without
regional Inductor support it is returned untouched, logging a
warning and recording ``"unavailable"`` in its meta.
Expand All @@ -335,7 +335,7 @@ def run(
"JointGraph.example_inputs) to size compiled regions"
)
if _regional_inductor_available():
self._compile_graph(graph_module, pass_config, example_inputs)
graph_module = self._compile_graph(graph_module, pass_config, example_inputs)
compiled = True
else:
self._warn_unavailable(mode)
Expand Down Expand Up @@ -417,6 +417,11 @@ def _compile_graph(
_propagate_fake_vals(graph_module)
if example_inputs is None:
example_inputs = _placeholder_fake_args(graph_module)
else:
# FSDP partitions state storage after capture; user symbols stay unchanged.
state_count = getattr(graph_module, "num_state_inputs", 0)
state_inputs = tuple(_placeholder_fake_args(graph_module)[:state_count])
example_inputs = state_inputs + tuple(example_inputs[state_count:])
_LOG.info(
"InductorPass: compiling graph in %r mode (%d placeholders, %d nodes)",
mode,
Expand Down
10 changes: 10 additions & 0 deletions hyper_parallel/compile/passes/parallel/fsdp_pass.py
Original file line number Diff line number Diff line change
Expand Up @@ -480,6 +480,16 @@ def run(
len(param_nodes),
)

# Backend propagation starts from physical shards, not captured logical weights.
parameters = dict(model.named_parameters())
for node in param_nodes:
value = node.meta.get("val")
parameter = parameters.get(node.meta["param_name"])
if isinstance(value, torch.Tensor) and parameter is not None:
node.meta["val"] = value.new_empty_strided(
parameter.shape, parameter.stride(), requires_grad=value.requires_grad
)

graph_module.recompile()
return graph_module

Expand Down
18 changes: 18 additions & 0 deletions hyper_parallel/models/build_options.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,8 @@ class CompileConfig:
``max_in_flight_gb`` follows PyTorch's overlap scheduler naming and binary
units; here it bounds early all-gather replicas until first use, per phase.
Eager FSDP2 module prefetch depths do not configure these graph passes.
``inductor_mode`` and ``inductor_configs`` select the terminal joint-graph
backend independently of decoder compilation; ``off`` executes FX directly.
"""

enabled: bool = False
Expand All @@ -138,6 +140,9 @@ class CompileConfig:
fsdp_allgather_prefetch: bool = True
fsdp_overlap_grad_reduce: bool = True
max_in_flight_gb: float = 0.25
# Joint-graph backend; backend/mode/options above configure decoder compilation.
inductor_mode: Literal["off", "regional", "full"] = "off"
inductor_configs: Optional[dict[str, Any]] = None

def selects_graph_compiler(self) -> bool:
"""Return whether this config selects the graph-compiler trainer path."""
Expand All @@ -153,6 +158,7 @@ def __post_init__(self) -> None:
if not isinstance(getattr(self, name), bool):
raise TypeError(f"compile.{name} must be a bool")
self._validate_max_in_flight_gb()
self._validate_joint_graph_backend()
if self.dynamic_arg_dims is not None and not isinstance(self.dynamic_arg_dims, dict):
raise ValueError("compile.dynamic_arg_dims must be a mapping or None")
if not isinstance(self.mode, str) or not self.mode.strip():
Expand Down Expand Up @@ -185,6 +191,18 @@ def _validate_max_in_flight_gb(self) -> None:
):
raise ValueError("compile.max_in_flight_gb must be a finite non-negative number")

def _validate_joint_graph_backend(self) -> None:
"""Validate the terminal backend independently of decoder options."""
if self.inductor_mode not in ("off", "regional", "full"):
raise ValueError("compile.inductor_mode must be 'off', 'regional' or 'full'")
if self.inductor_configs is not None and (
not isinstance(self.inductor_configs, dict)
or any(not isinstance(key, str) for key in self.inductor_configs)
):
raise ValueError("compile.inductor_configs must be a mapping with string keys or None")
if self.inductor_configs and self.inductor_mode == "off":
raise ValueError("compile.inductor_configs requires an enabled compile.inductor_mode")


@dataclass
class FSDP2MixedPrecisionConfig:
Expand Down
15 changes: 15 additions & 0 deletions tests/ut/auto_models/test_build_options.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,8 @@ def test_compile_config_fields(self):
("fsdp_allgather_prefetch", "True"),
("fsdp_overlap_grad_reduce", "True"),
("max_in_flight_gb", "0.25"),
("inductor_mode", "'off'"),
("inductor_configs", "None"),
],
)

Expand Down Expand Up @@ -108,6 +110,19 @@ def test_compile_config_validation(self):
with self.assertRaisesRegex(ValueError, "dynamo_cache_size_limit"):
CompileConfig(dynamo_cache_size_limit=0)

def test_joint_backend_validation(self):
"""Reject malformed backend options and configurations disabled by their mode."""
cases = (
{"inductor_mode": "eager"},
{"inductor_mode": "full", "inductor_configs": ["debug"]},
{"inductor_mode": "full", "inductor_configs": {1: True}},
{"inductor_configs": {"debug": True}},
)
for options in cases:
with self.subTest(options=options):
with self.assertRaisesRegex(ValueError, "compile.inductor_"):
CompileConfig(**options)

def test_use_joint_graph_selects_graph_compiler(self):
"""Graph-mode trainer selection is driven by compile.enabled + flag."""
config = CompileConfig(
Expand Down
39 changes: 39 additions & 0 deletions tests/ut/compile/test_compiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,45 @@ def _fixed_batch():
class TestGraphCompilerForwardBackward(unittest.TestCase):
"""``compile`` / lazy ``forward_backward`` without distributed."""

def test_terminal_wrapper_preserves_training_metadata(self) -> None:
"""Execute the returned module while preserving loss and gradient output layout."""
model = _make_model()
x, y = _fixed_batch()
expected_loss = _mse_train_fn(model, x=x, y=y)
expected_grads = torch.autograd.grad(expected_loss, tuple(model.parameters()))
compiler = GraphCompiler(model, _mse_train_fn, PassConfig(fsdp_enabled=False))
wrappers = []

class RuntimeWrapper(nn.Module):
"""Mirror the backend's positional state-and-input calling convention."""

def __init__(self, inner):
"""Keep the executable without its tracer metadata."""
super().__init__()
self.inner = inner

def forward(self, *args):
"""Forward every flat state and user input positionally."""
return self.inner(*args)

def wrap(graph_module, **_kwargs):
"""Return a terminal executable without copying tracer metadata."""
wrappers.append(RuntimeWrapper(graph_module))
return wrappers[-1]

self.assertIsNone(compiler.execution_backend)
with patch("hyper_parallel.compile.compiler.PassPipeline.run", side_effect=wrap):
loss, _ = compiler.forward_backward(x=x, y=y)

self.assertIs(compiler._joint_graph.graph_module, wrappers[0])
self.assertEqual(compiler.execution_backend, "fx")
self.assertEqual(wrappers[0].num_state_inputs, len(tuple(model.parameters())))
self.assertEqual(wrappers[0].state_fqns, compiler._joint_graph.state_fqns)
self.assertEqual(wrappers[0].gradient_usage, (True, True))
torch.testing.assert_close(loss, expected_loss, rtol=0, atol=0)
for parameter, expected in zip(model.parameters(), expected_grads):
torch.testing.assert_close(parameter.grad, expected, rtol=0, atol=0)

def test_unit_bindings_reach_passes_with_tied_expert(self) -> None:
"""Claim nested experts before the root and keep every alias on EDP."""
model = nn.Module()
Expand Down
68 changes: 60 additions & 8 deletions tests/ut/compile/test_inductor_pass.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,8 @@
itself; every per-stage forward/backward subgraph is compiled instead
(both ``regional`` and ``full``), while unknown ``call_module`` stubs are
still rejected.
4. Guards: missing ``example_inputs``, mode canonicalization, pipeline
placement (terminal), and that inductor settings stay PassConfig-owned
(PP-style) instead of being projected from ``CompileConfig``.
4. Guards: missing ``example_inputs``, mode canonicalization, terminal
placement, and explicit joint-backend settings projected from ``CompileConfig``.
"""

# The backend env var must be set before torch / hyper_parallel are imported,
Expand Down Expand Up @@ -62,6 +61,7 @@
InductorPass,
_propagate_fake_vals,
)
from hyper_parallel.compile.passes.parallel.fsdp_pass import FSDPPass
from hyper_parallel.compile.passes.parallel.pp_schedule import PipelineScheduleBase
from hyper_parallel.compile.passes.pipeline import PassPipeline
from hyper_parallel.compile.tracer.graph_tracer import trace_model_graph
Expand Down Expand Up @@ -447,7 +447,7 @@ def test_pipeline_allows_pp_with_inductor(self):
self.assertEqual(joint.graph_module.meta.get("inductor_compiled"), "regional")

def test_trainer_config_projection_strategy(self):
"""Test inductor config is PassConfig-owned (PP-style), not projected."""
"""Project joint backend options independently of decoder compilation options."""
trainer_config = SimpleNamespace(
accelerator=SimpleNamespace(
tp_size=1, sequence_parallel=False, loss_parallel=False
Expand All @@ -458,13 +458,65 @@ def test_trainer_config_projection_strategy(self):
compile=SimpleNamespace(
inductor_mode="regional",
inductor_configs={"reorder_for_peak_memory": True},
backend="decoder_backend",
options={"decoder_only": True},
),
)
config = build_pass_config_from_trainer_config(trainer_config)
# Like PP, inductor settings live on PassConfig only; the projected
# config keeps the defaults and ignores CompileConfig-level overrides.
self.assertEqual(config.inductor_mode, "off")
self.assertIsNone(config.inductor_configs)
self.assertEqual(config.inductor_mode, "regional")
self.assertEqual(config.inductor_configs, {"reorder_for_peak_memory": True})

def test_retains_backend_replacement(self):
"""The pass returns the executable wrapper produced by backend lowering."""
joint = self._traced()
wrapper = nn.Sequential(joint.graph_module)
wrapper.meta = {}
with mock.patch.object(InductorPass, "_compile_graph", return_value=wrapper):
result = InductorPass().run(
joint.graph_module, PassConfig(inductor_mode="full"), example_inputs=joint.example_inputs
)
self.assertIs(result, wrapper)
self.assertEqual(result.meta["inductor_compiled"], "full")

def test_compiler_rejects_unavailable_backend(self):
"""An unavailable backend cannot be mistaken for successful compilation."""
compiler = GraphCompiler(
_make_model(), _train_fn, pass_config=PassConfig(fsdp_enabled=False, inductor_mode="full")
)
with mock.patch(
"hyper_parallel.compile.passes.inductor_pass._regional_inductor_available", return_value=False
):
with self.assertRaisesRegex(RuntimeError, "unavailable"):
x, y = _fixed_batch()
compiler.compile(x=x, y=y)
self.assertFalse(compiler.is_compiled)
self.assertIsNone(compiler.execution_backend)

def test_fsdp_backend_inputs_describe_physical_storage(self):
"""Backend sizing uses padded shards while retaining the traced user inputs."""
model = nn.Linear(4, 5)
joint = trace_model_graph(model, _train_fn, {"x": torch.randn(2, 4), "y": torch.randn(2, 5)})
transport = mock.MagicMock()
transport.is_initialized.return_value = True
transport.get_world_size.return_value = 2
transport.get_rank.return_value = 1
with mock.patch("hyper_parallel.compile.passes.parallel.fsdp_pass.dist", transport), \
mock.patch("hyper_parallel.compile.passes.parallel.fsdp_pass._resolve_process_group"):
graph_module = FSDPPass().run(joint.graph_module, PassConfig(fsdp_degree=2), model=model)
parameters = dict(model.named_parameters())
for node in graph_module.graph.find_nodes(op="placeholder"):
name = node.meta.get("param_name")
if name in parameters:
self.assertEqual(tuple(node.meta["val"].shape), tuple(parameters[name].shape))
self.assertEqual(node.meta["val"].stride(), parameters[name].stride())
with mock.patch("hyper_parallel.compile.passes.inductor_pass._propagate_fake_vals"), \
mock.patch("torch.fx.passes.regional_inductor.regional_inductor") as backend:
InductorPass().run(graph_module, PassConfig(inductor_mode="full"), example_inputs=joint.example_inputs)
state_count = graph_module.num_state_inputs
runtime_inputs = backend.call_args.args[1]
self.assertEqual(tuple(runtime_inputs[0].shape), tuple(model.weight.shape))
for actual, expected in zip(runtime_inputs[state_count:], joint.example_inputs[state_count:]):
self.assertIs(actual, expected)


if __name__ == "__main__":
Expand Down
Loading