From 36d438fe96d0354602b794923b0135fe9e6be521 Mon Sep 17 00:00:00 2001 From: yu-chen-code Date: Thu, 12 Nov 2026 16:18:25 +0800 Subject: [PATCH] feat(compile): connect explicit joint graph backend execution --- hyper_parallel/compile/compiler.py | 21 +++++- hyper_parallel/compile/pass_config.py | 4 +- .../compile/passes/inductor_pass.py | 9 ++- .../compile/passes/parallel/fsdp_pass.py | 10 +++ hyper_parallel/models/build_options.py | 18 +++++ tests/ut/auto_models/test_build_options.py | 15 ++++ tests/ut/compile/test_compiler.py | 39 +++++++++++ tests/ut/compile/test_inductor_pass.py | 68 ++++++++++++++++--- 8 files changed, 170 insertions(+), 14 deletions(-) diff --git a/hyper_parallel/compile/compiler.py b/hyper_parallel/compile/compiler.py index cd2776c4d9..690bebb706 100644 --- a/hyper_parallel/compile/compiler.py +++ b/hyper_parallel/compile/compiler.py @@ -181,6 +181,13 @@ def is_compiled(self) -> bool: """Whether a joint graph has already been compiled.""" return self._joint_graph is not None + @property + def execution_backend(self) -> Optional[str]: + """Actual terminal backend, or None before the graph is ready.""" + if self._joint_graph is None: + return None + return getattr(self._joint_graph.graph_module, "meta", {}).get("inductor_compiled", "fx") + @property def parameter_layouts(self) -> Optional[dict]: """Final parameter-identity layouts, available after storage preparation. @@ -317,9 +324,17 @@ def compile(self, **inputs: Any) -> None: if ac_scope is not None: pass_kwargs["ac_scope"] = ac_scope - # Passes mutate ``graph_module`` in place and return it, so the - # transformed graph lives on ``joint_graph`` for ``forward_backward``. - pipeline.run(joint_graph.graph_module, **pass_kwargs) + # The terminal backend may return a wrapper around the transformed graph. + runtime_graph = pipeline.run(joint_graph.graph_module, **pass_kwargs) + if getattr(runtime_graph, "meta", {}).get("inductor_compiled") == "unavailable": + raise RuntimeError("The requested joint-graph Inductor backend is unavailable in this torch build") + for name in ( + "state_fqns", "state_is_param", "num_state_inputs", "user_inputs_spec", + "loss_dict_keys", "gradient_usage", "fsdp_sharded_param_names", "meta", + ): + if not hasattr(runtime_graph, name) and hasattr(joint_graph.graph_module, name): + setattr(runtime_graph, name, getattr(joint_graph.graph_module, name)) + joint_graph.graph_module = runtime_graph if joint_graph.input_guards is not None: joint_graph.input_guards.refresh() diff --git a/hyper_parallel/compile/pass_config.py b/hyper_parallel/compile/pass_config.py index dca4ccc302..dc86909335 100644 --- a/hyper_parallel/compile/pass_config.py +++ b/hyper_parallel/compile/pass_config.py @@ -497,7 +497,7 @@ def build_pass_config_from_trainer_config( enable_overlap: bool = False, mesh_context: Optional[Any] = None, ) -> PassConfig: - """Project trainer topology intent onto graph-mode pass config. + """Project trainer topology and joint-backend intent onto graph-mode passes. This helper stays in the pass-config module because it computes the graph-pass surface from the trainer topology without altering runtime @@ -578,6 +578,8 @@ def build_pass_config_from_trainer_config( compile_config, "fsdp_overlap_grad_reduce", True ), max_in_flight_gb=getattr(compile_config, "max_in_flight_gb", 0.25), + inductor_mode=getattr(compile_config, "inductor_mode", "off"), + inductor_configs=getattr(compile_config, "inductor_configs", None), ac_mode=ac_mode, ac_layer_count=ac_layer_count, ac_layer_indices=ac_layer_indices, diff --git a/hyper_parallel/compile/passes/inductor_pass.py b/hyper_parallel/compile/passes/inductor_pass.py index af5f71a820..9ea6a26920 100644 --- a/hyper_parallel/compile/passes/inductor_pass.py +++ b/hyper_parallel/compile/passes/inductor_pass.py @@ -311,7 +311,7 @@ def run( placeholder metas. Returns: - The same graph module with compiled regions spliced in and the + The runtime module with compiled regions spliced in and the positional calling convention restored. On torch builds without regional Inductor support it is returned untouched, logging a warning and recording ``"unavailable"`` in its meta. @@ -335,7 +335,7 @@ def run( "JointGraph.example_inputs) to size compiled regions" ) if _regional_inductor_available(): - self._compile_graph(graph_module, pass_config, example_inputs) + graph_module = self._compile_graph(graph_module, pass_config, example_inputs) compiled = True else: self._warn_unavailable(mode) @@ -417,6 +417,11 @@ def _compile_graph( _propagate_fake_vals(graph_module) if example_inputs is None: example_inputs = _placeholder_fake_args(graph_module) + else: + # FSDP partitions state storage after capture; user symbols stay unchanged. + state_count = getattr(graph_module, "num_state_inputs", 0) + state_inputs = tuple(_placeholder_fake_args(graph_module)[:state_count]) + example_inputs = state_inputs + tuple(example_inputs[state_count:]) _LOG.info( "InductorPass: compiling graph in %r mode (%d placeholders, %d nodes)", mode, diff --git a/hyper_parallel/compile/passes/parallel/fsdp_pass.py b/hyper_parallel/compile/passes/parallel/fsdp_pass.py index 5b4671eb7b..777ab44097 100644 --- a/hyper_parallel/compile/passes/parallel/fsdp_pass.py +++ b/hyper_parallel/compile/passes/parallel/fsdp_pass.py @@ -480,6 +480,16 @@ def run( len(param_nodes), ) + # Backend propagation starts from physical shards, not captured logical weights. + parameters = dict(model.named_parameters()) + for node in param_nodes: + value = node.meta.get("val") + parameter = parameters.get(node.meta["param_name"]) + if isinstance(value, torch.Tensor) and parameter is not None: + node.meta["val"] = value.new_empty_strided( + parameter.shape, parameter.stride(), requires_grad=value.requires_grad + ) + graph_module.recompile() return graph_module diff --git a/hyper_parallel/models/build_options.py b/hyper_parallel/models/build_options.py index 58d8e0c400..d6c377fb5d 100644 --- a/hyper_parallel/models/build_options.py +++ b/hyper_parallel/models/build_options.py @@ -123,6 +123,8 @@ class CompileConfig: ``max_in_flight_gb`` follows PyTorch's overlap scheduler naming and binary units; here it bounds early all-gather replicas until first use, per phase. Eager FSDP2 module prefetch depths do not configure these graph passes. + ``inductor_mode`` and ``inductor_configs`` select the terminal joint-graph + backend independently of decoder compilation; ``off`` executes FX directly. """ enabled: bool = False @@ -138,6 +140,9 @@ class CompileConfig: fsdp_allgather_prefetch: bool = True fsdp_overlap_grad_reduce: bool = True max_in_flight_gb: float = 0.25 + # Joint-graph backend; backend/mode/options above configure decoder compilation. + inductor_mode: Literal["off", "regional", "full"] = "off" + inductor_configs: Optional[dict[str, Any]] = None def selects_graph_compiler(self) -> bool: """Return whether this config selects the graph-compiler trainer path.""" @@ -153,6 +158,7 @@ def __post_init__(self) -> None: if not isinstance(getattr(self, name), bool): raise TypeError(f"compile.{name} must be a bool") self._validate_max_in_flight_gb() + self._validate_joint_graph_backend() if self.dynamic_arg_dims is not None and not isinstance(self.dynamic_arg_dims, dict): raise ValueError("compile.dynamic_arg_dims must be a mapping or None") if not isinstance(self.mode, str) or not self.mode.strip(): @@ -185,6 +191,18 @@ def _validate_max_in_flight_gb(self) -> None: ): raise ValueError("compile.max_in_flight_gb must be a finite non-negative number") + def _validate_joint_graph_backend(self) -> None: + """Validate the terminal backend independently of decoder options.""" + if self.inductor_mode not in ("off", "regional", "full"): + raise ValueError("compile.inductor_mode must be 'off', 'regional' or 'full'") + if self.inductor_configs is not None and ( + not isinstance(self.inductor_configs, dict) + or any(not isinstance(key, str) for key in self.inductor_configs) + ): + raise ValueError("compile.inductor_configs must be a mapping with string keys or None") + if self.inductor_configs and self.inductor_mode == "off": + raise ValueError("compile.inductor_configs requires an enabled compile.inductor_mode") + @dataclass class FSDP2MixedPrecisionConfig: diff --git a/tests/ut/auto_models/test_build_options.py b/tests/ut/auto_models/test_build_options.py index 9acee13238..c400247bbc 100644 --- a/tests/ut/auto_models/test_build_options.py +++ b/tests/ut/auto_models/test_build_options.py @@ -60,6 +60,8 @@ def test_compile_config_fields(self): ("fsdp_allgather_prefetch", "True"), ("fsdp_overlap_grad_reduce", "True"), ("max_in_flight_gb", "0.25"), + ("inductor_mode", "'off'"), + ("inductor_configs", "None"), ], ) @@ -108,6 +110,19 @@ def test_compile_config_validation(self): with self.assertRaisesRegex(ValueError, "dynamo_cache_size_limit"): CompileConfig(dynamo_cache_size_limit=0) + def test_joint_backend_validation(self): + """Reject malformed backend options and configurations disabled by their mode.""" + cases = ( + {"inductor_mode": "eager"}, + {"inductor_mode": "full", "inductor_configs": ["debug"]}, + {"inductor_mode": "full", "inductor_configs": {1: True}}, + {"inductor_configs": {"debug": True}}, + ) + for options in cases: + with self.subTest(options=options): + with self.assertRaisesRegex(ValueError, "compile.inductor_"): + CompileConfig(**options) + def test_use_joint_graph_selects_graph_compiler(self): """Graph-mode trainer selection is driven by compile.enabled + flag.""" config = CompileConfig( diff --git a/tests/ut/compile/test_compiler.py b/tests/ut/compile/test_compiler.py index 2b2519c94f..03617cc441 100644 --- a/tests/ut/compile/test_compiler.py +++ b/tests/ut/compile/test_compiler.py @@ -71,6 +71,45 @@ def _fixed_batch(): class TestGraphCompilerForwardBackward(unittest.TestCase): """``compile`` / lazy ``forward_backward`` without distributed.""" + def test_terminal_wrapper_preserves_training_metadata(self) -> None: + """Execute the returned module while preserving loss and gradient output layout.""" + model = _make_model() + x, y = _fixed_batch() + expected_loss = _mse_train_fn(model, x=x, y=y) + expected_grads = torch.autograd.grad(expected_loss, tuple(model.parameters())) + compiler = GraphCompiler(model, _mse_train_fn, PassConfig(fsdp_enabled=False)) + wrappers = [] + + class RuntimeWrapper(nn.Module): + """Mirror the backend's positional state-and-input calling convention.""" + + def __init__(self, inner): + """Keep the executable without its tracer metadata.""" + super().__init__() + self.inner = inner + + def forward(self, *args): + """Forward every flat state and user input positionally.""" + return self.inner(*args) + + def wrap(graph_module, **_kwargs): + """Return a terminal executable without copying tracer metadata.""" + wrappers.append(RuntimeWrapper(graph_module)) + return wrappers[-1] + + self.assertIsNone(compiler.execution_backend) + with patch("hyper_parallel.compile.compiler.PassPipeline.run", side_effect=wrap): + loss, _ = compiler.forward_backward(x=x, y=y) + + self.assertIs(compiler._joint_graph.graph_module, wrappers[0]) + self.assertEqual(compiler.execution_backend, "fx") + self.assertEqual(wrappers[0].num_state_inputs, len(tuple(model.parameters()))) + self.assertEqual(wrappers[0].state_fqns, compiler._joint_graph.state_fqns) + self.assertEqual(wrappers[0].gradient_usage, (True, True)) + torch.testing.assert_close(loss, expected_loss, rtol=0, atol=0) + for parameter, expected in zip(model.parameters(), expected_grads): + torch.testing.assert_close(parameter.grad, expected, rtol=0, atol=0) + def test_unit_bindings_reach_passes_with_tied_expert(self) -> None: """Claim nested experts before the root and keep every alias on EDP.""" model = nn.Module() diff --git a/tests/ut/compile/test_inductor_pass.py b/tests/ut/compile/test_inductor_pass.py index 10c206adb9..9077072295 100644 --- a/tests/ut/compile/test_inductor_pass.py +++ b/tests/ut/compile/test_inductor_pass.py @@ -27,9 +27,8 @@ itself; every per-stage forward/backward subgraph is compiled instead (both ``regional`` and ``full``), while unknown ``call_module`` stubs are still rejected. -4. Guards: missing ``example_inputs``, mode canonicalization, pipeline - placement (terminal), and that inductor settings stay PassConfig-owned - (PP-style) instead of being projected from ``CompileConfig``. +4. Guards: missing ``example_inputs``, mode canonicalization, terminal + placement, and explicit joint-backend settings projected from ``CompileConfig``. """ # The backend env var must be set before torch / hyper_parallel are imported, @@ -62,6 +61,7 @@ InductorPass, _propagate_fake_vals, ) +from hyper_parallel.compile.passes.parallel.fsdp_pass import FSDPPass from hyper_parallel.compile.passes.parallel.pp_schedule import PipelineScheduleBase from hyper_parallel.compile.passes.pipeline import PassPipeline from hyper_parallel.compile.tracer.graph_tracer import trace_model_graph @@ -447,7 +447,7 @@ def test_pipeline_allows_pp_with_inductor(self): self.assertEqual(joint.graph_module.meta.get("inductor_compiled"), "regional") def test_trainer_config_projection_strategy(self): - """Test inductor config is PassConfig-owned (PP-style), not projected.""" + """Project joint backend options independently of decoder compilation options.""" trainer_config = SimpleNamespace( accelerator=SimpleNamespace( tp_size=1, sequence_parallel=False, loss_parallel=False @@ -458,13 +458,65 @@ def test_trainer_config_projection_strategy(self): compile=SimpleNamespace( inductor_mode="regional", inductor_configs={"reorder_for_peak_memory": True}, + backend="decoder_backend", + options={"decoder_only": True}, ), ) config = build_pass_config_from_trainer_config(trainer_config) - # Like PP, inductor settings live on PassConfig only; the projected - # config keeps the defaults and ignores CompileConfig-level overrides. - self.assertEqual(config.inductor_mode, "off") - self.assertIsNone(config.inductor_configs) + self.assertEqual(config.inductor_mode, "regional") + self.assertEqual(config.inductor_configs, {"reorder_for_peak_memory": True}) + + def test_retains_backend_replacement(self): + """The pass returns the executable wrapper produced by backend lowering.""" + joint = self._traced() + wrapper = nn.Sequential(joint.graph_module) + wrapper.meta = {} + with mock.patch.object(InductorPass, "_compile_graph", return_value=wrapper): + result = InductorPass().run( + joint.graph_module, PassConfig(inductor_mode="full"), example_inputs=joint.example_inputs + ) + self.assertIs(result, wrapper) + self.assertEqual(result.meta["inductor_compiled"], "full") + + def test_compiler_rejects_unavailable_backend(self): + """An unavailable backend cannot be mistaken for successful compilation.""" + compiler = GraphCompiler( + _make_model(), _train_fn, pass_config=PassConfig(fsdp_enabled=False, inductor_mode="full") + ) + with mock.patch( + "hyper_parallel.compile.passes.inductor_pass._regional_inductor_available", return_value=False + ): + with self.assertRaisesRegex(RuntimeError, "unavailable"): + x, y = _fixed_batch() + compiler.compile(x=x, y=y) + self.assertFalse(compiler.is_compiled) + self.assertIsNone(compiler.execution_backend) + + def test_fsdp_backend_inputs_describe_physical_storage(self): + """Backend sizing uses padded shards while retaining the traced user inputs.""" + model = nn.Linear(4, 5) + joint = trace_model_graph(model, _train_fn, {"x": torch.randn(2, 4), "y": torch.randn(2, 5)}) + transport = mock.MagicMock() + transport.is_initialized.return_value = True + transport.get_world_size.return_value = 2 + transport.get_rank.return_value = 1 + with mock.patch("hyper_parallel.compile.passes.parallel.fsdp_pass.dist", transport), \ + mock.patch("hyper_parallel.compile.passes.parallel.fsdp_pass._resolve_process_group"): + graph_module = FSDPPass().run(joint.graph_module, PassConfig(fsdp_degree=2), model=model) + parameters = dict(model.named_parameters()) + for node in graph_module.graph.find_nodes(op="placeholder"): + name = node.meta.get("param_name") + if name in parameters: + self.assertEqual(tuple(node.meta["val"].shape), tuple(parameters[name].shape)) + self.assertEqual(node.meta["val"].stride(), parameters[name].stride()) + with mock.patch("hyper_parallel.compile.passes.inductor_pass._propagate_fake_vals"), \ + mock.patch("torch.fx.passes.regional_inductor.regional_inductor") as backend: + InductorPass().run(graph_module, PassConfig(inductor_mode="full"), example_inputs=joint.example_inputs) + state_count = graph_module.num_state_inputs + runtime_inputs = backend.call_args.args[1] + self.assertEqual(tuple(runtime_inputs[0].shape), tuple(model.weight.shape)) + for actual, expected in zip(runtime_inputs[state_count:], joint.example_inputs[state_count:]): + self.assertIs(actual, expected) if __name__ == "__main__":