From d7d0c09e359a459ba9d6416e08d0bfb38a63a105 Mon Sep 17 00:00:00 2001 From: yu-chen-code Date: Fri, 6 Nov 2026 22:47:53 +0800 Subject: [PATCH 1/2] fix(compile): enforce replay contracts and retain terminal executables --- hyper_parallel/compile/compiler.py | 15 ++- .../compile/tracer/dynamic_shapes.py | 50 ++++++++- hyper_parallel/compile/tracer/graph_tracer.py | 11 +- tests/ut/compile/test_dynamic_shapes.py | 103 +++++++++++++++++- 4 files changed, 168 insertions(+), 11 deletions(-) diff --git a/hyper_parallel/compile/compiler.py b/hyper_parallel/compile/compiler.py index a624ff4bdb..45633f4d71 100644 --- a/hyper_parallel/compile/compiler.py +++ b/hyper_parallel/compile/compiler.py @@ -254,12 +254,21 @@ def compile(self, **inputs: Any) -> None: if ac_scope is not None: pass_kwargs["ac_scope"] = ac_scope - # Passes mutate ``graph_module`` in place and return it, so the - # transformed graph lives on ``joint_graph`` for ``forward_backward``. - pipeline.run(joint_graph.graph_module, **pass_kwargs) + # A pass may replace the module, e.g. when installing an executable + # backend wrapper. Keep its returned graph as the runtime executable. + runtime_graph = pipeline.run(joint_graph.graph_module, **pass_kwargs) + for name in ( + "state_fqns", "state_is_param", "num_state_inputs", "user_inputs_spec", + "loss_dict_keys", "fsdp_sharded_param_names", + ): + if not hasattr(runtime_graph, name) and hasattr(joint_graph.graph_module, name): + setattr(runtime_graph, name, getattr(joint_graph.graph_module, name)) + joint_graph.graph_module = runtime_graph if joint_graph.input_guards is not None: joint_graph.input_guards.refresh() + if joint_graph.state_guards is not None: + joint_graph.state_guards.refresh(self.model) self._joint_graph = joint_graph def forward_backward(self, **inputs: Any) -> Any: diff --git a/hyper_parallel/compile/tracer/dynamic_shapes.py b/hyper_parallel/compile/tracer/dynamic_shapes.py index f0a084d22c..d02f64507a 100644 --- a/hyper_parallel/compile/tracer/dynamic_shapes.py +++ b/hyper_parallel/compile/tracer/dynamic_shapes.py @@ -122,10 +122,11 @@ class InputGuards: No real input tensors or storage are retained here. """ - def __init__(self, inputs: list[Any], fake_inputs: tuple[Any, ...], shape_env: ShapeEnv) -> None: + def __init__(self, inputs: list[Any], fake_inputs: tuple[Any, ...], shape_env: ShapeEnv | None) -> None: self.shape_env = shape_env self.fake_tensors = [value for value in fake_inputs if isinstance(value, torch.Tensor)] self.leaves = [self._metadata(value) for value in inputs] + self.static_geometry = None if shape_env is not None else self._geometry(inputs) self.aliases = self._aliases(inputs) self.expression = None self.refresh() @@ -144,9 +145,18 @@ def _aliases(inputs: list[Any]) -> list[int]: for index, value in enumerate(inputs) if isinstance(value, torch.Tensor) ] + @staticmethod + def _geometry(inputs: list[Any]) -> list[tuple]: + return [ + (tuple(value.shape), tuple(value.stride()) if value.layout == torch.strided else None, + value.storage_offset() if value.layout == torch.strided else None) + for value in inputs if isinstance(value, torch.Tensor) + ] + def refresh(self) -> None: """Include constraints introduced by tracing and graph passes.""" - self.expression = self.shape_env.produce_guards_expression(self.fake_tensors, ignore_static=False) + if self.shape_env is not None: + self.expression = self.shape_env.produce_guards_expression(self.fake_tensors, ignore_static=False) def validate(self, inputs: list[Any]) -> None: """Reject incompatible inputs before any graph or collective is executed.""" @@ -154,6 +164,8 @@ def validate(self, inputs: list[Any]) -> None: raise ValueError("Graph input rank, dtype, device, layout, requires_grad or Python constant changed") if self._aliases(inputs) != self.aliases: raise ValueError("Graph input tensor aliasing changed since compilation") + if self.static_geometry is not None and self._geometry(inputs) != self.static_geometry: + raise ValueError("Static graph input shape/stride changed; compile again for the new inputs") tensors = [value for value in inputs if isinstance(value, torch.Tensor)] if self.expression and not self.shape_env.evaluate_guards_expression(self.expression, tensors): raise ValueError( @@ -161,3 +173,37 @@ def validate(self, inputs: list[Any]) -> None: "related input sizes and shape-dependent branches. Compile with representative dynamic sizes > 1; " "sizes 0/1 and Python shape branches may specialize the graph." ) + + +class ModelStateGuards: + """Guard graph-specialized module modes and storage metadata, allowing value updates.""" + + def __init__(self, model: torch.nn.Module) -> None: + self.signature = self._signature(model) + + @staticmethod + def _signature(model: torch.nn.Module) -> tuple: + """Snapshot module identities, state roles and storage geometry without retaining values.""" + modes = tuple((name, id(module), type(module), module.training) for name, module in model.named_modules()) + state = [ + (kind, name, value) + for kind, entries in (("parameter", model.named_parameters(remove_duplicate=False)), + ("buffer", model.named_buffers(remove_duplicate=False))) + for name, value in entries + ] + tensors = tuple( + (kind, name, id(value), tuple(value.shape), value.dtype, value.device, value.layout, value.requires_grad, + tuple(value.stride()) if value.layout == torch.strided else None, + value.storage_offset() if value.layout == torch.strided else None) + for kind, name, value in state + ) + return modes, tensors + + def refresh(self, model: torch.nn.Module) -> None: + """Record the final storage contract after graph partitioning.""" + self.signature = self._signature(model) + + def validate(self, model: torch.nn.Module) -> None: + """Reject stale captures before running graph operators or collectives.""" + if self._signature(model) != self.signature: + raise ValueError("Graph model state or training mode changed; compile again before execution") diff --git a/hyper_parallel/compile/tracer/graph_tracer.py b/hyper_parallel/compile/tracer/graph_tracer.py index ea9a8f4627..3d913486f9 100644 --- a/hyper_parallel/compile/tracer/graph_tracer.py +++ b/hyper_parallel/compile/tracer/graph_tracer.py @@ -74,7 +74,7 @@ from torch.fx.traceback import preserve_node_meta from torch.nn.utils import stateless -from .dynamic_shapes import InputGuards, build_symbolic_inputs +from .dynamic_shapes import InputGuards, ModelStateGuards, build_symbolic_inputs # Older torch's make_fx predates record_stack_traces / record_module_stack # and rejects unknown kwargs; forward only the kwargs this torch supports. @@ -180,6 +180,7 @@ class JointGraph: state_fqns: List[str] example_inputs: tuple input_guards: InputGuards | None = None + state_guards: ModelStateGuards | None = None def extract_module_state(mod: nn.Module) -> Dict[str, torch.Tensor]: @@ -647,10 +648,10 @@ def trace_model_graph( # pylint: disable=too-many-locals num_layers=num_layers, state_fqns=state_fqns, example_inputs=fake_args, - input_guards=( - InputGuards(user_inputs_flat, fake_args[num_state_inputs:], fake_mode.shape_env) - if symbolic else None + input_guards=InputGuards( + user_inputs_flat, fake_args[num_state_inputs:], fake_mode.shape_env if symbolic else None ), + state_guards=ModelStateGuards(model), ) @@ -686,6 +687,8 @@ def run_traced_graph( f" Traced: {joint_graph.state_fqns}\n" f" Got: {list(model_state.keys())}" ) + if joint_graph.state_guards is not None: + joint_graph.state_guards.validate(model) user_flat, user_spec = torch.utils._pytree.tree_flatten(inputs) traced_spec = getattr(joint_graph.graph_module, "user_inputs_spec", None) diff --git a/tests/ut/compile/test_dynamic_shapes.py b/tests/ut/compile/test_dynamic_shapes.py index 4ed0348f69..1b93af86e6 100644 --- a/tests/ut/compile/test_dynamic_shapes.py +++ b/tests/ut/compile/test_dynamic_shapes.py @@ -48,6 +48,105 @@ def setUp(self) -> None: def _compiler(self, **kwargs): return GraphCompiler(self.model, _loss, pass_config=self.config, **kwargs) + def test_static_inputs_reject_specialization_changes_before_execution(self): + """Static geometry and Python constants cannot silently replay stale branches.""" + compiler = GraphCompiler( + self.model, lambda model, x, scale: model(x).sum() * scale, pass_config=self.config + ) + compiler.forward_backward(x=torch.randn(3, 4), scale=2) + invalid = [ + {"x": torch.randn(5, 4), "scale": 2}, + {"x": torch.randn(4, 3).t(), "scale": 2}, + {"x": torch.randn(4, 4)[1:], "scale": 2}, + {"x": torch.randn(3, 4, dtype=torch.float64), "scale": 2}, + {"x": torch.randn(3, 4), "scale": 3}, + ] + with patch.object(compiler._joint_graph.graph_module, "forward") as execute: + for inputs in invalid: + with self.subTest(inputs=inputs), self.assertRaisesRegex(ValueError, "[Gg]raph input"): + compiler.forward_backward(**inputs) + execute.assert_not_called() + + def test_state_values_remain_live_but_structural_changes_are_rejected(self): + """Optimizer updates are valid; changed mode, identity and gradient contract are not.""" + compiler = self._compiler() + x, y = torch.randn(3, 4), torch.randn(3, 3) + compiler.forward_backward(x=x, y=y) + with torch.no_grad(): + self.model.weight.add_(0.1) + actual, _ = compiler.forward_backward(x=x, y=y) + torch.testing.assert_close(actual, _loss(self.model, x, y), rtol=0, atol=0) + for change in ("mode", "parameter", "requires_grad"): + with self.subTest(change=change): + model = copy.deepcopy(self.model) + case = GraphCompiler(model, _loss, pass_config=self.config) + case.forward_backward(x=x, y=y) + if change == "mode": + model.eval() + elif change == "parameter": + model.weight = torch.nn.Parameter(torch.clone(model.weight)) + else: + model.weight.requires_grad_(False) + with patch.object(case._joint_graph.graph_module, "forward") as execute: + with self.assertRaisesRegex(ValueError, "Graph model state"): + case.forward_backward(x=x, y=y) + execute.assert_not_called() + + def test_replacement_executable_preserves_runtime_metadata(self): + """A terminal pass may return a wrapper; its loss dictionary and gradients survive.""" + class Executable(torch.nn.Module): + """Equivalent terminal backend wrapper without tracing metadata.""" + + def __init__(self, inner): + super().__init__() + self.inner = inner + self.calls = 0 + + def forward(self, *args): + self.calls += 1 + return self.inner(*args) + + compiler = GraphCompiler( + self.model, lambda model, x, y: (_loss(model, x, y), {"main": _loss(model, x, y).detach()}), + pass_config=self.config, + ) + with patch("hyper_parallel.compile.compiler.PassPipeline.run", + side_effect=lambda graph, **kwargs: Executable(graph)): + x, y = torch.randn(3, 4), torch.randn(3, 3) + actual, losses = compiler.forward_backward(x=x, y=y) + self.assertEqual(compiler._joint_graph.graph_module.calls, 1) + torch.testing.assert_close(actual, losses["main"], rtol=0, atol=0) + expected = torch.autograd.grad(_loss(self.model, x, y), tuple(self.model.parameters())) + for parameter, gradient in zip(self.model.parameters(), expected): + torch.testing.assert_close(parameter.grad, gradient, rtol=0, atol=0) + + def test_module_and_state_kind_changes_invalidate_capture(self): + """Reusing a tensor does not preserve a changed module or parameter contract.""" + class ShiftedLinear(torch.nn.Module): + """A distinct forward with the same parameter names and shapes.""" + + def __init__(self, weight): + super().__init__() + self.weight = weight + + def forward(self, inputs): + return inputs @ self.weight.T + 1 + + model = torch.nn.Sequential(torch.nn.Linear(4, 3, bias=False)) + compiler = GraphCompiler(model, _loss, pass_config=self.config) + inputs = {"x": torch.randn(3, 4), "y": torch.randn(3, 3)} + compiler.forward_backward(**inputs) + replacement = ShiftedLinear(model[0].weight) + model[0] = replacement + with self.assertRaisesRegex(ValueError, "Graph model state"): + compiler.forward_backward(**inputs) + compiler.compile(**inputs) + parameter = model[0].weight + delattr(model[0], "weight") + model[0].register_buffer("weight", parameter) + with self.assertRaisesRegex(ValueError, "Graph model state"): + compiler.forward_backward(**inputs) + def _assert_step(self, compiler, batch, sequence): x = torch.randn(batch, sequence, 4) y = torch.randn(batch, sequence, 3) @@ -225,12 +324,12 @@ def test_sample_storage_is_not_retained(self): def test_static_default_unchanged(self): """ Feature: Symbolic graph shape contracts. - Description: Opting out retains concrete tensor metadata and no symbolic guards. + Description: Opting out retains concrete tensor metadata and static input guards. Expectation: Dynamic dimensions reuse graphs and static dimensions retain guards. """ compiler = self._compiler() self._assert_step(compiler, 2, 5) - self.assertIsNone(compiler._joint_graph.input_guards) + self.assertIsNone(compiler._joint_graph.input_guards.shape_env) for dim in compiler._joint_graph.example_inputs[2].shape: self.assertIs(type(dim), int) From 11a9f2f9ca53d0706b06648ff2b02149696c8131 Mon Sep 17 00:00:00 2001 From: yu-chen-code Date: Fri, 6 Nov 2026 22:49:45 +0800 Subject: [PATCH 2/2] feat(compile): project joint backend and FSDP bucket configuration --- hyper_parallel/compile/compiler.py | 12 +++- hyper_parallel/compile/pass_config.py | 4 ++ .../compile/passes/inductor_pass.py | 7 ++- .../compile/passes/parallel/fsdp_pass.py | 9 +++ hyper_parallel/models/build_options.py | 22 ++++++- tests/ut/auto_models/test_build_options.py | 14 +++++ tests/ut/auto_models/test_public_api.py | 4 ++ tests/ut/compile/test_inductor_pass.py | 58 ++++++++++++++++--- 8 files changed, 119 insertions(+), 11 deletions(-) diff --git a/hyper_parallel/compile/compiler.py b/hyper_parallel/compile/compiler.py index 45633f4d71..a05425dea1 100644 --- a/hyper_parallel/compile/compiler.py +++ b/hyper_parallel/compile/compiler.py @@ -168,6 +168,13 @@ def is_compiled(self) -> bool: """Whether a joint graph has already been compiled.""" return self._joint_graph is not None + @property + def execution_backend(self) -> Optional[str]: + """Actual terminal backend mode; None until a graph is ready.""" + if self._joint_graph is None: + return None + return getattr(self._joint_graph.graph_module, "meta", {}).get("inductor_compiled", "fx") + @property def parameter_layouts(self) -> Optional[dict]: """Final parameter-identity layouts, available after lazy compilation. @@ -257,9 +264,11 @@ def compile(self, **inputs: Any) -> None: # A pass may replace the module, e.g. when installing an executable # backend wrapper. Keep its returned graph as the runtime executable. runtime_graph = pipeline.run(joint_graph.graph_module, **pass_kwargs) + if getattr(runtime_graph, "meta", {}).get("inductor_compiled") == "unavailable": + raise RuntimeError("The requested joint-graph Inductor backend is unavailable in this torch build") for name in ( "state_fqns", "state_is_param", "num_state_inputs", "user_inputs_spec", - "loss_dict_keys", "fsdp_sharded_param_names", + "loss_dict_keys", "fsdp_sharded_param_names", "meta", ): if not hasattr(runtime_graph, name) and hasattr(joint_graph.graph_module, name): setattr(runtime_graph, name, getattr(joint_graph.graph_module, name)) @@ -270,6 +279,7 @@ def compile(self, **inputs: Any) -> None: if joint_graph.state_guards is not None: joint_graph.state_guards.refresh(self.model) self._joint_graph = joint_graph + _LOG.info("Joint graph ready: execution_backend=%s", self.execution_backend) def forward_backward(self, **inputs: Any) -> Any: """ diff --git a/hyper_parallel/compile/pass_config.py b/hyper_parallel/compile/pass_config.py index d1def58abf..d42fe17ac1 100644 --- a/hyper_parallel/compile/pass_config.py +++ b/hyper_parallel/compile/pass_config.py @@ -569,6 +569,10 @@ def build_pass_config_from_trainer_config( compile_config, "fsdp_overlap_grad_reduce", True ), max_in_flight_gb=getattr(compile_config, "max_in_flight_gb", 0.25), + fsdp_bucketing=getattr(compile_config, "fsdp_bucketing", False), + fsdp_bucket_modules=getattr(compile_config, "fsdp_bucket_modules", None), + inductor_mode=getattr(compile_config, "inductor_mode", "off"), + inductor_configs=getattr(compile_config, "inductor_configs", None), ac_mode=ac_mode, ac_layer_count=ac_layer_count, ac_layer_indices=ac_layer_indices, diff --git a/hyper_parallel/compile/passes/inductor_pass.py b/hyper_parallel/compile/passes/inductor_pass.py index af5f71a820..3294abc29c 100644 --- a/hyper_parallel/compile/passes/inductor_pass.py +++ b/hyper_parallel/compile/passes/inductor_pass.py @@ -335,7 +335,7 @@ def run( "JointGraph.example_inputs) to size compiled regions" ) if _regional_inductor_available(): - self._compile_graph(graph_module, pass_config, example_inputs) + graph_module = self._compile_graph(graph_module, pass_config, example_inputs) compiled = True else: self._warn_unavailable(mode) @@ -417,6 +417,11 @@ def _compile_graph( _propagate_fake_vals(graph_module) if example_inputs is None: example_inputs = _placeholder_fake_args(graph_module) + else: + # Partitioning may have changed the state storage since tracing. + state_count = getattr(graph_module, "num_state_inputs", 0) + state_inputs = tuple(_placeholder_fake_args(graph_module)[:state_count]) + example_inputs = state_inputs + tuple(example_inputs[state_count:]) _LOG.info( "InductorPass: compiling graph in %r mode (%d placeholders, %d nodes)", mode, diff --git a/hyper_parallel/compile/passes/parallel/fsdp_pass.py b/hyper_parallel/compile/passes/parallel/fsdp_pass.py index 9fd08182e8..f4d7c1d51e 100644 --- a/hyper_parallel/compile/passes/parallel/fsdp_pass.py +++ b/hyper_parallel/compile/passes/parallel/fsdp_pass.py @@ -439,6 +439,15 @@ def run( len(param_nodes), ) + # Backend propagation starts from runtime shard inputs, rather than the + # full logical tensors used to capture the forward/backward operators. + parameters = dict(model.named_parameters()) + for node in param_nodes: + value = node.meta.get("val") + parameter = parameters.get(node.meta.get("param_name")) + if isinstance(value, torch.Tensor) and parameter is not None: + node.meta["val"] = value.new_empty(parameter.shape, requires_grad=value.requires_grad) + graph_module.recompile() return graph_module diff --git a/hyper_parallel/models/build_options.py b/hyper_parallel/models/build_options.py index 58d8e0c400..1a4129e21e 100644 --- a/hyper_parallel/models/build_options.py +++ b/hyper_parallel/models/build_options.py @@ -138,6 +138,11 @@ class CompileConfig: fsdp_allgather_prefetch: bool = True fsdp_overlap_grad_reduce: bool = True max_in_flight_gb: float = 0.25 + # Joint-graph backend options; decoder torch.compile uses backend/mode/options above. + inductor_mode: Literal["off", "regional", "full"] = "off" + inductor_configs: Optional[dict[str, Any]] = None + fsdp_bucketing: bool = False + fsdp_bucket_modules: Optional[list[str | list[str]]] = None def selects_graph_compiler(self) -> bool: """Return whether this config selects the graph-compiler trainer path.""" @@ -147,12 +152,13 @@ def __post_init__(self) -> None: """Validate values that the YAML resolver cannot express precisely.""" boolean_fields = ( "enabled", "use_joint_graph", "fullgraph", "dynamic", "fsdp_allgather_prefetch", - "fsdp_overlap_grad_reduce", + "fsdp_overlap_grad_reduce", "fsdp_bucketing", ) for name in boolean_fields: if not isinstance(getattr(self, name), bool): raise TypeError(f"compile.{name} must be a bool") self._validate_max_in_flight_gb() + self._validate_joint_graph_options() if self.dynamic_arg_dims is not None and not isinstance(self.dynamic_arg_dims, dict): raise ValueError("compile.dynamic_arg_dims must be a mapping or None") if not isinstance(self.mode, str) or not self.mode.strip(): @@ -185,6 +191,20 @@ def _validate_max_in_flight_gb(self) -> None: ): raise ValueError("compile.max_in_flight_gb must be a finite non-negative number") + def _validate_joint_graph_options(self) -> None: + """Validate the backend and bucketing options consumed by graph passes.""" + if self.inductor_mode not in ("off", "regional", "full"): + raise ValueError("compile.inductor_mode must be 'off', 'regional' or 'full'") + if self.inductor_configs is not None and ( + not isinstance(self.inductor_configs, dict) + or any(not isinstance(key, str) for key in self.inductor_configs) + ): + raise ValueError("compile.inductor_configs must be a mapping with string keys or None") + if self.inductor_configs and self.inductor_mode == "off": + raise ValueError("compile.inductor_configs requires an enabled compile.inductor_mode") + if self.fsdp_bucket_modules is not None and not isinstance(self.fsdp_bucket_modules, list): + raise ValueError("compile.fsdp_bucket_modules must be a list or None") + @dataclass class FSDP2MixedPrecisionConfig: diff --git a/tests/ut/auto_models/test_build_options.py b/tests/ut/auto_models/test_build_options.py index b8afe7d33a..ec4ff780da 100644 --- a/tests/ut/auto_models/test_build_options.py +++ b/tests/ut/auto_models/test_build_options.py @@ -64,6 +64,10 @@ def test_compile_config_fields(self): ("fsdp_allgather_prefetch", "True"), ("fsdp_overlap_grad_reduce", "True"), ("max_in_flight_gb", "0.25"), + ("inductor_mode", "'off'"), + ("inductor_configs", "None"), + ("fsdp_bucketing", "False"), + ("fsdp_bucket_modules", "None"), ], ) @@ -117,6 +121,16 @@ def test_compile_config_validation(self): CompileConfig(mode="reduce-overhead", options={"triton": True}) with self.assertRaisesRegex(ValueError, "dynamo_cache_size_limit"): CompileConfig(dynamo_cache_size_limit=0) + for options in ( + {"inductor_mode": "unknown"}, + {"inductor_configs": {1: True}}, + {"inductor_configs": {"max_autotune": True}}, + {"fsdp_bucket_modules": "layers"}, + ): + with self.subTest(options=options), self.assertRaises(ValueError): + CompileConfig(**options) + with self.assertRaisesRegex(TypeError, "fsdp_bucketing"): + CompileConfig(fsdp_bucketing=1) @arg_mark(plat_marks=["cpu_linux", "cpu_macos"], level_mark="level0", card_mark="allcards", essential_mark="essential") diff --git a/tests/ut/auto_models/test_public_api.py b/tests/ut/auto_models/test_public_api.py index 457a0ac280..2a1d7c3785 100644 --- a/tests/ut/auto_models/test_public_api.py +++ b/tests/ut/auto_models/test_public_api.py @@ -212,6 +212,10 @@ def test_compile_config_fields(self): ("fsdp_allgather_prefetch", "True"), ("fsdp_overlap_grad_reduce", "True"), ("max_in_flight_gb", "0.25"), + ("inductor_mode", "'off'"), + ("inductor_configs", "None"), + ("fsdp_bucketing", "False"), + ("fsdp_bucket_modules", "None"), ], ) diff --git a/tests/ut/compile/test_inductor_pass.py b/tests/ut/compile/test_inductor_pass.py index 10c206adb9..6db6088ba6 100644 --- a/tests/ut/compile/test_inductor_pass.py +++ b/tests/ut/compile/test_inductor_pass.py @@ -27,9 +27,8 @@ itself; every per-stage forward/backward subgraph is compiled instead (both ``regional`` and ``full``), while unknown ``call_module`` stubs are still rejected. -4. Guards: missing ``example_inputs``, mode canonicalization, pipeline - placement (terminal), and that inductor settings stay PassConfig-owned - (PP-style) instead of being projected from ``CompileConfig``. +4. Guards: missing ``example_inputs``, mode canonicalization, terminal pipeline + placement and explicit joint backend options projected from ``CompileConfig``. """ # The backend env var must be set before torch / hyper_parallel are imported, @@ -54,6 +53,7 @@ install_regional_inductor_stub() from hyper_parallel.compile.compiler import GraphCompiler +from hyper_parallel.compile.passes.parallel.fsdp_pass import FSDPPass from hyper_parallel.compile.pass_config import ( PassConfig, build_pass_config_from_trainer_config, @@ -447,7 +447,7 @@ def test_pipeline_allows_pp_with_inductor(self): self.assertEqual(joint.graph_module.meta.get("inductor_compiled"), "regional") def test_trainer_config_projection_strategy(self): - """Test inductor config is PassConfig-owned (PP-style), not projected.""" + """Project explicit joint backend options without reusing decoder options.""" trainer_config = SimpleNamespace( accelerator=SimpleNamespace( tp_size=1, sequence_parallel=False, loss_parallel=False @@ -458,13 +458,55 @@ def test_trainer_config_projection_strategy(self): compile=SimpleNamespace( inductor_mode="regional", inductor_configs={"reorder_for_peak_memory": True}, + fsdp_bucketing=True, + fsdp_bucket_modules=["layers.0", ["layers.1", "layers.2"]], + backend="decoder_backend", + options={"decoder_only": True}, ), ) config = build_pass_config_from_trainer_config(trainer_config) - # Like PP, inductor settings live on PassConfig only; the projected - # config keeps the defaults and ignores CompileConfig-level overrides. - self.assertEqual(config.inductor_mode, "off") - self.assertIsNone(config.inductor_configs) + self.assertEqual(config.inductor_mode, "regional") + self.assertEqual(config.inductor_configs, {"reorder_for_peak_memory": True}) + self.assertTrue(config.fsdp_bucketing) + self.assertEqual(config.fsdp_bucket_modules, ["layers.0", ["layers.1", "layers.2"]]) + + def test_backend_retains_replacement_and_rejects_unavailable(self): + """Retain terminal wrappers and never report unavailable lowering as compiled.""" + joint = self._traced() + wrapper = nn.Module() + wrapper.meta = {} + with mock.patch.object(InductorPass, "_compile_graph", return_value=wrapper): + result = InductorPass().run( + joint.graph_module, PassConfig(inductor_mode="full"), example_inputs=joint.example_inputs + ) + self.assertIs(result, wrapper) + self.assertEqual(result.meta["inductor_compiled"], "full") + compiler = GraphCompiler( + _make_model(), _train_fn, pass_config=PassConfig(fsdp_enabled=False, inductor_mode="full") + ) + with mock.patch("hyper_parallel.compile.passes.inductor_pass._regional_inductor_available", + return_value=False): + with self.assertRaisesRegex(RuntimeError, "unavailable"): + x, y = _fixed_batch() + compiler.compile(x=x, y=y) + self.assertFalse(compiler.is_compiled) + + def test_fsdp_placeholder_metadata_describes_runtime_storage(self): + """Backend shape propagation must not all-gather a full parameter a second time.""" + model = nn.Linear(4, 5) + joint = trace_model_graph(model, _train_fn, {"x": torch.randn(2, 4), "y": torch.randn(2, 5)}) + transport = mock.MagicMock() + transport.is_initialized.return_value = True + transport.get_world_size.return_value = 2 + transport.get_rank.return_value = 1 + with mock.patch("hyper_parallel.compile.passes.parallel.fsdp_pass.dist", transport), \ + mock.patch("hyper_parallel.compile.passes.parallel.fsdp_pass._resolve_process_group"): + result = FSDPPass().run(joint.graph_module, PassConfig(fsdp_degree=2), model=model) + parameters = dict(model.named_parameters()) + for node in result.graph.find_nodes(op="placeholder"): + name = node.meta.get("param_name") + if name in parameters: + self.assertEqual(tuple(node.meta["val"].shape), tuple(parameters[name].shape)) if __name__ == "__main__":