@@ -138,7 +138,7 @@ def _annotate_ag_spec(node: fx.Node, degree: int, params: dict) -> None:
138138 node .meta ["fsdp_bucket_spec" ] = {
139139 "input_shape" : shape ,
140140 "output_shape" : (shape [0 ] * degree , * shape [1 :]),
141- "dtype" : param .dtype ,
141+ "dtype" : node . meta . get ( "param_dtype" , param .dtype ) ,
142142 "device" : str (param .device ),
143143 }
144144
@@ -168,6 +168,18 @@ def annotate_fsdp_bucket_specs(graph_module: fx.GraphModule, model: nn.Module) -
168168 _annotate_ag_spec (node , degree , params )
169169
170170
171+ def _cast_node (graph : fx .Graph , node : fx .Node , dtype : Optional [torch .dtype ]) -> fx .Node :
172+ """Insert a precision conversion while retaining fake dtype/shape metadata."""
173+ value = node .meta .get ("val" )
174+ if dtype is None or (value is not None and value .dtype == dtype ):
175+ return node
176+ cast = graph .call_method ("to" , args = (node ,), kwargs = {"dtype" : dtype })
177+ cast .meta ["autograd_backward" ] = node .meta .get ("autograd_backward" , False )
178+ if isinstance (value , FakeTensor ):
179+ cast .meta ["val" ] = value .to (dtype )
180+ return cast
181+
182+
171183def _free_tensor_storage (tensor : torch .Tensor ) -> None :
172184 """Release ``tensor``'s storage in place (FSDP reshard free step).
173185
@@ -338,6 +350,7 @@ def __init__(
338350 self ._parameter_groups : Dict [str , tuple [str , int ]] = {}
339351 self ._storage_prepared = False
340352 self ._logical_parameter_shapes : Dict [str , tuple ] = {}
353+ self ._parameter_dtypes : Dict [str , tuple ] = {}
341354 # Data-parallel mode ("fsdp" | "ddp" | "hsdp") and the replicate-axis
342355 # group/degree used by DDP / HSDP gradient reduction.
343356 self ._dp_mode = "fsdp"
@@ -496,6 +509,7 @@ def _setup_dp_config(
496509 self ._parameter_groups = dict (kwargs .get ("parameter_groups" , {}))
497510 self ._storage_prepared = kwargs .get ("storage_prepared" , False )
498511 self ._logical_parameter_shapes = kwargs .get ("logical_parameter_shapes" , {})
512+ self ._parameter_dtypes = kwargs .get ("parameter_dtypes" , {})
499513 if not self ._expert_param_names .issubset (self ._gradient_reductions ):
500514 raise ValueError ("Expert parameters require explicit gradient reduction policies" )
501515 self ._expert_fsdp_group_name = kwargs .get (
@@ -729,7 +743,9 @@ def _mark_sharded_param(
729743 # Newly inserted collective nodes have no shape-propagated ``val``.
730744 value = param if param is not None else node .meta .get ("val" )
731745 if isinstance (value , torch .Tensor ) and isinstance (value .numel (), int ):
732- node .meta ["fsdp_unsharded_nbytes" ] = value .numel () * value .element_size ()
746+ dtype = self ._parameter_dtypes .get (fqn , (value .dtype , None ))[0 ]
747+ element_size = torch .empty ((), device = "meta" , dtype = dtype ).element_size ()
748+ node .meta ["fsdp_unsharded_nbytes" ] = value .numel () * element_size
733749 self ._fsdp_modules .add (self ._get_parent_module_fqn (fqn ))
734750 return node
735751
@@ -984,10 +1000,14 @@ def _insert_all_gather_for_params(
9841000 # AllGather + immediate wait for correctness; FsdpOverlapPass may
9851001 # sink the wait past independent compute later.
9861002 with _insertion_point (graph , anchor , param_node ):
1003+ dtype = self ._parameter_dtypes .get (param_node .meta .get ("param_name" ), (None , None ))[0 ]
1004+ gather_input = _cast_node (graph , param_node , dtype )
9871005 ag_node = graph .call_function (
9881006 _c10d_functional .all_gather_into_tensor ,
989- args = (param_node , degree , group_name ),
1007+ args = (gather_input , degree , group_name ),
9901008 )
1009+ if dtype is not None :
1010+ ag_node .meta ["param_dtype" ] = dtype
9911011 ag_node .meta ["comm_type" ] = "fsdp_all_gather"
9921012 ag_node .meta ["comm_group" ] = group_name
9931013 ag_node .meta ["param_node" ] = param_node .name
@@ -1036,7 +1056,14 @@ def _insert_all_gather_for_params(
10361056 replacement .meta ["state_idx" ] = param_node .meta .get ("state_idx" )
10371057
10381058 for user in list (param_node .users .keys ()):
1039- if user not in (ag_node , wait_node ):
1059+ if user in (gather_input , ag_node , wait_node ):
1060+ continue
1061+ if str (user .target ) == "aten._to_copy.default" and user .kwargs == {"dtype" : dtype }:
1062+ # The shard was cast before gather; do not retain a second
1063+ # full-size compute copy that defeats resharding.
1064+ user .replace_all_uses_with (replacement )
1065+ graph .erase_node (user )
1066+ else :
10401067 user .replace_input_with (param_node , replacement )
10411068
10421069 self ._processed_params .add (param_node .name )
@@ -1149,6 +1176,8 @@ def _insert_grad_reduction( # pylint: disable=too-many-locals
11491176 group_name , degree , self ._gradient_reductions .get (fqn , _DEFAULT_GRADIENT_REDUCTION )
11501177 )
11511178 with graph .inserting_before (output_node ):
1179+ dtype = self ._parameter_dtypes .get (fqn , (None , None ))[1 ]
1180+ grad_node = _cast_node (graph , grad_node , dtype )
11521181 new_returned [i ] = self ._build_grad_reduction_nodes (
11531182 graph ,
11541183 grad_node ,
@@ -1449,10 +1478,14 @@ def _insert_backward_gather(
14491478 record .param_node .meta .get ("param_name" , "" )
14501479 )
14511480 with graph .inserting_before (anchor ):
1481+ dtype = self ._parameter_dtypes .get (record .param_node .meta .get ("param_name" ), (None , None ))[0 ]
1482+ gather_input = _cast_node (graph , record .param_node , dtype )
14521483 ag_node = graph .call_function (
14531484 _c10d_functional .all_gather_into_tensor ,
1454- args = (record . param_node , degree , group_name ),
1485+ args = (gather_input , degree , group_name ),
14551486 )
1487+ if dtype is not None :
1488+ ag_node .meta ["param_dtype" ] = dtype
14561489 ag_node .meta ["comm_type" ] = "fsdp_all_gather_backward"
14571490 ag_node .meta ["comm_group" ] = group_name
14581491 ag_node .meta ["param_node" ] = record .param_node .name
0 commit comments