Repository navigation
Expand file tree
/
Copy pathtrainer.py
More file actions
124 lines (108 loc) · 4.81 KB
/
Copy pathtrainer.py
File metadata and controls
124 lines (108 loc) · 4.81 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
# Copyright 2026 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Aggregate trainer configuration tree.
Split from ``auto_models/trainer/config.py`` in stage 7 (05 §15.2.5);
class name, fields and defaults are unchanged.
"""
from dataclasses import dataclass, field, fields
from typing import Any, List, Literal, Optional
from hyper_parallel.models.build_options import CompileConfig, FSDP2Config
from hyper_parallel.components.checkpoint.config import CheckpointingConfig
from hyper_parallel.trainer.config.data import DataLoaderConfig, DatasetConfig
from hyper_parallel.trainer.config.optimization import (
MixedPrecisionConfig,
OptimizerConfig,
)
from hyper_parallel.trainer.config.parallelism import (
AcceleratorConfig,
ActivationCheckpointConfig,
PlanOverride,
)
from hyper_parallel.trainer.config.target import Target, _serialize_config_value
from hyper_parallel.trainer.config.stress_detect import StressDetectConfig
from hyper_parallel.trainer.config.training import (
DebugConfig,
MemoryConfig,
ModelIntegrationConfig,
ProfilingConfig,
TrainingConfig,
WandbConfig,
)
@dataclass
class TrainerConfig:
"""Resolved component tree; runtime objects are built by the task trainer."""
model: Target[Any]
optimizer: OptimizerConfig
lr_scheduler: Optional[Target[Any]] = None
loss_fn: Optional[Target[Any]] = None
# Final floating-point dtype after model weights are loaded or initialized
# from scratch. None preserves the dtype produced by the initialization path.
model_init_dtype: Optional[Literal["float16", "bfloat16", "float32"]] = None
training: TrainingConfig = field(default_factory=TrainingConfig)
# parallelism configs
accelerator: AcceleratorConfig = field(default_factory=AcceleratorConfig)
fsdp_config: FSDP2Config = field(default_factory=FSDP2Config)
plan_overrides: List[PlanOverride] = field(default_factory=list)
mixed_precision: MixedPrecisionConfig = field(
default_factory=MixedPrecisionConfig
)
activation_checkpoint: ActivationCheckpointConfig = field(
default_factory=ActivationCheckpointConfig
)
activation_swap: Literal["none", "attention"] = "none"
compile: CompileConfig = field(default_factory=CompileConfig)
# data
dataset: Optional[DatasetConfig] = None
dataloader: Optional[DataLoaderConfig] = None
packed_sequence: Optional[Any] = None
checkpoint: CheckpointingConfig = field(default_factory=CheckpointingConfig)
debug: DebugConfig = field(default_factory=DebugConfig)
model_integration: ModelIntegrationConfig = field(default_factory=ModelIntegrationConfig)
memory: MemoryConfig = field(default_factory=MemoryConfig)
wandb: WandbConfig = field(default_factory=WandbConfig)
profiler: ProfilingConfig = field(default_factory=ProfilingConfig)
stress_detect: StressDetectConfig = field(default_factory=StressDetectConfig)
magi: Optional[Any] = None
peft: Optional[Any] = None
def __post_init__(self) -> None:
"""Validate combinations that span multiple config sections."""
if (
self.compile.enabled
and not self.compile.selects_graph_compiler()
and self.accelerator.pp_size > 1
):
raise ValueError("compile is not supported together with pipeline parallelism")
reduce_dtype = self.fsdp_config.mix_precision.reduce_dtype
if self.optimizer.fp32_main_params and reduce_dtype != "float32":
raise ValueError(
"optimizer.fp32_main_params=true requires "
"fsdp_config.mix_precision.reduce_dtype='float32'; "
f"got {reduce_dtype!r}"
)
def to_dict(self) -> dict[str, Any]:
"""Serialize the resolved trainer configuration for logging."""
return {
config_field.name: _serialize_config_value(
getattr(self, config_field.name)
)
for config_field in fields(self)
}
def save_configs(config: TrainerConfig, output_dir: str) -> None:
"""Accept trainer config persistence requests without writing files.
Args:
config: Resolved trainer configuration.
output_dir: Intended configuration output directory.
"""
del config, output_dir