From 3f7fc2a1a75fb597e3bfb17c7ed95a393cb5b7ab Mon Sep 17 00:00:00 2001 From: Summer Date: Tue, 20 Oct 2026 21:57:26 +0000 Subject: [PATCH 1/2] Add qwen35_397b_grpo reference RCP logs for GBS 1152 (6 seeds) --- llm_post_training/README.md | 5 +- llm_post_training/rcp_logs/GBS1152/seed_1.out | 64 +++++++++++++++++++ llm_post_training/rcp_logs/GBS1152/seed_2.out | 64 +++++++++++++++++++ llm_post_training/rcp_logs/GBS1152/seed_3.out | 64 +++++++++++++++++++ llm_post_training/rcp_logs/GBS1152/seed_4.out | 64 +++++++++++++++++++ llm_post_training/rcp_logs/GBS1152/seed_5.out | 64 +++++++++++++++++++ llm_post_training/rcp_logs/GBS1152/seed_6.out | 64 +++++++++++++++++++ 7 files changed, 388 insertions(+), 1 deletion(-) create mode 100644 llm_post_training/rcp_logs/GBS1152/seed_1.out create mode 100644 llm_post_training/rcp_logs/GBS1152/seed_2.out create mode 100644 llm_post_training/rcp_logs/GBS1152/seed_3.out create mode 100644 llm_post_training/rcp_logs/GBS1152/seed_4.out create mode 100644 llm_post_training/rcp_logs/GBS1152/seed_5.out create mode 100644 llm_post_training/rcp_logs/GBS1152/seed_6.out diff --git a/llm_post_training/README.md b/llm_post_training/README.md index d4c60e3ea..3c7c39035 100644 --- a/llm_post_training/README.md +++ b/llm_post_training/README.md @@ -396,6 +396,7 @@ The target-0.69 RCP configurations are: | 256 | 16 x 16 | `1.0e-6` | `0.125` | 18 | 30 | | 512 | 32 x 16 | `1.4142135624e-6` | `0.08838834765` | 10 | 20 | | 1024 | 64 x 16 | `2.0e-6` | `0.0625` | 7 | 10 | +| 1152 | 72 x 16 | `2.1213203436e-6` | `0.05892556510` | 6 | 10 | The learning-rate and clipping schedules represented by this table are: @@ -748,8 +749,9 @@ The checked-in RCP logs provide six independent seeds at each qualified GBS: | 256 | `rcp_logs/256/seed_*.out` | 6 / 6 | Step 18 | 0.703125 - 0.75390625 | | 512 | `rcp_logs/512/seed_*.out` | 6 / 6 | Step 10 | 0.6953125 - 0.73046875 | | 1024 | `rcp_logs/1024/seed_*.out` | 6 / 6 | Step 7 | 0.70703125 - 0.75000000 | +| 1152 | `rcp_logs/GBS1152/seed_*.out` | 6 / 6 | Step 6 | 0.72908366 - 0.78087652 | -All 18 runs reach the target at the first scheduled validation. This is stable +All 24 runs reach the target at the first scheduled validation. This is stable observed convergence at the configured evaluation schedule; it does not imply zero latent crossing-step variance before the first observation. @@ -765,6 +767,7 @@ qualified time-to-target run. | 256 | No | 18 | Every step | No | 30 | | 512 | No | 10 | Every step | No | 20 | | 1024 | No | 7 | Every step | No | 10 | +| 1152 | No | 6 | Every step | No | 10 | The validation dataloader uses the entire 251-task validation JSONL at every evaluation. The four trajectories for each task use validation temperature diff --git a/llm_post_training/rcp_logs/GBS1152/seed_1.out b/llm_post_training/rcp_logs/GBS1152/seed_1.out new file mode 100644 index 000000000..b984851c9 --- /dev/null +++ b/llm_post_training/rcp_logs/GBS1152/seed_1.out @@ -0,0 +1,64 @@ +:::MLLOG {"namespace": "", "time_ms": 1792395580842, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "", "lineno": 3}} +:::MLLOG {"namespace": "", "time_ms": 1792395596065, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597621, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "qwen35_397b_grpo", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792395597621, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "Google", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792395597621, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792395597622, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792395597622, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "64xa4x", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792395597622, "event_type": "POINT_IN_TIME", "key": "seed", "value": 1885626193, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597623, "event_type": "POINT_IN_TIME", "key": "max_steps", "value": 10, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597623, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 1152, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597623, "event_type": "POINT_IN_TIME", "key": "micro_batch_size", "value": 1, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597623, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 65536, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597623, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 11200, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597623, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 251, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597623, "event_type": "POINT_IN_TIME", "key": "init_checkpoint_step", "value": 0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597624, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "adamw", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597624, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2.121320343559642e-06, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597624, "event_type": "POINT_IN_TIME", "key": "opt_end_learning_rate", "value": 2.121320343559642e-06, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597624, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_1", "value": 0.9, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597624, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_2", "value": 0.999, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597624, "event_type": "POINT_IN_TIME", "key": "opt_adamw_epsilon", "value": 1e-08, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597624, "event_type": "POINT_IN_TIME", "key": "opt_adamw_weight_decay", "value": 0.0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597625, "event_type": "POINT_IN_TIME", "key": "opt_gradient_clip_norm", "value": 0.05892556509887896, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597625, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597625, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_steps", "value": 100000, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597625, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_schedule", "value": "constant", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597625, "event_type": "POINT_IN_TIME", "key": "tensor_parallelism", "value": 4, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597625, "event_type": "POINT_IN_TIME", "key": "pipeline_parallelism", "value": 2, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597625, "event_type": "POINT_IN_TIME", "key": "context_parallelism", "value": 1, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597625, "event_type": "POINT_IN_TIME", "key": "expert_parallelism", "value": 32, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597626, "event_type": "POINT_IN_TIME", "key": "generation_backend", "value": "vllm", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597626, "event_type": "POINT_IN_TIME", "key": "generation_tensor_parallelism", "value": 8, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597626, "event_type": "POINT_IN_TIME", "key": "generation_pipeline_parallelism", "value": 1, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597626, "event_type": "POINT_IN_TIME", "key": "generation_expert_parallelism", "value": 8, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597626, "event_type": "POINT_IN_TIME", "key": "generation_training_rollout_temperature", "value": 1.0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597626, "event_type": "POINT_IN_TIME", "key": "generation_training_rollout_top_p", "value": 1.0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597626, "event_type": "POINT_IN_TIME", "key": "generation_validation_rollout_temperature", "value": 0.1, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597626, "event_type": "POINT_IN_TIME", "key": "generation_validation_rollout_top_p", "value": 0.95, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597627, "event_type": "POINT_IN_TIME", "key": "num_prompts_per_step", "value": 72, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597627, "event_type": "POINT_IN_TIME", "key": "num_generations_per_prompt", "value": 16, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597627, "event_type": "POINT_IN_TIME", "key": "truncated_importance_sampling_ratio_min", "value": 0.999, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597627, "event_type": "POINT_IN_TIME", "key": "truncated_importance_sampling_ratio", "value": 1.002, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597627, "event_type": "POINT_IN_TIME", "key": "truncated_importance_sampling_type", "value": "seq-mask-tis", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395597627, "event_type": "POINT_IN_TIME", "key": "target_accuracy", "value": 0.69, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395939770, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 144, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792395939842, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 291}} +:::MLLOG {"namespace": "", "time_ms": 1792395939842, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 291}} +:::MLLOG {"namespace": "", "time_ms": 1792395939842, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "step": 0}} +:::MLLOG {"namespace": "", "time_ms": 1792398265750, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.08669434789044317, "reward": 0.3255208432674408, "grad_norm": 0.04490421712398529, "global_valid_toks": 6374981.0, "global_valid_seqs": 1135.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 1152, "step": 1}} +:::MLLOG {"namespace": "", "time_ms": 1792398265761, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 1046.0511003360007, "policy_training_time": 441.2504875110026, "exposed_generation_time": 359.42124389399396, "weight_sync_time": 15.67865129099664, "valid_tokens_per_sec_per_gpu": 23.805978047584077}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 1152, "step": 1}} +:::MLLOG {"namespace": "", "time_ms": 1792399300324, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.07544504791439977, "reward": 0.2795138955116272, "grad_norm": 0.06886813789606094, "global_valid_toks": 6226554.0, "global_valid_seqs": 1133.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 2304, "step": 2}} +:::MLLOG {"namespace": "", "time_ms": 1792399300335, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 934.6668656110487, "policy_training_time": 283.87288653299765, "exposed_generation_time": 549.8984528569927, "weight_sync_time": 15.811214244000439, "valid_tokens_per_sec_per_gpu": 26.022615604971634}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 2304, "step": 2}} +:::MLLOG {"namespace": "", "time_ms": 1792400389497, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.08843275631807046, "reward": 0.3272569477558136, "grad_norm": 0.03901178017258644, "global_valid_toks": 6115870.0, "global_valid_seqs": 1109.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 3456, "step": 3}} +:::MLLOG {"namespace": "", "time_ms": 1792400389507, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 986.5537100890506, "policy_training_time": 286.8524610050008, "exposed_generation_time": 601.8947445570084, "weight_sync_time": 16.254230523998558, "valid_tokens_per_sec_per_gpu": 24.21572889867656}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 3456, "step": 3}} +:::MLLOG {"namespace": "", "time_ms": 1792401489217, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.05171314728067955, "reward": 0.3671875, "grad_norm": 0.032063666731119156, "global_valid_toks": 6746553.0, "global_valid_seqs": 1117.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 4608, "step": 4}} +:::MLLOG {"namespace": "", "time_ms": 1792401489228, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 998.8582938328836, "policy_training_time": 287.20581423099793, "exposed_generation_time": 614.0916588618857, "weight_sync_time": 15.83781676000217, "valid_tokens_per_sec_per_gpu": 26.38384525508998}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 4608, "step": 4}} +:::MLLOG {"namespace": "", "time_ms": 1792402672077, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.043433990846097004, "reward": 0.3880208432674408, "grad_norm": 0.022654572501778603, "global_valid_toks": 6909070.0, "global_valid_seqs": 1139.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 5760, "step": 5}} +:::MLLOG {"namespace": "", "time_ms": 1792402672087, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 1080.1048807139523, "policy_training_time": 291.4241938240011, "exposed_generation_time": 688.9525599839835, "weight_sync_time": 16.256288522999967, "valid_tokens_per_sec_per_gpu": 24.986975958909184}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 5760, "step": 5}} +:::MLLOG {"namespace": "", "time_ms": 1792403752361, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "step": 6}} +:::MLLOG {"namespace": "", "time_ms": 1792403752366, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "step": 6}} +:::MLLOG {"namespace": "", "time_ms": 1792404963418, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"validation_time": 1208.7332950900018}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "step": 6}} +:::MLLOG {"namespace": "", "time_ms": 1792404963427, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7529880404472351, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912}} +:::MLLOG {"namespace": "", "time_ms": 1792404963447, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "step": 6}} +:::MLLOG {"namespace": "", "time_ms": 1792404963462, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "status": "success"}} diff --git a/llm_post_training/rcp_logs/GBS1152/seed_2.out b/llm_post_training/rcp_logs/GBS1152/seed_2.out new file mode 100644 index 000000000..5b27cb00a --- /dev/null +++ b/llm_post_training/rcp_logs/GBS1152/seed_2.out @@ -0,0 +1,64 @@ +:::MLLOG {"namespace": "", "time_ms": 1792409070886, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "", "lineno": 3}} +:::MLLOG {"namespace": "", "time_ms": 1792409085470, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086988, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "qwen35_397b_grpo", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792409086989, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "Google", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792409086989, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792409086989, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792409086989, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "64xa4x", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792409086990, "event_type": "POINT_IN_TIME", "key": "seed", "value": 1885626194, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086990, "event_type": "POINT_IN_TIME", "key": "max_steps", "value": 10, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086990, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 1152, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086990, "event_type": "POINT_IN_TIME", "key": "micro_batch_size", "value": 1, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086990, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 65536, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086991, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 11200, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086991, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 251, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086991, "event_type": "POINT_IN_TIME", "key": "init_checkpoint_step", "value": 0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086991, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "adamw", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086991, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2.121320343559642e-06, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086991, "event_type": "POINT_IN_TIME", "key": "opt_end_learning_rate", "value": 2.121320343559642e-06, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086991, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_1", "value": 0.9, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086991, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_2", "value": 0.999, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086992, "event_type": "POINT_IN_TIME", "key": "opt_adamw_epsilon", "value": 1e-08, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086992, "event_type": "POINT_IN_TIME", "key": "opt_adamw_weight_decay", "value": 0.0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086992, "event_type": "POINT_IN_TIME", "key": "opt_gradient_clip_norm", "value": 0.05892556509887896, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086992, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086992, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_steps", "value": 100000, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086992, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_schedule", "value": "constant", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086992, "event_type": "POINT_IN_TIME", "key": "tensor_parallelism", "value": 4, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086993, "event_type": "POINT_IN_TIME", "key": "pipeline_parallelism", "value": 2, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086993, "event_type": "POINT_IN_TIME", "key": "context_parallelism", "value": 1, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086993, "event_type": "POINT_IN_TIME", "key": "expert_parallelism", "value": 32, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086993, "event_type": "POINT_IN_TIME", "key": "generation_backend", "value": "vllm", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086993, "event_type": "POINT_IN_TIME", "key": "generation_tensor_parallelism", "value": 8, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086993, "event_type": "POINT_IN_TIME", "key": "generation_pipeline_parallelism", "value": 1, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086993, "event_type": "POINT_IN_TIME", "key": "generation_expert_parallelism", "value": 8, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086993, "event_type": "POINT_IN_TIME", "key": "generation_training_rollout_temperature", "value": 1.0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086993, "event_type": "POINT_IN_TIME", "key": "generation_training_rollout_top_p", "value": 1.0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086994, "event_type": "POINT_IN_TIME", "key": "generation_validation_rollout_temperature", "value": 0.1, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086994, "event_type": "POINT_IN_TIME", "key": "generation_validation_rollout_top_p", "value": 0.95, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086994, "event_type": "POINT_IN_TIME", "key": "num_prompts_per_step", "value": 72, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086994, "event_type": "POINT_IN_TIME", "key": "num_generations_per_prompt", "value": 16, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086994, "event_type": "POINT_IN_TIME", "key": "truncated_importance_sampling_ratio_min", "value": 0.999, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086994, "event_type": "POINT_IN_TIME", "key": "truncated_importance_sampling_ratio", "value": 1.002, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086994, "event_type": "POINT_IN_TIME", "key": "truncated_importance_sampling_type", "value": "seq-mask-tis", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409086994, "event_type": "POINT_IN_TIME", "key": "target_accuracy", "value": 0.69, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409439232, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 144, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792409439245, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 291}} +:::MLLOG {"namespace": "", "time_ms": 1792409439245, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 291}} +:::MLLOG {"namespace": "", "time_ms": 1792409439246, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "step": 0}} +:::MLLOG {"namespace": "", "time_ms": 1792411531893, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.0771462738775881, "reward": 0.3342013955116272, "grad_norm": 0.035924144089221954, "global_valid_toks": 6410120.0, "global_valid_seqs": 1125.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 1152, "step": 1}} +:::MLLOG {"namespace": "", "time_ms": 1792411531904, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 852.8131277549983, "policy_training_time": 448.82925375199557, "exposed_generation_time": 158.23929852899892, "weight_sync_time": 15.963463129002776, "valid_tokens_per_sec_per_gpu": 29.361099677154034}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 1152, "step": 1}} +:::MLLOG {"namespace": "", "time_ms": 1792412786147, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.0786270181342843, "reward": 0.2907986044883728, "grad_norm": 0.03894314914941788, "global_valid_toks": 6246860.0, "global_valid_seqs": 1133.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 2304, "step": 2}} +:::MLLOG {"namespace": "", "time_ms": 1792412786159, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 1147.7954790770018, "policy_training_time": 290.0102486329997, "exposed_generation_time": 758.6409980729877, "weight_sync_time": 15.971765525995579, "valid_tokens_per_sec_per_gpu": 21.259708127289954}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 2304, "step": 2}} +:::MLLOG {"namespace": "", "time_ms": 1792413894359, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.048311202495824546, "reward": 0.3177083432674408, "grad_norm": 0.03969188034534454, "global_valid_toks": 6244026.0, "global_valid_seqs": 1137.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 3456, "step": 3}} +:::MLLOG {"namespace": "", "time_ms": 1792413894370, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 997.8777984450135, "policy_training_time": 285.3113960139963, "exposed_generation_time": 615.3612512830732, "weight_sync_time": 16.1170976949943, "valid_tokens_per_sec_per_gpu": 24.442598683433896}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 3456, "step": 3}} +:::MLLOG {"namespace": "", "time_ms": 1792415024192, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.07440516249334905, "reward": 0.3776041567325592, "grad_norm": 0.037770725786685944, "global_valid_toks": 6783848.0, "global_valid_seqs": 1128.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 4608, "step": 4}} +:::MLLOG {"namespace": "", "time_ms": 1792415024203, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 1030.2980118350097, "policy_training_time": 283.4291297390009, "exposed_generation_time": 649.9524009839733, "weight_sync_time": 16.08494757599692, "valid_tokens_per_sec_per_gpu": 25.720137227871867}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 4608, "step": 4}} +:::MLLOG {"namespace": "", "time_ms": 1792416152937, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.017084969229472335, "reward": 0.4262152910232544, "grad_norm": 0.030019046738743782, "global_valid_toks": 6826968.0, "global_valid_seqs": 1141.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 5760, "step": 5}} +:::MLLOG {"namespace": "", "time_ms": 1792416152948, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 1025.4367929960135, "policy_training_time": 291.85446353800216, "exposed_generation_time": 634.4717013639602, "weight_sync_time": 16.12673225400067, "valid_tokens_per_sec_per_gpu": 26.006326213519895}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 5760, "step": 5}} +:::MLLOG {"namespace": "", "time_ms": 1792417139520, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "step": 6}} +:::MLLOG {"namespace": "", "time_ms": 1792417139522, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "step": 6}} +:::MLLOG {"namespace": "", "time_ms": 1792418216584, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"validation_time": 1074.6327412009996}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "step": 6}} +:::MLLOG {"namespace": "", "time_ms": 1792418216586, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7649402618408203, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912}} +:::MLLOG {"namespace": "", "time_ms": 1792418216592, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "step": 6}} +:::MLLOG {"namespace": "", "time_ms": 1792418216593, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "status": "success"}} diff --git a/llm_post_training/rcp_logs/GBS1152/seed_3.out b/llm_post_training/rcp_logs/GBS1152/seed_3.out new file mode 100644 index 000000000..35e45a094 --- /dev/null +++ b/llm_post_training/rcp_logs/GBS1152/seed_3.out @@ -0,0 +1,64 @@ +:::MLLOG {"namespace": "", "time_ms": 1792420449799, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "", "lineno": 3}} +:::MLLOG {"namespace": "", "time_ms": 1792420464765, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466327, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "qwen35_397b_grpo", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792420466327, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "Google", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792420466327, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792420466328, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792420466328, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "64xa4x", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792420466328, "event_type": "POINT_IN_TIME", "key": "seed", "value": 1885626195, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466329, "event_type": "POINT_IN_TIME", "key": "max_steps", "value": 10, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466329, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 1152, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466329, "event_type": "POINT_IN_TIME", "key": "micro_batch_size", "value": 1, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466329, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 65536, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466329, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 11200, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466329, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 251, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466329, "event_type": "POINT_IN_TIME", "key": "init_checkpoint_step", "value": 0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466330, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "adamw", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466330, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2.121320343559642e-06, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466330, "event_type": "POINT_IN_TIME", "key": "opt_end_learning_rate", "value": 2.121320343559642e-06, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466330, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_1", "value": 0.9, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466330, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_2", "value": 0.999, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466330, "event_type": "POINT_IN_TIME", "key": "opt_adamw_epsilon", "value": 1e-08, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466330, "event_type": "POINT_IN_TIME", "key": "opt_adamw_weight_decay", "value": 0.0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466330, "event_type": "POINT_IN_TIME", "key": "opt_gradient_clip_norm", "value": 0.05892556509887896, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466331, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466331, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_steps", "value": 100000, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466331, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_schedule", "value": "constant", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466331, "event_type": "POINT_IN_TIME", "key": "tensor_parallelism", "value": 4, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466331, "event_type": "POINT_IN_TIME", "key": "pipeline_parallelism", "value": 2, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466331, "event_type": "POINT_IN_TIME", "key": "context_parallelism", "value": 1, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466331, "event_type": "POINT_IN_TIME", "key": "expert_parallelism", "value": 32, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466331, "event_type": "POINT_IN_TIME", "key": "generation_backend", "value": "vllm", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466332, "event_type": "POINT_IN_TIME", "key": "generation_tensor_parallelism", "value": 8, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466332, "event_type": "POINT_IN_TIME", "key": "generation_pipeline_parallelism", "value": 1, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466332, "event_type": "POINT_IN_TIME", "key": "generation_expert_parallelism", "value": 8, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466332, "event_type": "POINT_IN_TIME", "key": "generation_training_rollout_temperature", "value": 1.0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466332, "event_type": "POINT_IN_TIME", "key": "generation_training_rollout_top_p", "value": 1.0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466332, "event_type": "POINT_IN_TIME", "key": "generation_validation_rollout_temperature", "value": 0.1, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466332, "event_type": "POINT_IN_TIME", "key": "generation_validation_rollout_top_p", "value": 0.95, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466332, "event_type": "POINT_IN_TIME", "key": "num_prompts_per_step", "value": 72, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466332, "event_type": "POINT_IN_TIME", "key": "num_generations_per_prompt", "value": 16, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466333, "event_type": "POINT_IN_TIME", "key": "truncated_importance_sampling_ratio_min", "value": 0.999, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466333, "event_type": "POINT_IN_TIME", "key": "truncated_importance_sampling_ratio", "value": 1.002, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466333, "event_type": "POINT_IN_TIME", "key": "truncated_importance_sampling_type", "value": "seq-mask-tis", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420466333, "event_type": "POINT_IN_TIME", "key": "target_accuracy", "value": 0.69, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420822057, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 144, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792420822088, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 291}} +:::MLLOG {"namespace": "", "time_ms": 1792420822089, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 291}} +:::MLLOG {"namespace": "", "time_ms": 1792420822089, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "step": 0}} +:::MLLOG {"namespace": "", "time_ms": 1792422978856, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.057937295925512444, "reward": 0.3368055522441864, "grad_norm": 0.03951261565089226, "global_valid_toks": 6362702.0, "global_valid_seqs": 1129.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 1152, "step": 1}} +:::MLLOG {"namespace": "", "time_ms": 1792422978867, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 816.6749021290016, "policy_training_time": 449.5412449440046, "exposed_generation_time": 117.57541634199151, "weight_sync_time": 15.950563087011687, "valid_tokens_per_sec_per_gpu": 30.433535575425367}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 1152, "step": 1}} +:::MLLOG {"namespace": "", "time_ms": 1792424016962, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.05359632449108176, "reward": 0.2717013955116272, "grad_norm": 0.035613011568784714, "global_valid_toks": 6191037.0, "global_valid_seqs": 1134.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 2304, "step": 2}} +:::MLLOG {"namespace": "", "time_ms": 1792424016973, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 931.1672027530876, "policy_training_time": 281.34208702300384, "exposed_generation_time": 553.5728483961138, "weight_sync_time": 16.185765920003178, "valid_tokens_per_sec_per_gpu": 25.97142404688266}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 2304, "step": 2}} +:::MLLOG {"namespace": "", "time_ms": 1792425180425, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.06663928779016715, "reward": 0.3333333432674408, "grad_norm": 0.08338885009288788, "global_valid_toks": 6245576.0, "global_valid_seqs": 1130.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 3456, "step": 3}} +:::MLLOG {"namespace": "", "time_ms": 1792425180436, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 1056.66368091607, "policy_training_time": 279.7323001839977, "exposed_generation_time": 681.452736766063, "weight_sync_time": 16.26199054000608, "valid_tokens_per_sec_per_gpu": 23.088501753792954}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 3456, "step": 3}} +:::MLLOG {"namespace": "", "time_ms": 1792426486341, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.10934331854514312, "reward": 0.3645833432674408, "grad_norm": 0.03438789024949074, "global_valid_toks": 6777864.0, "global_valid_seqs": 1119.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 4608, "step": 4}} +:::MLLOG {"namespace": "", "time_ms": 1792426486352, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 1202.4676912019495, "policy_training_time": 280.0481981220073, "exposed_generation_time": 826.8290657169855, "weight_sync_time": 16.03413219298818, "valid_tokens_per_sec_per_gpu": 22.018081187308557}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 4608, "step": 4}} +:::MLLOG {"namespace": "", "time_ms": 1792427683982, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.06646921163337538, "reward": 0.4166666567325592, "grad_norm": 0.06934916973114014, "global_valid_toks": 6767198.0, "global_valid_seqs": 1108.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 5760, "step": 5}} +:::MLLOG {"namespace": "", "time_ms": 1792427683994, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 1093.5069395960018, "policy_training_time": 291.0466097739991, "exposed_generation_time": 704.4289129771932, "weight_sync_time": 16.112445185994147, "valid_tokens_per_sec_per_gpu": 24.17393637873595}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 5760, "step": 5}} +:::MLLOG {"namespace": "", "time_ms": 1792428672414, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "step": 6}} +:::MLLOG {"namespace": "", "time_ms": 1792428672416, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "step": 6}} +:::MLLOG {"namespace": "", "time_ms": 1792429811273, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"validation_time": 1136.758086604008}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "step": 6}} +:::MLLOG {"namespace": "", "time_ms": 1792429811281, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.737051784992218, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912}} +:::MLLOG {"namespace": "", "time_ms": 1792429811284, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "step": 6}} +:::MLLOG {"namespace": "", "time_ms": 1792429811285, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "status": "success"}} diff --git a/llm_post_training/rcp_logs/GBS1152/seed_4.out b/llm_post_training/rcp_logs/GBS1152/seed_4.out new file mode 100644 index 000000000..449ee833d --- /dev/null +++ b/llm_post_training/rcp_logs/GBS1152/seed_4.out @@ -0,0 +1,64 @@ +:::MLLOG {"namespace": "", "time_ms": 1792432047803, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "", "lineno": 3}} +:::MLLOG {"namespace": "", "time_ms": 1792432062308, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063807, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "qwen35_397b_grpo", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792432063807, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "Google", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792432063807, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792432063807, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792432063808, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "64xa4x", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792432063808, "event_type": "POINT_IN_TIME", "key": "seed", "value": 1885626196, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063809, "event_type": "POINT_IN_TIME", "key": "max_steps", "value": 10, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063809, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 1152, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063809, "event_type": "POINT_IN_TIME", "key": "micro_batch_size", "value": 1, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063809, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 65536, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063809, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 11200, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063809, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 251, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063809, "event_type": "POINT_IN_TIME", "key": "init_checkpoint_step", "value": 0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063809, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "adamw", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063810, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2.121320343559642e-06, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063810, "event_type": "POINT_IN_TIME", "key": "opt_end_learning_rate", "value": 2.121320343559642e-06, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063810, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_1", "value": 0.9, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063810, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_2", "value": 0.999, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063810, "event_type": "POINT_IN_TIME", "key": "opt_adamw_epsilon", "value": 1e-08, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063810, "event_type": "POINT_IN_TIME", "key": "opt_adamw_weight_decay", "value": 0.0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063810, "event_type": "POINT_IN_TIME", "key": "opt_gradient_clip_norm", "value": 0.05892556509887896, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063810, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063811, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_steps", "value": 100000, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063811, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_schedule", "value": "constant", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063811, "event_type": "POINT_IN_TIME", "key": "tensor_parallelism", "value": 4, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063811, "event_type": "POINT_IN_TIME", "key": "pipeline_parallelism", "value": 2, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063811, "event_type": "POINT_IN_TIME", "key": "context_parallelism", "value": 1, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063811, "event_type": "POINT_IN_TIME", "key": "expert_parallelism", "value": 32, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063811, "event_type": "POINT_IN_TIME", "key": "generation_backend", "value": "vllm", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063811, "event_type": "POINT_IN_TIME", "key": "generation_tensor_parallelism", "value": 8, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063812, "event_type": "POINT_IN_TIME", "key": "generation_pipeline_parallelism", "value": 1, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063812, "event_type": "POINT_IN_TIME", "key": "generation_expert_parallelism", "value": 8, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063812, "event_type": "POINT_IN_TIME", "key": "generation_training_rollout_temperature", "value": 1.0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063812, "event_type": "POINT_IN_TIME", "key": "generation_training_rollout_top_p", "value": 1.0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063812, "event_type": "POINT_IN_TIME", "key": "generation_validation_rollout_temperature", "value": 0.1, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063812, "event_type": "POINT_IN_TIME", "key": "generation_validation_rollout_top_p", "value": 0.95, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063812, "event_type": "POINT_IN_TIME", "key": "num_prompts_per_step", "value": 72, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063812, "event_type": "POINT_IN_TIME", "key": "num_generations_per_prompt", "value": 16, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063812, "event_type": "POINT_IN_TIME", "key": "truncated_importance_sampling_ratio_min", "value": 0.999, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063813, "event_type": "POINT_IN_TIME", "key": "truncated_importance_sampling_ratio", "value": 1.002, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063813, "event_type": "POINT_IN_TIME", "key": "truncated_importance_sampling_type", "value": "seq-mask-tis", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432063813, "event_type": "POINT_IN_TIME", "key": "target_accuracy", "value": 0.69, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432412629, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 144, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792432412672, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 291}} +:::MLLOG {"namespace": "", "time_ms": 1792432412672, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 291}} +:::MLLOG {"namespace": "", "time_ms": 1792432412672, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "step": 0}} +:::MLLOG {"namespace": "", "time_ms": 1792434489969, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.06962769507663324, "reward": 0.3142361044883728, "grad_norm": 0.04123876616358757, "global_valid_toks": 6326212.0, "global_valid_seqs": 1129.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 1152, "step": 1}} +:::MLLOG {"namespace": "", "time_ms": 1792434489979, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 781.980356472006, "policy_training_time": 441.98768672200094, "exposed_generation_time": 93.71879272602382, "weight_sync_time": 15.672613694987376, "valid_tokens_per_sec_per_gpu": 31.601517123128215}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 1152, "step": 1}} +:::MLLOG {"namespace": "", "time_ms": 1792435527093, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.06063778587849811, "reward": 0.2881944477558136, "grad_norm": 0.03735434263944626, "global_valid_toks": 6272139.0, "global_valid_seqs": 1126.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 2304, "step": 2}} +:::MLLOG {"namespace": "", "time_ms": 1792435527103, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 932.0932266499585, "policy_training_time": 283.3017112509988, "exposed_generation_time": 549.9561121049337, "weight_sync_time": 15.981026542009204, "valid_tokens_per_sec_per_gpu": 26.285506930253682}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 2304, "step": 2}} +:::MLLOG {"namespace": "", "time_ms": 1792436616585, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.07878219578560675, "reward": 0.3315972089767456, "grad_norm": 0.03562584146857262, "global_valid_toks": 6266207.0, "global_valid_seqs": 1133.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 3456, "step": 3}} +:::MLLOG {"namespace": "", "time_ms": 1792436616595, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 988.8278692489112, "policy_training_time": 285.52213446800306, "exposed_generation_time": 604.2781789101427, "weight_sync_time": 16.053921250000712, "valid_tokens_per_sec_per_gpu": 24.753925182491464}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 3456, "step": 3}} +:::MLLOG {"namespace": "", "time_ms": 1792437692990, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.05191605548316147, "reward": 0.3793402910232544, "grad_norm": 0.033122722059488297, "global_valid_toks": 6914050.0, "global_valid_seqs": 1128.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 4608, "step": 4}} +:::MLLOG {"namespace": "", "time_ms": 1792437693000, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 972.3146510239894, "policy_training_time": 281.97502401599195, "exposed_generation_time": 592.1221299828758, "weight_sync_time": 15.953968533998705, "valid_tokens_per_sec_per_gpu": 27.77702442728454}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 4608, "step": 4}} +:::MLLOG {"namespace": "", "time_ms": 1792438852260, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.034301756364584435, "reward": 0.4123263955116272, "grad_norm": 0.031360529363155365, "global_valid_toks": 6845969.0, "global_valid_seqs": 1143.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 5760, "step": 5}} +:::MLLOG {"namespace": "", "time_ms": 1792438852271, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 1063.358516604916, "policy_training_time": 285.9267945570027, "exposed_generation_time": 680.6914365860575, "weight_sync_time": 16.097107121997396, "valid_tokens_per_sec_per_gpu": 25.148683147459888}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 5760, "step": 5}} +:::MLLOG {"namespace": "", "time_ms": 1792439858865, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "step": 6}} +:::MLLOG {"namespace": "", "time_ms": 1792439858867, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "step": 6}} +:::MLLOG {"namespace": "", "time_ms": 1792441047197, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"validation_time": 1186.2563511730114}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "step": 6}} +:::MLLOG {"namespace": "", "time_ms": 1792441047204, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7609561681747437, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912}} +:::MLLOG {"namespace": "", "time_ms": 1792441047208, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "step": 6}} +:::MLLOG {"namespace": "", "time_ms": 1792441047208, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "status": "success"}} diff --git a/llm_post_training/rcp_logs/GBS1152/seed_5.out b/llm_post_training/rcp_logs/GBS1152/seed_5.out new file mode 100644 index 000000000..46a35241f --- /dev/null +++ b/llm_post_training/rcp_logs/GBS1152/seed_5.out @@ -0,0 +1,64 @@ +:::MLLOG {"namespace": "", "time_ms": 1792443287214, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "", "lineno": 3}} +:::MLLOG {"namespace": "", "time_ms": 1792443301708, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303210, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "qwen35_397b_grpo", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792443303211, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "Google", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792443303211, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792443303211, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792443303211, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "64xa4x", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792443303212, "event_type": "POINT_IN_TIME", "key": "seed", "value": 1885626197, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303212, "event_type": "POINT_IN_TIME", "key": "max_steps", "value": 10, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303212, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 1152, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303212, "event_type": "POINT_IN_TIME", "key": "micro_batch_size", "value": 1, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303213, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 65536, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303213, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 11200, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303213, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 251, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303213, "event_type": "POINT_IN_TIME", "key": "init_checkpoint_step", "value": 0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303213, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "adamw", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303213, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2.121320343559642e-06, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303213, "event_type": "POINT_IN_TIME", "key": "opt_end_learning_rate", "value": 2.121320343559642e-06, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303214, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_1", "value": 0.9, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303214, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_2", "value": 0.999, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303214, "event_type": "POINT_IN_TIME", "key": "opt_adamw_epsilon", "value": 1e-08, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303214, "event_type": "POINT_IN_TIME", "key": "opt_adamw_weight_decay", "value": 0.0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303214, "event_type": "POINT_IN_TIME", "key": "opt_gradient_clip_norm", "value": 0.05892556509887896, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303214, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303214, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_steps", "value": 100000, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303214, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_schedule", "value": "constant", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303215, "event_type": "POINT_IN_TIME", "key": "tensor_parallelism", "value": 4, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303215, "event_type": "POINT_IN_TIME", "key": "pipeline_parallelism", "value": 2, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303215, "event_type": "POINT_IN_TIME", "key": "context_parallelism", "value": 1, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303215, "event_type": "POINT_IN_TIME", "key": "expert_parallelism", "value": 32, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303215, "event_type": "POINT_IN_TIME", "key": "generation_backend", "value": "vllm", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303215, "event_type": "POINT_IN_TIME", "key": "generation_tensor_parallelism", "value": 8, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303215, "event_type": "POINT_IN_TIME", "key": "generation_pipeline_parallelism", "value": 1, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303215, "event_type": "POINT_IN_TIME", "key": "generation_expert_parallelism", "value": 8, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303216, "event_type": "POINT_IN_TIME", "key": "generation_training_rollout_temperature", "value": 1.0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303216, "event_type": "POINT_IN_TIME", "key": "generation_training_rollout_top_p", "value": 1.0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303216, "event_type": "POINT_IN_TIME", "key": "generation_validation_rollout_temperature", "value": 0.1, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303216, "event_type": "POINT_IN_TIME", "key": "generation_validation_rollout_top_p", "value": 0.95, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303216, "event_type": "POINT_IN_TIME", "key": "num_prompts_per_step", "value": 72, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303216, "event_type": "POINT_IN_TIME", "key": "num_generations_per_prompt", "value": 16, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303216, "event_type": "POINT_IN_TIME", "key": "truncated_importance_sampling_ratio_min", "value": 0.999, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303216, "event_type": "POINT_IN_TIME", "key": "truncated_importance_sampling_ratio", "value": 1.002, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303217, "event_type": "POINT_IN_TIME", "key": "truncated_importance_sampling_type", "value": "seq-mask-tis", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443303217, "event_type": "POINT_IN_TIME", "key": "target_accuracy", "value": 0.69, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443644943, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 144, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792443644956, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 291}} +:::MLLOG {"namespace": "", "time_ms": 1792443644956, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 291}} +:::MLLOG {"namespace": "", "time_ms": 1792443644957, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "step": 0}} +:::MLLOG {"namespace": "", "time_ms": 1792445924621, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.08059799253533129, "reward": 0.3263888955116272, "grad_norm": 0.05112424120306969, "global_valid_toks": 6349554.0, "global_valid_seqs": 1124.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 1152, "step": 1}} +:::MLLOG {"namespace": "", "time_ms": 1792445924631, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 710.8184477889881, "policy_training_time": 450.29829919998883, "exposed_generation_time": 15.719613993001985, "weight_sync_time": 15.954912497007172, "valid_tokens_per_sec_per_gpu": 34.89350253872835}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 1152, "step": 1}} +:::MLLOG {"namespace": "", "time_ms": 1792447155083, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.04409805298928404, "reward": 0.2508680522441864, "grad_norm": 0.03884110599756241, "global_valid_toks": 6194811.0, "global_valid_seqs": 1129.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 2304, "step": 2}} +:::MLLOG {"namespace": "", "time_ms": 1792447155094, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 1129.5925368939206, "policy_training_time": 281.8207847070007, "exposed_generation_time": 751.6063310819882, "weight_sync_time": 15.987358725993545, "valid_tokens_per_sec_per_gpu": 21.42230908792067}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 2304, "step": 2}} +:::MLLOG {"namespace": "", "time_ms": 1792448464033, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.07075023947254522, "reward": 0.3315972089767456, "grad_norm": 0.03821666166186333, "global_valid_toks": 6227403.0, "global_valid_seqs": 1124.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 3456, "step": 3}} +:::MLLOG {"namespace": "", "time_ms": 1792448464044, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 1209.9380968059995, "policy_training_time": 284.54910597999697, "exposed_generation_time": 829.2193543700996, "weight_sync_time": 16.135849803991732, "valid_tokens_per_sec_per_gpu": 20.104989695725216}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 3456, "step": 3}} +:::MLLOG {"namespace": "", "time_ms": 1792449660175, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.05873232959856978, "reward": 0.3637152910232544, "grad_norm": 0.029306791722774506, "global_valid_toks": 6631011.0, "global_valid_seqs": 1111.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 4608, "step": 4}} +:::MLLOG {"namespace": "", "time_ms": 1792449660186, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 1090.3390941319813, "policy_training_time": 281.8352632340102, "exposed_generation_time": 712.178018018094, "weight_sync_time": 16.095821405004244, "valid_tokens_per_sec_per_gpu": 23.756267071548862}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 4608, "step": 4}} +:::MLLOG {"namespace": "", "time_ms": 1792450845936, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.042008583346614614, "reward": 0.421875, "grad_norm": 0.029041338711977005, "global_valid_toks": 6951050.0, "global_valid_seqs": 1136.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 5760, "step": 5}} +:::MLLOG {"namespace": "", "time_ms": 1792450845947, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 1087.4713949731376, "policy_training_time": 289.80562735299463, "exposed_generation_time": 698.8959885570948, "weight_sync_time": 15.910583048011176, "valid_tokens_per_sec_per_gpu": 24.968508769989956}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 5760, "step": 5}} +:::MLLOG {"namespace": "", "time_ms": 1792452047974, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "step": 6}} +:::MLLOG {"namespace": "", "time_ms": 1792452047976, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "step": 6}} +:::MLLOG {"namespace": "", "time_ms": 1792453570482, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"validation_time": 1520.4628061440017}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "step": 6}} +:::MLLOG {"namespace": "", "time_ms": 1792453570496, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7808765172958374, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912}} +:::MLLOG {"namespace": "", "time_ms": 1792453570520, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "step": 6}} +:::MLLOG {"namespace": "", "time_ms": 1792453570536, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "status": "success"}} diff --git a/llm_post_training/rcp_logs/GBS1152/seed_6.out b/llm_post_training/rcp_logs/GBS1152/seed_6.out new file mode 100644 index 000000000..5586b225c --- /dev/null +++ b/llm_post_training/rcp_logs/GBS1152/seed_6.out @@ -0,0 +1,64 @@ +:::MLLOG {"namespace": "", "time_ms": 1792471980373, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "", "lineno": 3}} +:::MLLOG {"namespace": "", "time_ms": 1792471994885, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996408, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "qwen35_397b_grpo", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792471996409, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "Google", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792471996409, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792471996409, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792471996409, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "64xa4x", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 193}} +:::MLLOG {"namespace": "", "time_ms": 1792471996410, "event_type": "POINT_IN_TIME", "key": "seed", "value": 1885626199, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996410, "event_type": "POINT_IN_TIME", "key": "max_steps", "value": 10, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996411, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 1152, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996411, "event_type": "POINT_IN_TIME", "key": "micro_batch_size", "value": 1, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996411, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 65536, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996411, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 11200, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996411, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 251, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996411, "event_type": "POINT_IN_TIME", "key": "init_checkpoint_step", "value": 0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996411, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "adamw", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996411, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2.121320343559642e-06, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996412, "event_type": "POINT_IN_TIME", "key": "opt_end_learning_rate", "value": 2.121320343559642e-06, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996412, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_1", "value": 0.9, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996412, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_2", "value": 0.999, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996412, "event_type": "POINT_IN_TIME", "key": "opt_adamw_epsilon", "value": 1e-08, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996412, "event_type": "POINT_IN_TIME", "key": "opt_adamw_weight_decay", "value": 0.0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996412, "event_type": "POINT_IN_TIME", "key": "opt_gradient_clip_norm", "value": 0.05892556509887896, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996412, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996412, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_steps", "value": 100000, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996413, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_schedule", "value": "constant", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996413, "event_type": "POINT_IN_TIME", "key": "tensor_parallelism", "value": 4, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996413, "event_type": "POINT_IN_TIME", "key": "pipeline_parallelism", "value": 2, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996413, "event_type": "POINT_IN_TIME", "key": "context_parallelism", "value": 1, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996413, "event_type": "POINT_IN_TIME", "key": "expert_parallelism", "value": 32, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996413, "event_type": "POINT_IN_TIME", "key": "generation_backend", "value": "vllm", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996413, "event_type": "POINT_IN_TIME", "key": "generation_tensor_parallelism", "value": 8, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996413, "event_type": "POINT_IN_TIME", "key": "generation_pipeline_parallelism", "value": 1, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996414, "event_type": "POINT_IN_TIME", "key": "generation_expert_parallelism", "value": 8, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996414, "event_type": "POINT_IN_TIME", "key": "generation_training_rollout_temperature", "value": 1.0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996414, "event_type": "POINT_IN_TIME", "key": "generation_training_rollout_top_p", "value": 1.0, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996414, "event_type": "POINT_IN_TIME", "key": "generation_validation_rollout_temperature", "value": 0.1, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996414, "event_type": "POINT_IN_TIME", "key": "generation_validation_rollout_top_p", "value": 0.95, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996414, "event_type": "POINT_IN_TIME", "key": "num_prompts_per_step", "value": 72, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996414, "event_type": "POINT_IN_TIME", "key": "num_generations_per_prompt", "value": 16, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996414, "event_type": "POINT_IN_TIME", "key": "truncated_importance_sampling_ratio_min", "value": 0.999, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996415, "event_type": "POINT_IN_TIME", "key": "truncated_importance_sampling_ratio", "value": 1.002, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996415, "event_type": "POINT_IN_TIME", "key": "truncated_importance_sampling_type", "value": "seq-mask-tis", "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792471996415, "event_type": "POINT_IN_TIME", "key": "target_accuracy", "value": 0.69, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792472348815, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 144, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125}} +:::MLLOG {"namespace": "", "time_ms": 1792472348885, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 291}} +:::MLLOG {"namespace": "", "time_ms": 1792472348885, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 291}} +:::MLLOG {"namespace": "", "time_ms": 1792472348885, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "step": 0}} +:::MLLOG {"namespace": "", "time_ms": 1792474424910, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.06462137152993819, "reward": 0.3472222089767456, "grad_norm": 0.056837353855371475, "global_valid_toks": 6426387.0, "global_valid_seqs": 1129.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 1152, "step": 1}} +:::MLLOG {"namespace": "", "time_ms": 1792474424920, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 812.2915521929972, "policy_training_time": 447.4749774719967, "exposed_generation_time": 119.4416183590074, "weight_sync_time": 15.71747045101074, "valid_tokens_per_sec_per_gpu": 30.904019807884954}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 1152, "step": 1}} +:::MLLOG {"namespace": "", "time_ms": 1792475560512, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.06568954717658926, "reward": 0.2864583432674408, "grad_norm": 0.038027323782444, "global_valid_toks": 6196493.0, "global_valid_seqs": 1119.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 2304, "step": 2}} +:::MLLOG {"namespace": "", "time_ms": 1792475560523, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 1036.090761377156, "policy_training_time": 283.3702145930001, "exposed_generation_time": 654.4255396610242, "weight_sync_time": 15.965919063994079, "valid_tokens_per_sec_per_gpu": 23.361901952563514}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 2304, "step": 2}} +:::MLLOG {"namespace": "", "time_ms": 1792476775415, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.08030944083293434, "reward": 0.3012152910232544, "grad_norm": 0.12514430284500122, "global_valid_toks": 5943498.0, "global_valid_seqs": 1099.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 3456, "step": 3}} +:::MLLOG {"namespace": "", "time_ms": 1792476775426, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 1116.7200236239878, "policy_training_time": 278.9347823559947, "exposed_generation_time": 741.7170122609386, "weight_sync_time": 16.105086369003402, "valid_tokens_per_sec_per_gpu": 20.79016098158311}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 3456, "step": 3}} +:::MLLOG {"namespace": "", "time_ms": 1792477790206, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.07174365404353011, "reward": 0.3428819477558136, "grad_norm": 0.03435346484184265, "global_valid_toks": 6115045.0, "global_valid_seqs": 1068.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 4608, "step": 4}} +:::MLLOG {"namespace": "", "time_ms": 1792477790217, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 917.870715112047, "policy_training_time": 270.51906595000764, "exposed_generation_time": 553.6386410451232, "weight_sync_time": 16.0726920110028, "valid_tokens_per_sec_per_gpu": 26.024247356376392}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 4608, "step": 4}} +:::MLLOG {"namespace": "", "time_ms": 1792478860726, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"reduced_train_loss": -0.04479562629421707, "reward": 0.3914930522441864, "grad_norm": 0.030379267409443855, "global_valid_toks": 6012459.0, "global_valid_seqs": 1106.0}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 5760, "step": 5}} +:::MLLOG {"namespace": "", "time_ms": 1792478860737, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"train_step_time": 972.4216696020885, "policy_training_time": 275.3300196079945, "exposed_generation_time": 602.9253504500375, "weight_sync_time": 16.17350711299514, "valid_tokens_per_sec_per_gpu": 24.152246605487985}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 5760, "step": 5}} +:::MLLOG {"namespace": "", "time_ms": 1792479839203, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "step": 6}} +:::MLLOG {"namespace": "", "time_ms": 1792479839205, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "step": 6}} +:::MLLOG {"namespace": "", "time_ms": 1792481050952, "event_type": "POINT_IN_TIME", "key": "tracked_stats", "value": {"validation_time": 1209.4548397170001}, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "step": 6}} +:::MLLOG {"namespace": "", "time_ms": 1792481050966, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7290836572647095, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912}} +:::MLLOG {"namespace": "", "time_ms": 1792481050987, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "step": 6}} +:::MLLOG {"namespace": "", "time_ms": 1792481051026, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "/opt/nemo-rl/nemo_rl/algorithms/mlperf_grpo_logging.py", "lineno": 125, "samples_count": 6912, "status": "success"}} From c4f21cd1de52f0f29a70af91b41796e6cce758fb Mon Sep 17 00:00:00 2001 From: Summer Date: Tue, 20 Oct 2026 23:22:08 +0000 Subject: [PATCH 2/2] Address review comments on llm_post_training/README.md --- llm_post_training/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llm_post_training/README.md b/llm_post_training/README.md index 3c7c39035..86b299736 100644 --- a/llm_post_training/README.md +++ b/llm_post_training/README.md @@ -416,7 +416,7 @@ controls as named command-line arguments with type checks: | Control | Command-line argument | Qualified value or rule | |---|---|---| -| Global batch size | `--gbs` | One of 256, 512, or 1024 for the checked-in RCPs | +| Global batch size | `--gbs` | One of 256, 512, 1024, or 1152 for the checked-in RCPs | | Learning rate and minimum LR | `--lr` | Exact GBS-dependent value in the table above | | Maximum gradient norm | `--clip` | Exact GBS-dependent value in the table above | | First validation step | `--val-start` | Exact GBS-dependent value in the table above | @@ -749,7 +749,7 @@ The checked-in RCP logs provide six independent seeds at each qualified GBS: | 256 | `rcp_logs/256/seed_*.out` | 6 / 6 | Step 18 | 0.703125 - 0.75390625 | | 512 | `rcp_logs/512/seed_*.out` | 6 / 6 | Step 10 | 0.6953125 - 0.73046875 | | 1024 | `rcp_logs/1024/seed_*.out` | 6 / 6 | Step 7 | 0.70703125 - 0.75000000 | -| 1152 | `rcp_logs/GBS1152/seed_*.out` | 6 / 6 | Step 6 | 0.72908366 - 0.78087652 | +| 1152 | `rcp_logs/1152/seed_*.out` | 6 / 6 | Step 6 | 0.72908366 - 0.78087652 | All 24 runs reach the target at the first scheduled validation. This is stable observed convergence at the configured evaluation schedule; it does not imply