Skip to content

Compute summarizer scores after the RCP check updates scaling.json #484

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
matthew-frank wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Compute summarizer scores after the RCP check updates scaling.json #484

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8572dd3
Compute summarizer scores after the RCP check updates scaling.json
matthew-frank Oct 20, 2026
2a938c6
Merge branch 'master' into matthew-frank/fix-scaling-json-order
matthew-frank Oct 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 21 additions & 5 deletions mlperf_logging/rcp_checker/rcp_checker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -468,10 +468,27 @@ def _set_results_scaling(self, scale_factor, results_dir):
with open(filepath, "w") as scaling_file:
scaling_file.write(json_content)

def _reset_results_scaling(self, results_dir):
def _update_results_scaling(self, scale_factor, results_dir):
'''Make results_dir/scaling.json hold scale_factor (no file means 1.0).
An existing file that already holds scale_factor is left untouched.'''
filepath = results_dir+'/scaling.json'
old_factor = 1.0
if os.path.exists(filepath):
try:
with open(filepath) as scaling_file:
old_factor = float(json.load(scaling_file)['scaling_factor'])
except (OSError, ValueError, KeyError, TypeError):
old_factor = None
if old_factor == scale_factor:
return
if os.path.exists(filepath):
logging.warning(" Replacing scaling factor %s in %s with newly computed %.4f",
old_factor, filepath, scale_factor)
if scale_factor == 1.0:
os.remove(filepath)
else:
self._set_results_scaling(scale_factor, results_dir)
logging.info(" Results scaling set to normalization factor of %.4f", scale_factor)

def _eval_submission_record(self, rcp_record, subm_epochs, results_dir):
'''Compare reference and submission convergence.'''
Expand All @@ -483,17 +500,16 @@ def _eval_submission_record(self, rcp_record, subm_epochs, results_dir):
samples_rejected = 4 if rcp_record["Benchmark"] == 'unet3d' else 1
mean_subm_epochs = np.mean(subm_epochs[samples_rejected:len(subm_epochs)-samples_rejected])
norm_factor = self._find_norm_factor(rcp_record, mean_subm_epochs)
self._reset_results_scaling(results_dir)
if mean_subm_epochs >= (rcp_record["RCP Mean"] / rcp_record["Max Speedup"]):
logging.info(" RCP Record: %s", rcp_record)
logging.info(" Submission mean epochs: %.4f", mean_subm_epochs)
if mean_subm_epochs < rcp_record["RCP Mean"]:
mesg = " Submission mean epochs faster than RCP mean but within max speedup range. Score should be normalized by factor of {} / {} = {}"
mesg = mesg.format(rcp_record["RCP Mean"], mean_subm_epochs, norm_factor)
logging.info(mesg)
if results_dir != '':
self._set_results_scaling(norm_factor, results_dir)
logging.info(" Results scaling set to normalization factor of %.4f", norm_factor)
# norm_factor is 1.0 when the submission is no faster than the RCP mean.
if results_dir != '':
self._update_results_scaling(norm_factor, results_dir)
return True, norm_factor
else:
logging.info(" RCP Record: %s", rcp_record)
Expand Down
12 changes: 7 additions & 5 deletions mlperf_logging/result_summarizer/result_summarizer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -510,12 +510,9 @@ def _compute_strong_scaling_scores(desc, system_folder, usage, ruleset, division
has_power = _has_power(benchmark_folder)
benchmark = _benchmark_alias(folder_parts[-1])
system = folder_parts[-3] if usage == 'hpc' else folder_parts[-2]
# Compute base perf/power scores
score, power_score, rcp_scaling_factor = _compute_strong_score_standalone(
benchmark, system, has_power, benchmark_folder, usage, ruleset, desc
)

# RCP/GBS/Epochs additions for closed division
# RCP/GBS/Epochs additions for closed division. The RCP check rewrites
# scaling.json, so it must run before the scores below read it.
benchmark_gbs = None
benchmark_epochs = None
benchmark_rcp = None
Expand Down Expand Up @@ -576,6 +573,11 @@ def _compute_strong_scaling_scores(desc, system_folder, usage, ruleset, division
)
traceback.print_exc()

# Compute base perf/power scores
score, power_score, rcp_scaling_factor = _compute_strong_score_standalone(
benchmark, system, has_power, benchmark_folder, usage, ruleset, desc
)

# Map into metric-suffixed keys for schema
detailed_bechmark_scores[f"{benchmark}:rcp_scaling_factor"] = float(
rcp_scaling_factor
Expand Down
Loading