From 6e63c7b385ed2180a4fae9593114573b7ccd5f22 Mon Sep 17 00:00:00 2001 From: Birfy <15378933+Birfy@users.noreply.github.com> Date: Sun, 15 Nov 2026 06:32:17 +0800 Subject: [PATCH] fix(eval): name the real cause of a noise refusal and stop faking gain_over_noise (#1089) --- agentdescent/stats_eval.py | 11 +++++++-- agentdescent/trust_evidence.py | 26 ++++++++++++++++++--- changelog.d/1089-noise-refusal-cause.md | 6 +++++ tests/test_noise_calibration.py | 27 ++++++++++++++++++++- tests/test_trust_evidence_noise.py | 31 +++++++++++++++++++++++++ 5 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 changelog.d/1089-noise-refusal-cause.md diff --git a/agentdescent/stats_eval.py b/agentdescent/stats_eval.py index 85df9f7f..aeb356b9 100644 --- a/agentdescent/stats_eval.py +++ b/agentdescent/stats_eval.py @@ -265,6 +265,11 @@ class NoiseCalibration: :func:`decompose_variance`. A gain is ``distinguishable`` only when it is larger than both the floor's half-width and ``min_effect``; otherwise it is noise, not promotion evidence. + + ``gain_over_noise`` is the gain divided by the floor's half-width. It is + ``0.0`` when the gain is not positive and also when the ratio is undefined: + fewer than 2 baseline repeats bound no floor, and a zero-spread baseline has + a zero half-width. ``0.0`` there means "undefined", not "no signal". """ noise_floor: Tuple[float, float] @@ -353,8 +358,10 @@ def calibrate_noise( if candidate_values: candidate_mean = sum(candidate_values) / len(candidate_values) gain = 0.0 if candidate_mean is None else candidate_mean - baseline_mean - gain_over_noise = (0.0 if gain <= 0.0 else - gain / max(half_width, 1e-9)) + # The ratio is undefined without a measurable floor (fewer than 2 + # repeats, or zero spread); report 0.0 rather than divide by an epsilon. + gain_over_noise = (0.0 if gain <= 0.0 or len(baseline_runs) < 2 + or half_width <= 0.0 else gain / half_width) # A gain exactly at the floor edge is still noise; the epsilon keeps a # floating-point representation of the edge from flipping the verdict. bar = max(half_width, float(min_effect)) diff --git a/agentdescent/trust_evidence.py b/agentdescent/trust_evidence.py index e951a467..44a8b94a 100644 --- a/agentdescent/trust_evidence.py +++ b/agentdescent/trust_evidence.py @@ -332,6 +332,28 @@ def to_dict(self) -> dict: return body +def _noise_refusal_note(noise: Any, *, min_effect: float) -> str: + """Name the actual reason a gain is refused as noise. + + ``distinguishable`` is false for three causes: too few baseline repeats to + bound a floor, a measurable floor the gain does not clear, or a floor + narrower than ``min_effect`` (the effect-size bar decides). Every branch + keeps the phrase ``not distinguishable from noise``. + """ + head = f"gain {noise.gain:+.3f} is not distinguishable from noise" + if noise.repeats < 2: + return (f"{head} (the noise floor cannot be estimated from " + f"{noise.repeats} baseline repeat(s); at least 2 are needed)") + lo, hi = noise.noise_floor + half_width = (hi - lo) / 2.0 + if half_width < float(min_effect): + return (f"{head} (gain does not exceed min_effect " + f"{float(min_effect):.3f}; noise floor half-width is " + f"{half_width:.3f})") + return (f"{head} (gain does not exceed the noise floor: baseline mean " + f"{noise.baseline_mean:+.3f} \u00b1 {half_width:.3f})") + + def build_trust_evidence( before: Mapping[str, float], after: Mapping[str, float], *, eval_id: str = "paired", dataset_id: str = "paired", version: int = 1, @@ -415,9 +437,7 @@ def build_trust_evidence( groups, expected_repeats=noise_repeats, alpha=alpha, seed=seed, min_effect=min_effect) if not noise.distinguishable: - lo, hi = noise.noise_floor - note = (f"gain {noise.gain:+.3f} is not distinguishable from noise " - f"(noise floor [{lo:+.3f}, {hi:+.3f}])") + note = _noise_refusal_note(noise, min_effect=min_effect) detail = (f"{eval_result.detail}; {note}" if eval_result.detail else note) eval_result = replace(eval_result, passed=False, detail=detail) diff --git a/changelog.d/1089-noise-refusal-cause.md b/changelog.d/1089-noise-refusal-cause.md new file mode 100644 index 00000000..3a0ae908 --- /dev/null +++ b/changelog.d/1089-noise-refusal-cause.md @@ -0,0 +1,6 @@ +Fixed +Trust evidence now names the actual reason a gain is refused as noise: too few +baseline repeats to bound a floor, a gain inside the floor (printed as a band +around the baseline mean rather than an absolute interval against a delta), or +a gain below `min_effect`. `gain_over_noise` is `0.0` when the floor is +undefined instead of a ratio over an epsilon (#1089). diff --git a/tests/test_noise_calibration.py b/tests/test_noise_calibration.py index 3f2a3082..fdea4d25 100644 --- a/tests/test_noise_calibration.py +++ b/tests/test_noise_calibration.py @@ -39,7 +39,8 @@ def test_a_strong_gain_clears_the_noise_floor(): assert report.gain == pytest.approx(0.6) assert report.distinguishable is True assert report.verdict == "distinguishable from noise" - assert report.gain_over_noise > 1.0 + # zero-spread floor: the ratio is undefined, not a 6e+08 epsilon artefact + assert report.gain_over_noise == 0.0 def test_high_run_to_run_variance_widens_the_floor_and_warns(): @@ -87,3 +88,27 @@ def test_calibration_validates_its_inputs(): calibrate_noise({}) with pytest.raises(ValueError, match="min_effect"): calibrate_noise({"baseline": _runs(0.5)}, min_effect=float("nan")) + + +@pytest.mark.parametrize("baseline,candidate,min_effect,distinguishable", [ + ([_flat(0.5)], _runs(0.8), 0.0, False), # one repeat: no floor + (_runs(0.5), _runs(0.52), 0.5, False), # zero spread, below min_effect + (_runs(0.2), _runs(0.8), 0.0, True), # zero spread, passes +]) +def test_gain_over_noise_is_zero_when_the_floor_is_undefined( + baseline, candidate, min_effect, distinguishable): + report = calibrate_noise({"baseline": baseline, "candidate": candidate}, + min_effect=min_effect) + assert report.gain_over_noise == 0.0 + assert report.distinguishable is distinguishable + + +def test_gain_over_noise_keeps_the_ratio_on_a_measurable_floor(): + report = calibrate_noise( + {"baseline": [_flat(0.3), _flat(0.5), _flat(0.7)], + "candidate": _runs(0.55)}, seed=0) + lo, hi = report.noise_floor + assert hi > lo + assert report.gain_over_noise == pytest.approx( + report.gain / ((hi - lo) / 2.0)) + assert report.distinguishable is False diff --git a/tests/test_trust_evidence_noise.py b/tests/test_trust_evidence_noise.py index de048d43..c9ac28a8 100644 --- a/tests/test_trust_evidence_noise.py +++ b/tests/test_trust_evidence_noise.py @@ -57,3 +57,34 @@ def test_no_repeats_keeps_the_previous_behaviour(): _flat(0.2), _flat(0.8), contaminated=False, risk=0.1) assert "noise" not in evidence.metadata assert evidence.eval_result.passed is True + + +def test_one_baseline_repeat_names_the_repeat_count(): + evidence = _evidence(0.5, 0.8, [0.5], [0.8]) + detail = evidence.eval_result.detail + assert "not distinguishable from noise" in detail + assert "1 baseline repeat(s)" in detail + assert "noise floor [" not in detail + assert evidence.metadata["noise"]["gain_over_noise"] == 0.0 + + +def test_zero_spread_below_min_effect_names_min_effect(): + evidence = build_trust_evidence( + _flat(0.5), _flat(0.52), contaminated=False, risk=0.1, + min_effect=0.5, baseline_runs=_runs(0.5), candidate_runs=_runs(0.52)) + detail = evidence.eval_result.detail + assert "not distinguishable from noise" in detail + assert "min_effect 0.500" in detail + + +def test_a_floor_refusal_prints_a_band_around_the_baseline_mean(): + evidence = _evidence(0.5, 0.55, [0.3, 0.5, 0.7], [0.55, 0.55, 0.55]) + detail = evidence.eval_result.detail + assert "not distinguishable from noise" in detail + assert "baseline mean +0.500 \u00b1" in detail + + +def test_a_passing_zero_spread_gain_has_no_note_and_no_fake_ratio(): + evidence = _evidence(0.2, 0.8, [0.2, 0.2, 0.2], [0.8, 0.8, 0.8]) + assert "not distinguishable" not in evidence.eval_result.detail + assert evidence.metadata["noise"]["gain_over_noise"] == 0.0