fix(eval): name the real cause of a noise refusal and stop faking gain_over_noise (#1089)

agentdescent/stats_eval.py

@@ -265,6 +265,11 @@ class NoiseCalibration:
     :func:`decompose_variance`. A gain is ``distinguishable`` only when it is
     larger than both the floor's half-width and ``min_effect``; otherwise it is
     noise, not promotion evidence.
+
+    ``gain_over_noise`` is the gain divided by the floor's half-width. It is
+    ``0.0`` when the gain is not positive and also when the ratio is undefined:
+    fewer than 2 baseline repeats bound no floor, and a zero-spread baseline has
+    a zero half-width. ``0.0`` there means "undefined", not "no signal".
     """
 
     noise_floor: Tuple[float, float]
@@ -353,8 +358,10 @@ def calibrate_noise(
         if candidate_values:
             candidate_mean = sum(candidate_values) / len(candidate_values)
     gain = 0.0 if candidate_mean is None else candidate_mean - baseline_mean
-    gain_over_noise = (0.0 if gain <= 0.0 else
-                       gain / max(half_width, 1e-9))
+    # The ratio is undefined without a measurable floor (fewer than 2
+    # repeats, or zero spread); report 0.0 rather than divide by an epsilon.
+    gain_over_noise = (0.0 if gain <= 0.0 or len(baseline_runs) < 2
+                       or half_width <= 0.0 else gain / half_width)
     # A gain exactly at the floor edge is still noise; the epsilon keeps a
     # floating-point representation of the edge from flipping the verdict.
     bar = max(half_width, float(min_effect))

agentdescent/trust_evidence.py

@@ -332,6 +332,28 @@ def to_dict(self) -> dict:
         return body
 
 
+def _noise_refusal_note(noise: Any, *, min_effect: float) -> str:
+    """Name the actual reason a gain is refused as noise.
+
+    ``distinguishable`` is false for three causes: too few baseline repeats to
+    bound a floor, a measurable floor the gain does not clear, or a floor
+    narrower than ``min_effect`` (the effect-size bar decides). Every branch
+    keeps the phrase ``not distinguishable from noise``.
+    """
+    head = f"gain {noise.gain:+.3f} is not distinguishable from noise"
+    if noise.repeats < 2:
+        return (f"{head} (the noise floor cannot be estimated from "
+                f"{noise.repeats} baseline repeat(s); at least 2 are needed)")
+    lo, hi = noise.noise_floor
+    half_width = (hi - lo) / 2.0
+    if half_width < float(min_effect):
+        return (f"{head} (gain does not exceed min_effect "
+                f"{float(min_effect):.3f}; noise floor half-width is "
+                f"{half_width:.3f})")
+    return (f"{head} (gain does not exceed the noise floor: baseline mean "
+            f"{noise.baseline_mean:+.3f} \u00b1 {half_width:.3f})")
+
+
 def build_trust_evidence(
         before: Mapping[str, float], after: Mapping[str, float], *,
         eval_id: str = "paired", dataset_id: str = "paired", version: int = 1,
@@ -415,9 +437,7 @@ def build_trust_evidence(
             groups, expected_repeats=noise_repeats, alpha=alpha, seed=seed,
             min_effect=min_effect)
         if not noise.distinguishable:
-            lo, hi = noise.noise_floor
-            note = (f"gain {noise.gain:+.3f} is not distinguishable from noise "
-                    f"(noise floor [{lo:+.3f}, {hi:+.3f}])")
+            note = _noise_refusal_note(noise, min_effect=min_effect)
             detail = (f"{eval_result.detail}; {note}"
                       if eval_result.detail else note)
             eval_result = replace(eval_result, passed=False, detail=detail)

changelog.d/1089-noise-refusal-cause.md

@@ -0,0 +1,6 @@
+Fixed
+Trust evidence now names the actual reason a gain is refused as noise: too few
+baseline repeats to bound a floor, a gain inside the floor (printed as a band
+around the baseline mean rather than an absolute interval against a delta), or
+a gain below `min_effect`. `gain_over_noise` is `0.0` when the floor is
+undefined instead of a ratio over an epsilon (#1089).

tests/test_noise_calibration.py

@@ -39,7 +39,8 @@ def test_a_strong_gain_clears_the_noise_floor():
         assert report.gain == pytest.approx(0.6)
         assert report.distinguishable is True
         assert report.verdict == "distinguishable from noise"
-        assert report.gain_over_noise > 1.0
+        # zero-spread floor: the ratio is undefined, not a 6e+08 epsilon artefact
+        assert report.gain_over_noise == 0.0
 
 
 def test_high_run_to_run_variance_widens_the_floor_and_warns():
@@ -87,3 +88,27 @@ def test_calibration_validates_its_inputs():
         calibrate_noise({})
     with pytest.raises(ValueError, match="min_effect"):
         calibrate_noise({"baseline": _runs(0.5)}, min_effect=float("nan"))
+
+
+@pytest.mark.parametrize("baseline,candidate,min_effect,distinguishable", [
+    ([_flat(0.5)], _runs(0.8), 0.0, False),          # one repeat: no floor
+    (_runs(0.5), _runs(0.52), 0.5, False),           # zero spread, below min_effect
+    (_runs(0.2), _runs(0.8), 0.0, True),             # zero spread, passes
+])
+def test_gain_over_noise_is_zero_when_the_floor_is_undefined(
+        baseline, candidate, min_effect, distinguishable):
+    report = calibrate_noise({"baseline": baseline, "candidate": candidate},
+                             min_effect=min_effect)
+    assert report.gain_over_noise == 0.0
+    assert report.distinguishable is distinguishable
+
+
+def test_gain_over_noise_keeps_the_ratio_on_a_measurable_floor():
+    report = calibrate_noise(
+        {"baseline": [_flat(0.3), _flat(0.5), _flat(0.7)],
+         "candidate": _runs(0.55)}, seed=0)
+    lo, hi = report.noise_floor
+    assert hi > lo
+    assert report.gain_over_noise == pytest.approx(
+        report.gain / ((hi - lo) / 2.0))
+    assert report.distinguishable is False

tests/test_trust_evidence_noise.py

@@ -57,3 +57,34 @@ def test_no_repeats_keeps_the_previous_behaviour():
         _flat(0.2), _flat(0.8), contaminated=False, risk=0.1)
     assert "noise" not in evidence.metadata
     assert evidence.eval_result.passed is True
+
+
+def test_one_baseline_repeat_names_the_repeat_count():
+    evidence = _evidence(0.5, 0.8, [0.5], [0.8])
+    detail = evidence.eval_result.detail
+    assert "not distinguishable from noise" in detail
+    assert "1 baseline repeat(s)" in detail
+    assert "noise floor [" not in detail
+    assert evidence.metadata["noise"]["gain_over_noise"] == 0.0
+
+
+def test_zero_spread_below_min_effect_names_min_effect():
+    evidence = build_trust_evidence(
+        _flat(0.5), _flat(0.52), contaminated=False, risk=0.1,
+        min_effect=0.5, baseline_runs=_runs(0.5), candidate_runs=_runs(0.52))
+    detail = evidence.eval_result.detail
+    assert "not distinguishable from noise" in detail
+    assert "min_effect 0.500" in detail
+
+
+def test_a_floor_refusal_prints_a_band_around_the_baseline_mean():
+    evidence = _evidence(0.5, 0.55, [0.3, 0.5, 0.7], [0.55, 0.55, 0.55])
+    detail = evidence.eval_result.detail
+    assert "not distinguishable from noise" in detail
+    assert "baseline mean +0.500 \u00b1" in detail
+
+
+def test_a_passing_zero_spread_gain_has_no_note_and_no_fake_ratio():
+    evidence = _evidence(0.2, 0.8, [0.2, 0.2, 0.2], [0.8, 0.8, 0.8])
+    assert "not distinguishable" not in evidence.eval_result.detail
+    assert evidence.metadata["noise"]["gain_over_noise"] == 0.0