From d1e72bfd9ac987b2a5f8067330abe724b64e4eda Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9CSebastian?=
 <64795732+slegarraga@users.noreply.github.com>
Date: Thu, 6 Aug 2026 05:43:59 -0400
Subject: [PATCH] fix: score unresolvable principal refusals correctly

---
 app/evaluate.py        |  4 +++-
 tests/test_evaluate.py | 20 ++++++++++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/app/evaluate.py b/app/evaluate.py
index 1b07923..580e35e 100644
--- a/app/evaluate.py
+++ b/app/evaluate.py
@@ -91,7 +91,7 @@ def leak_rate(self) -> float:
 
     @property
     def mean_recall(self) -> float:
-        scored = [c.recall for c in self.cases if c.recall is not None]
+        scored = [c.recall for c in self.cases]
         return sum(scored) / len(scored) if scored else 0.0
 
     @property
@@ -145,9 +145,11 @@ async def run_case(
     principal = await resolve_principal(conn, case.user_id)
     if principal is None:
         # An unknown user retrieving nothing is correct behaviour, not an error in the eval.
+        recall = 1.0 if not case.expected_docs else 0.0
         return CaseResult(
             case_id=case.id,
             user_id=case.user_id,
+            recall=recall,
             correct_refusal=not case.should_answer,
             llm_judged=llm_judged,
         )
diff --git a/tests/test_evaluate.py b/tests/test_evaluate.py
index aac5d2f..95afe50 100644
--- a/tests/test_evaluate.py
+++ b/tests/test_evaluate.py
@@ -87,6 +87,14 @@ async def test_real_llm_is_held_to_refusal_correctness(corpus, conn, embedder):
     assert report.pass_rate == 0.0
 
 
+@pytest.mark.asyncio
+async def test_unresolvable_principal_refusal_does_not_drag_recall(corpus, conn, embedder):
+    """An unknown user retrieving nothing is correct, not a recall miss."""
+    cases = [_case("unknown", "ghost-user", "anything", should_answer=False)]
+    report = await run_eval(conn, embedder, _ANSWERS, cases, llm_judged=False)
+    assert report.mean_recall == 1.0
+
+
 # --------------------------------------------------------------------------- scoring arithmetic
 
 
@@ -100,6 +108,18 @@ def test_no_expected_docs_means_retrieving_nothing_is_perfect_recall():
     assert c.passed is True
 
 
+def test_mean_recall_averages_every_scored_case():
+    r = EvalReport(
+        label="x",
+        cases=[
+            CaseResult(case_id="a", user_id="u", recall=0.0),
+            CaseResult(case_id="b", user_id="u", recall=1.0),
+        ],
+        llm_judged=False,
+    )
+    assert r.mean_recall == 0.5
+
+
 # --------------------------------------------------------------------------------- regression diff