fix: score unresolvable principal refusals correctly

tests/test_evaluate.py

@@ -87,6 +87,14 @@ async def test_real_llm_is_held_to_refusal_correctness(corpus, conn, embedder):
     assert report.pass_rate == 0.0
 
 
+@pytest.mark.asyncio
+async def test_unresolvable_principal_refusal_does_not_drag_recall(corpus, conn, embedder):
+    """An unknown user retrieving nothing is correct, not a recall miss."""
+    cases = [_case("unknown", "ghost-user", "anything", should_answer=False)]
+    report = await run_eval(conn, embedder, _ANSWERS, cases, llm_judged=False)
+    assert report.mean_recall == 1.0
+
+
 # --------------------------------------------------------------------------- scoring arithmetic
 
 
@@ -100,6 +108,18 @@ def test_no_expected_docs_means_retrieving_nothing_is_perfect_recall():
     assert c.passed is True
 
 
+def test_mean_recall_averages_every_scored_case():
+    r = EvalReport(
+        label="x",
+        cases=[
+            CaseResult(case_id="a", user_id="u", recall=0.0),
+            CaseResult(case_id="b", user_id="u", recall=1.0),
+        ],
+        llm_judged=False,
+    )
+    assert r.mean_recall == 0.5
+
+
 # --------------------------------------------------------------------------------- regression diff
 
 

vaultrag/evaluate.py

@@ -91,7 +91,7 @@ def leak_rate(self) -> float:
 
     @property
     def mean_recall(self) -> float:
-        scored = [c.recall for c in self.cases if c.recall is not None]
+        scored = [c.recall for c in self.cases]
         return sum(scored) / len(scored) if scored else 0.0
 
     @property
@@ -145,9 +145,11 @@ async def run_case(
     principal = await resolve_principal(conn, case.user_id)
     if principal is None:
         # An unknown user retrieving nothing is correct behaviour, not an error in the eval.
+        recall = 1.0 if not case.expected_docs else 0.0
         return CaseResult(
             case_id=case.id,
             user_id=case.user_id,
+            recall=recall,
             correct_refusal=not case.should_answer,
             llm_judged=llm_judged,
         )