diff --git a/agentrace/checks.py b/agentrace/checks.py index 13c437f..a78f4ca 100644 --- a/agentrace/checks.py +++ b/agentrace/checks.py @@ -96,18 +96,20 @@ def check_unverified_claim(run: AgentRun) -> list[Finding]: r"\bcould not (?:independently )?verify\b", r"\bunverified\b", ] - hits = [] + hits = 0 + first_context = "" for p in hedges: for m in re.finditer(p, run.result, re.I): - hits.append(_context(run.result, m.start())) - break + if hits == 0: + first_context = _context(run.result, m.start()) + hits += 1 if hits: return [ Finding( "hedged_claim", "low", - f"{len(hits)} hedged claim(s). Fine if the hedge survives downstream, a problem if it gets flattened into fact.", - hits[0], + f"{hits} hedged claim(s). Fine if the hedge survives downstream, a problem if it gets flattened into fact.", + first_context, ) ] return [] diff --git a/tests/test_agentrace.py b/tests/test_agentrace.py index 96e5e8d..12d26fd 100644 --- a/tests/test_agentrace.py +++ b/tests/test_agentrace.py @@ -179,6 +179,12 @@ def test_hedged_claim_is_caught_but_only_low(): assert findings and findings[0].severity == "low" +def test_hedged_claim_counts_every_occurrence(): + r = _run(result="probably one. probably two. probably three. " + "x" * 200) + findings = [f for f in analyse(r) if f.check == "hedged_claim"] + assert findings and "3 hedged claim(s)" in findings[0].message + + def test_many_urls_without_verification_is_flagged(): urls = " ".join(f"https://example{i}.com/careers" for i in range(8)) assert "unverified_urls" in _codes(_run(result=urls + " " + "x" * 200))