@@ -10,21 +10,32 @@
 For each source the app calls (data/remote/ExternalLookup.kt), this fetches a
 sample number and checks the HTTP status and the marker field the Kotlin parser
 keys on. Exit status is non-zero when any source stops answering in that shape.
+
+--sample measures what SkipCalls adds: it asks about numbers drawn from the
+published database, from the unpublished community pending pool and from
+well-known business lines, and counts how often SkipCalls names a spam
+category the app acts on and how often that number isn't already in the
+local database.
 """
 
 from __future__ import annotations
 
+import argparse
+import json
+import random
 import re
 import sys
+import time
 import urllib.error
 import urllib.request
-from collections.abc import Callable
+from collections.abc import Callable, Iterable
 from dataclasses import dataclass
 from pathlib import Path
 
 ROOT = Path(__file__).resolve().parent.parent
 EXTERNAL_LOOKUP = ROOT / "app" / "src" / "main" / "java" / "com" / "sysadmindoc" / "callshield" / "data" / "remote" / "ExternalLookup.kt"
 FETCH_TIMEOUT_SECONDS = 20
+DATA = ROOT / "data"
 
 
 @dataclass(frozen=True)
@@ -94,7 +105,169 @@ def check(source: Source, fetcher: Fetch) -> tuple[bool, str]:
     return True, "answers in the shape the app parses, and still flags a known spam number"
 
 
+# Well-known US customer-service lines. SkipCalls naming one of these as spam
+# would be a false alarm the app shows to the user.
+BUSINESS_LINES = (
+    "+18002752273",  # Apple Support
+    "+18008291040",  # IRS
+    "+18007721213",  # Social Security
+    "+18006334227",  # Medicare
+    "+18002758777",  # USPS
+    "+18007425877",  # UPS
+    "+18004633339",  # FedEx
+    "+18002255345",  # DHL Express
+    "+18882804331",  # Amazon
+    "+18004321000",  # Bank of America
+    "+18009359935",  # Chase
+    "+18008693557",  # Wells Fargo
+    "+18009505114",  # Citi
+    "+18002274825",  # Capital One
+    "+18003472683",  # Discover
+    "+18005284800",  # American Express
+    "+18882211161",  # PayPal
+    "+18009346489",  # Xfinity
+    "+18009220204",  # Verizon
+    "+18003310500",  # AT&T
+    "+18009378997",  # T-Mobile
+    "+18002211212",  # Delta
+    "+18008648331",  # United
+    "+18004337300",  # American Airlines
+    "+18004359792",  # Southwest
+    "+18006427676",  # Microsoft
+    "+18006249897",  # Dell
+    "+18004746836",  # HP
+    "+18882378289",  # Best Buy
+    "+18009256278",  # Walmart
+    "+18005913869",  # Target
+    "+18004663337",  # Home Depot
+    "+18004456937",  # Lowe's
+    "+18007742678",  # Costco
+    "+18007467287",  # CVS
+    "+18009254733",  # Walgreens
+    "+18002077847",  # GEICO
+    "+18007828332",  # State Farm
+    "+18007764737",  # Progressive
+    "+18002557828",  # Allstate
+    "+18005315000",  # DirecTV
+    "+18003333474",  # DISH
+    "+18883973742",  # Experian
+    "+18009168800",  # TransUnion
+    "+18773824357",  # FTC
+    "+18008722657",  # U.S. Bank
+    "+18887622265",  # PNC
+    "+18887519000",  # TD Bank
+    "+18008472911",  # Visa
+    "+18006278372",  # Mastercard
+)
+
+SKIPCALLS_FLAGGED_CATEGORIES = frozenset({"scam", "robocall", "telemarketer", "fraud"})
+
+
+@dataclass(frozen=True)
+class Answer:
+    number: str
+    # "flagged" (a category the app acts on), "uncategorized" (is_spam with any
+    # other category, which the app shows as neutral), "clean" or "error".
+    verdict: str
+
+
+def classify(status: int, body: str) -> str:
+    if status != 200:
+        return "error"
+    try:
+        parsed = json.loads(body)
+    except ValueError:
+        return "error"
+    if not isinstance(parsed, dict) or not isinstance(parsed.get("is_spam"), bool):
+        return "error"
+    if not parsed["is_spam"]:
+        return "clean"
+    category = str(parsed.get("status_description", "")).strip().lower()
+    return "flagged" if category in SKIPCALLS_FLAGGED_CATEGORIES else "uncategorized"
+
+
+def summarize(name: str, answers: list[Answer], local_numbers: set[str]) -> dict:
+    """One group's counts. `adds` is a flag for a number the local database doesn't hold."""
+    flagged = [a for a in answers if a.verdict == "flagged"]
+    adds = [a for a in flagged if a.number not in local_numbers]
+    answered = [a for a in answers if a.verdict != "error"]
+    return {
+        "group": name,
+        "asked": len(answers),
+        "answered": len(answered),
+        "flagged": len(flagged),
+        "adds": len(adds),
+        "uncategorized": sum(1 for a in answers if a.verdict == "uncategorized"),
+        "errors": len(answers) - len(answered),
+        "add_examples": [a.number for a in adds[:5]],
+    }
+
+
+def database_numbers() -> set[str]:
+    database = json.loads((DATA / "spam_numbers.json").read_text(encoding="utf-8"))
+    return {row["number"] for row in database["numbers"]}
+
+
+def pending_numbers(published: set[str]) -> list[str]:
+    pending = json.loads((DATA / "community_pending.json").read_text(encoding="utf-8"))
+    return sorted(
+        number for number, record in pending["numbers"].items() if not record.get("published") and number not in published
+    )
+
+
+def ask_all(numbers: Iterable[str], fetcher: Fetch, delay_seconds: float) -> list[Answer]:
+    answers = []
+    for index, number in enumerate(numbers):
+        if index and delay_seconds:
+            time.sleep(delay_seconds)
+        digits = re.sub(r"\D", "", number)
+        try:
+            status, body = fetcher(SOURCES[0].url_prefix + digits)
+        except OSError:
+            status, body = 0, ""
+        answers.append(Answer(number, classify(status, body)))
+    return answers
+
+
+def sample(fetcher: Fetch, seed: int, per_group: int, delay_seconds: float) -> list[dict]:
+    published = database_numbers()
+    pending = pending_numbers(published)
+    rng = random.Random(seed)
+    groups = {
+        "database": rng.sample(sorted(published), min(per_group, len(published))),
+        "pending": rng.sample(pending, min(per_group, len(pending))),
+        "business": list(BUSINESS_LINES),
+    }
+    return [summarize(name, ask_all(numbers, fetcher, delay_seconds), published) for name, numbers in groups.items()]
+
+
+def percent(count: int, total: int) -> str:
+    return f"{100 * count / total:.1f}%" if total else "n/a"
+
+
+def print_summaries(summaries: list[dict]) -> None:
+    for s in summaries:
+        print(
+            f"{s['group']:<9} asked {s['asked']:>3}, answered {s['answered']:>3}: "
+            f"flagged {s['flagged']:>3} ({percent(s['flagged'], s['answered'])}), "
+            f"adds to the local database {s['adds']:>3} ({percent(s['adds'], s['answered'])}), "
+            f"is_spam without a category the app acts on {s['uncategorized']:>3}, errors {s['errors']}"
+        )
+        if s["add_examples"]:
+            print(f"          adds, first few: {', '.join(s['add_examples'])}")
+
+
 def main() -> int:
+    parser = argparse.ArgumentParser(description="Check or measure the live caller-lookup sources.")
+    parser.add_argument("--sample", action="store_true", help="measure what SkipCalls adds instead of checking its shape")
+    parser.add_argument("--per-group", type=int, default=200, help="numbers drawn from the database and the pending pool")
+    parser.add_argument("--seed", type=int, default=20260930)
+    parser.add_argument("--delay", type=float, default=0.5, help="seconds between requests")
+    args = parser.parse_args()
+    if args.sample:
+        print_summaries(sample(fetch, args.seed, args.per_group, args.delay))
+        return 0
+
     configured = app_source_urls(EXTERNAL_LOOKUP.read_text(encoding="utf-8"))
     probed = {source.url_prefix for source in SOURCES}
     failures = 0
 
@@ -80,5 +80,54 @@ def offline(url):
         self.assertIn("no route to host", detail)
 
 
+class SampleTest(unittest.TestCase):
+    def test_only_the_categories_the_app_acts_on_count_as_flagged(self):
+        self.assertEqual("flagged", probe_live_sources.classify(200, SPAM_ANSWER))
+        self.assertEqual(
+            "uncategorized",
+            probe_live_sources.classify(200, '{"is_spam":true,"status_description":"unknown"}'),
+        )
+        self.assertEqual("clean", probe_live_sources.classify(200, CLEAN_ANSWER))
+        self.assertEqual("error", probe_live_sources.classify(429, ""))
+        self.assertEqual("error", probe_live_sources.classify(200, PARKED_PAGE))
+
+    def test_a_flag_adds_only_when_the_local_database_lacks_the_number(self):
+        answers = [
+            probe_live_sources.Answer("+18333041447", "flagged"),
+            probe_live_sources.Answer("+15550100000", "flagged"),
+            probe_live_sources.Answer("+18002752273", "uncategorized"),
+            probe_live_sources.Answer("+15550100001", "clean"),
+            probe_live_sources.Answer("+15550100002", "error"),
+        ]
+        summary = probe_live_sources.summarize("pending", answers, {"+18333041447"})
+        self.assertEqual(5, summary["asked"])
+        self.assertEqual(4, summary["answered"])
+        self.assertEqual(2, summary["flagged"])
+        self.assertEqual(1, summary["adds"])
+        self.assertEqual(["+15550100000"], summary["add_examples"])
+        self.assertEqual(1, summary["uncategorized"])
+        self.assertEqual(1, summary["errors"])
+
+    def test_numbers_are_asked_as_digits_and_a_dropped_connection_is_an_error(self):
+        asked = []
+
+        def fetcher(url):
+            asked.append(url)
+            if url.endswith("15550100002"):
+                raise OSError("reset")
+            return 200, SPAM_ANSWER
+
+        answers = probe_live_sources.ask_all(["+18333041447", "+15550100002"], fetcher, 0)
+        self.assertEqual(SKIPCALLS.url_prefix + "18333041447", asked[0])
+        self.assertEqual(["flagged", "error"], [a.verdict for a in answers])
+
+    def test_business_lines_are_distinct_e164_numbers(self):
+        lines = probe_live_sources.BUSINESS_LINES
+        self.assertEqual(50, len(lines))
+        self.assertEqual(len(lines), len(set(lines)))
+        for line in lines:
+            self.assertRegex(line, r"^\+1\d{10}$")
+
+
 if __name__ == "__main__":
     unittest.main()