Skip to content

Commit 2b048d6

Browse files
committed
chore: drop the dead sunlei/denylist community text list source
The repo answers 404 (gone, probed 2026-10-10), so every import printed a skip and the source contributed nothing while the manifest still listed it. The fetch, its manifest entry and its source name are removed. The block also wrote its counts under community_reports, which the merge owns, so the importer no longer claims that source fresh. A test pins the manifest's source list and keeps the retired URL out of the importer.
1 parent accb763 commit 2b048d6

5 files changed

Lines changed: 28 additions & 68 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,9 @@ All notable changes to CallShield will be documented in this file.
160160
block those 10,000 numbers, with Undo. It never offers your own exchange
161161
or one where a contact's number is, so it needs access to your contacts
162162
to check. North American numbers only.
163+
- The data import no longer asks for the sunlei/denylist text list. That
164+
GitHub repository is gone, so every import got a 404 and skipped it, and
165+
it never added a number. Its source entry is gone too.
163166

164167
## v1.11.0 (2026-09-30)
165168

‎data/source-manifest.json‎

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -115,20 +115,6 @@
115115
"stale_after_days": 30,
116116
"evidence_ttl_days": 3650
117117
},
118-
{
119-
"id": "community_text_lists",
120-
"access_mode": "public_feed",
121-
"geography": "global",
122-
"license": "Upstream list terms; verify before redistribution",
123-
"attribution": "Community-maintained text blocklists",
124-
"cadence": "on demand",
125-
"parser_version": "community-text-v1",
126-
"evidence_type": "community_curated",
127-
"confidence_tier": "unverified",
128-
"redistributable": false,
129-
"stale_after_days": 30,
130-
"evidence_ttl_days": 365
131-
},
132118
{
133119
"id": "maintainer_review",
134120
"access_mode": "maintainer_review",

‎scripts/import_all_sources.py‎

Lines changed: 0 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -873,47 +873,6 @@ def fetch_toastedspam(allow_insecure: bool = False) -> list[dict]:
873873
return list(numbers.values())
874874

875875

876-
# ── Source 5: ScamCallers community text lists (GitHub mirror) ─────────
877-
def fetch_community_text_lists() -> list[dict]:
878-
print("\n[Community Text Blocklists]")
879-
sources = [
880-
# Repo-hosted plain-text spam number lists (one number per line, +1XXXXXXXXXX)
881-
("https://raw.githubusercontent.com/sunlei/denylist/master/denylist.txt", "sunlei/denylist"),
882-
]
883-
numbers = {}
884-
for url, name in sources:
885-
try:
886-
resp = requests.get(url, timeout=20)
887-
if not resp.ok:
888-
print(f" {name}: HTTP {resp.status_code}, skipping")
889-
continue
890-
count = 0
891-
for line in resp.text.splitlines():
892-
line = line.strip()
893-
if not line or line.startswith('#'):
894-
continue
895-
normalized = normalize_phone(line)
896-
if normalized:
897-
if normalized in numbers:
898-
numbers[normalized]["reports"] += 1
899-
else:
900-
numbers[normalized] = {
901-
"number": normalized,
902-
"type": "robocall",
903-
"reports": 1,
904-
"first_seen": datetime.now().strftime("%Y-%m-%d"),
905-
"last_seen": datetime.now().strftime("%Y-%m-%d"),
906-
"description": f"Community list: {name}",
907-
}
908-
count += 1
909-
print(f" {name}: {count:,} numbers")
910-
except Exception as e:
911-
print(f" {name}: Error — {e}")
912-
913-
print(f" Total from community lists: {len(numbers):,} unique numbers")
914-
return list(numbers.values())
915-
916-
917876
COMPLAINT_SOURCE_IDS = {"ftc_complaints", "fcc_complaints"}
918877

919878

@@ -1289,17 +1248,6 @@ def main():
12891248
"last_success_at": retrieved_at if ts else None,
12901249
}
12911250

1292-
# Source 5: Community text lists
1293-
cl = fetch_community_text_lists()
1294-
attach_source_evidence(cl, manifest, "community_text_lists", retrieved_at=retrieved_at)
1295-
all_numbers.extend(cl)
1296-
source_stats["community_reports"] = {
1297-
"status": "ok",
1298-
"accepted": len(cl),
1299-
"checksum": payload_checksum(cl),
1300-
"last_success_at": retrieved_at,
1301-
}
1302-
13031251
# Deduplicate across all sources (accumulate reports)
13041252
deduped = {}
13051253
for n in all_numbers:
@@ -1325,7 +1273,6 @@ def main():
13251273
"fcc_complaints",
13261274
"toastedspam",
13271275
"community_reports",
1328-
"community_text_lists",
13291276
}
13301277
if pb:
13311278
source_names.add("phoneblock_bulk")

‎scripts/test_release_drift.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ def test_checkout_is_synchronized(self) -> None:
3737
self.assertEqual([], report["issues"], report)
3838
self.assertEqual("1.11.0", report["version_name"])
3939
self.assertEqual(71, report["version_code"])
40-
self.assertEqual(10, report["sources"]["source_count"])
40+
self.assertEqual(9, report["sources"]["source_count"])
4141
self.assertEqual(3, len(report["advisories"]))
4242

4343
def test_readme_resource_counts_must_match_the_resource_file(self) -> None:

‎scripts/test_source_registry.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,30 @@ def test_real_manifest_keeps_number_evidence_for_at_least_a_year(self):
226226
self.assertGreaterEqual(ttl[source_id], 365, source_id)
227227
self.assertGreaterEqual(ttl["saracroche_prefixes"], 90)
228228

229+
def test_real_manifest_lists_exactly_the_sources_the_pipeline_uses(self):
230+
# sunlei/denylist answered 404 for every import and stayed listed until
231+
# 2026-10-10. Adding or dropping a source changes this list on purpose.
232+
manifest = source_registry.load_source_manifest(Path(__file__).parent.parent / "data" / "source-manifest.json")
233+
self.assertEqual(
234+
[source["id"] for source in manifest["sources"]],
235+
[
236+
"ftc_complaints",
237+
"fcc_complaints",
238+
"phoneblock_bulk",
239+
"saracroche_prefixes",
240+
"nomorobo_irs",
241+
"toastedspam",
242+
"community_reports",
243+
"github_database",
244+
"maintainer_review",
245+
],
246+
)
247+
248+
def test_importer_requests_no_retired_source(self):
249+
importer = (Path(__file__).parent / "import_all_sources.py").read_text(encoding="utf-8")
250+
for retired in ("sunlei/denylist", "community_text_lists"):
251+
self.assertNotIn(retired, importer)
252+
229253
def test_refresh_recomputes_expiry_from_retrieval_time_once(self):
230254
manifest = {"version": 1, "sources": [self._source("fcc", 14, ttl=365)]}
231255
rows = [

0 commit comments

Comments
 (0)