fix(url): match OpenPhish entries by the link's full host

Remote lookups reduce a link to its registrable domain so nothing more
leaves the phone, and OpenPhish got the same URL although it matches a feed
downloaded to the phone. A feed entry on login.example.com couldn't match a
link on that host, and matching the base domain would have flagged its
siblings. An adapter now says whether it matches on the device; OpenPhish
does, and gets the link's scheme and full host (path, query, fragment and
credentials still stripped). It flags the entry's host and anything under
it, not siblings, and an entry on a whole domain still covers its
subdomains. PhishTank still receives only the base domain.

Author: SysAdminDoc

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to CallShield will be documented in this file.
 
 ## Unreleased
 
+- A link in a text is checked against OpenPhish by its full host, so a phishing
+  page on one subdomain is caught without flagging its sibling subdomains.
+  OpenPhish is matched on the phone. PhishTank still receives only the site's
+  base domain.
 - Number details, Lookup and the block explanation describe FCC and FTC
   complaints in words, one line per kind, instead of the importer's field names
   ("FCC callback_business") and the same complaint repeated.

app/src/main/java/com/sysadmindoc/callshield/data/remote/UrlSafetyChecker.kt

@@ -85,7 +85,7 @@ object UrlSafetyChecker {
                     verdict = UrlThreatVerdict.UNKNOWN,
                 )
             }
-            val evidence = lookupUrlEvidence(lookupUrl)
+            val evidence = lookupUrlEvidence(lookupUrl, normalizeOnDeviceLookupUrl(url).ifBlank { lookupUrl })
             evidence.firstOrNull(UrlThreatResult::isMalicious)?.let(::toCheckResult)
                 ?: evidence.firstOrNull()?.let(::toCheckResult)
                 ?: UrlCheckResult(
@@ -99,17 +99,21 @@ object UrlSafetyChecker {
     internal suspend fun checkUrlMatches(url: String): List<UrlCheckResult> {
         val lookupUrl = normalizeRemoteLookupUrl(url)
         if (lookupUrl.isBlank()) return emptyList()
-        return lookupUrlEvidence(lookupUrl)
+        return lookupUrlEvidence(lookupUrl, normalizeOnDeviceLookupUrl(url).ifBlank { lookupUrl })
             .filter(UrlThreatResult::isMalicious)
             .map(::toCheckResult)
     }
 
-    private suspend fun lookupUrlEvidence(lookupUrl: String): List<UrlThreatResult> {
+    private suspend fun lookupUrlEvidence(
+        remoteLookupUrl: String,
+        onDeviceLookupUrl: String = remoteLookupUrl,
+    ): List<UrlThreatResult> {
         if (enabledAdapters.isEmpty()) return emptyList()
         val nowMillis = System.currentTimeMillis()
         return coroutineScope {
             enabledAdapters
                 .map { adapter ->
+                    val lookupUrl = if (adapter.matchesOnDevice) onDeviceLookupUrl else remoteLookupUrl
                     async {
                         val cached = threatCache.get(adapter.source, adapter.sourceVersion, lookupUrl)
                         if (cached != null) {
@@ -216,6 +220,23 @@ object UrlSafetyChecker {
             .toString()
     }
 
+    /**
+     * The link's scheme and full host for a list matched on the phone. The path,
+     * query, fragment and any user name or password still go.
+     */
+    internal fun normalizeOnDeviceLookupUrl(rawUrl: String): String {
+        val parsedUrl = normalizeCandidateUrl(rawUrl).toHttpUrlOrNull() ?: return ""
+        return parsedUrl
+            .newBuilder()
+            .username("")
+            .password("")
+            .encodedPath("/")
+            .query(null)
+            .fragment(null)
+            .build()
+            .toString()
+    }
+
     internal fun clearThreatCacheForTests() {
         threatCache.clear()
     }

app/src/main/java/com/sysadmindoc/callshield/data/remote/UrlThreatAdapters.kt

@@ -141,6 +141,7 @@ internal class OpenPhishThreatAdapter(
     override val source: UrlThreatSource = UrlThreatSource.OPENPHISH
     override val sourceVersion: String = source.defaultVersion
     override val isConfigured: Boolean = true
+    override val matchesOnDevice: Boolean = true
 
     @Volatile
     private var feedSnapshot: FeedSnapshot? = null
@@ -161,7 +162,9 @@ internal class OpenPhishThreatAdapter(
                 UrlThreatCache.DEFAULT_TTL_MILLIS,
                 (snapshot.expiresAtMillis - nowMillis).coerceAtLeast(1L),
             )
-        return if (host != null && host in snapshot.hosts) {
+        // A feed entry names the host a phishing page sits on. It flags that host
+        // and anything under it, but not a sibling on the same domain.
+        return if (host != null && host.parentHosts().any { it in snapshot.hosts }) {
             UrlThreatResult.malicious(
                 source = source,
                 sourceVersion = sourceVersion,
@@ -226,6 +229,9 @@ internal class OpenPhishThreatAdapter(
         }
     }
 
+    /** This host and each parent domain of it, the host first. */
+    private fun String.parentHosts(): Sequence<String> = generateSequence(this) { it.substringAfter('.', "").ifEmpty { null } }
+
     private data class FeedSnapshot(
         val hosts: Set<String>,
         val expiresAtMillis: Long,

app/src/main/java/com/sysadmindoc/callshield/data/remote/UrlThreatModels.kt

@@ -104,6 +104,13 @@ internal interface UrlThreatAdapter {
     val sourceVersion: String
     val isConfigured: Boolean
 
+    /**
+     * True for an adapter that matches against a list downloaded to the phone.
+     * Nothing about the link leaves the phone, so it gets the link's full host;
+     * a remote service gets only the registrable domain.
+     */
+    val matchesOnDevice: Boolean get() = false
+
     suspend fun lookup(
         canonicalUrl: String,
         nowMillis: Long = System.currentTimeMillis(),

app/src/test/java/com/sysadmindoc/callshield/data/remote/OpenPhishThreatAdapterTest.kt

@@ -10,6 +10,41 @@ import org.junit.Assert.assertEquals
 import org.junit.Test
 
 class OpenPhishThreatAdapterTest {
+    private fun feedOf(vararg urls: String): OkHttpClient =
+        OkHttpClient
+            .Builder()
+            .addInterceptor { chain ->
+                Response
+                    .Builder()
+                    .request(chain.request())
+                    .protocol(Protocol.HTTP_1_1)
+                    .code(200)
+                    .message("OK")
+                    .body(urls.joinToString("\n").toResponseBody("text/plain".toMediaType()))
+                    .build()
+            }.build()
+
+    @Test
+    fun `a feed entry on a subdomain flags that host and below it, not its siblings`() =
+        runBlocking {
+            // Links used to reach the adapter as their registrable domain, so a
+            // login.example.test entry never matched a link on login.example.test.
+            val adapter = OpenPhishThreatAdapter(feedOf("https://login.example.test/verify", "https://evil.test/"))
+
+            assertEquals(UrlThreatVerdict.MALICIOUS, adapter.lookup("https://login.example.test/", 0L).verdict)
+            assertEquals(UrlThreatVerdict.MALICIOUS, adapter.lookup("https://eu.login.example.test/", 0L).verdict)
+            assertEquals(UrlThreatVerdict.CLEAN, adapter.lookup("https://shop.example.test/", 0L).verdict)
+            assertEquals(UrlThreatVerdict.CLEAN, adapter.lookup("https://example.test/", 0L).verdict)
+            // An entry on a whole domain still covers its subdomains.
+            assertEquals(UrlThreatVerdict.MALICIOUS, adapter.lookup("https://pay.evil.test/", 0L).verdict)
+        }
+
+    @Test
+    fun `only the on-device feed asks for the full host`() {
+        assertEquals(true, OpenPhishThreatAdapter(feedOf()).matchesOnDevice)
+        assertEquals(false, PhishTankThreatAdapter().matchesOnDevice)
+    }
+
     @Test
     fun `feed waits six hours and a 304 renews the existing hosts`() =
         runBlocking {

app/src/test/java/com/sysadmindoc/callshield/data/remote/UrlSafetyCheckerTest.kt

@@ -97,6 +97,17 @@ class UrlSafetyCheckerTest {
         assertEquals("https://example.test/", normalized)
     }
 
+    @Test
+    fun `an on-device list gets the full host and nothing else`() {
+        // OpenPhish is matched on the phone, so it can see the subdomain its feed
+        // entries name. The path, query, fragment and credentials still go.
+        assertEquals(
+            "https://login.example.test/",
+            UrlSafetyChecker.normalizeOnDeviceLookupUrl("https://alice:secret@login.example.test/pay?token=secret#frag"),
+        )
+        assertEquals("", UrlSafetyChecker.normalizeOnDeviceLookupUrl("not a link"))
+    }
+
     @Test
     fun `remote lookup fails closed when registrable domain cannot be resolved`() {
         val normalized =