feat: sweep the full dns picture to find who to report a domain to

app/controllers/admin/report/abuse_contacts_controller.rb

@@ -111,7 +111,7 @@ def abuse_contact_params
           :notes,
           web_form_fields: {},
           registrar_patterns: [],
-          nameserver_patterns: [],
+          hostname_patterns: [],
           ip_ranges: []
         )
       end

app/models/report/abuse_contact.rb

@@ -75,17 +75,22 @@ def find_for_registrar(registrar_name)
       end
     end
 
-    # Find contact matching nameservers
-    def find_for_nameservers(nameservers)
-      return nil if nameservers.blank?
-
-      active.find do |contact|
-        patterns = contact.nameserver_patterns || []
-        patterns.any? do |pattern|
-          nameservers.any? { |ns| File.fnmatch?(pattern, ns, File::FNM_CASEFOLD) }
-        end
-      end
+    # Find contact matching any hostname the domain's zone points at
+    #
+    # Nameservers, CNAME targets, reverse lookups and MX exchanges all name a
+    # provider the same way, so one list of globs per contact matches all of
+    # them. A CNAME is often the only record that names the platform serving a
+    # phishing page, because its addresses are shared anycast ones.
+    #
+    # @param hostnames [Array<String>] hostnames from the zone
+    # @return [Report::AbuseContact, nil]
+    def find_for_hostnames(hostnames)
+      hostnames = Array(hostnames).compact_blank
+      return nil if hostnames.empty?
+
+      active.by_priority.find { |contact| contact.matches_hostname?(hostnames) }
     end
+    alias_method :find_for_nameservers, :find_for_hostnames
 
     # Find contact whose published IP ranges cover any of the given addresses
     #
@@ -118,6 +123,30 @@ def parse_ip(address)
 
   # Instance methods
 
+  # Check whether any of the given hostnames matches this contact's patterns
+  #
+  # A bare pattern also matches its own subdomains, so "digitalocean.com"
+  # covers "ns1.digitalocean.com" without every entry needing a glob.
+  #
+  # @param hostnames [Array<String>] hostnames from the zone
+  # @return [Boolean]
+  def matches_hostname?(hostnames)
+    patterns = Array(hostname_patterns).compact_blank
+    return false if patterns.empty?
+
+    names = Array(hostnames).map { |name| name.to_s.downcase.chomp(".") }
+
+    patterns.any? do |pattern|
+      pattern = pattern.to_s.downcase.chomp(".")
+
+      names.any? do |name|
+        File.fnmatch?(pattern, name, File::FNM_CASEFOLD) ||
+          name == pattern ||
+          name.end_with?(".#{pattern}")
+      end
+    end
+  end
+
   # Check whether any of the given addresses falls inside this contact's ranges
   #
   # @param addresses [Array<IPAddr>] parsed addresses

app/models/report/domain_lookup.rb

@@ -61,14 +61,25 @@ def mark_looked_up!(source: nil)
 
   # Update matched contacts based on patterns
   #
-  # Hosting falls back to the addresses the domain resolves to. A phishing site
-  # on a DigitalOcean droplet normally keeps its registrar's nameservers, so
-  # nameserver patterns alone never identify the provider serving the page.
+  # Hosting is matched strongest signal first:
+  #
+  #   1. CNAME and reverse lookups name the platform or machine serving the
+  #      page. A site on platform hosting resolves to a shared anycast address
+  #      that belongs to a CDN, so these are often the only records that name
+  #      the party who can take it down.
+  #   2. The addresses name the network that owns them.
+  #   3. Nameservers name only the DNS operator, which is frequently a
+  #      different company: github.com delegates to Route 53 but is served by
+  #      GitHub, so matching here first would report to the wrong party.
+  #
+  # MX is deliberately absent. It names who carries the mail, not who serves
+  # the page; mail_hosts surfaces it for a human to route by hand.
   def match_contacts!
     self.matched_registrar_contact = Report::AbuseContact.find_for_registrar(registrar_name)
     self.matched_hosting_contact =
-      Report::AbuseContact.find_for_nameservers(nameservers) ||
-      Report::AbuseContact.find_for_ip(resolved_addresses)
+      Report::AbuseContact.find_for_hostnames(serving_hostnames) ||
+      Report::AbuseContact.find_for_ip(resolved_addresses) ||
+      Report::AbuseContact.find_for_hostnames(nameserver_hostnames)
     self.hosting_provider = matched_hosting_contact.name if matched_hosting_contact
     save!
   end
@@ -80,6 +91,39 @@ def resolved_addresses
     Array(a_records) + Array(aaaa_records)
   end
 
+  # Hostnames naming the party that serves the page
+  #
+  # @return [Array<String>]
+  def serving_hostnames
+    Report::DnsSweepService.serving_hostnames(dns_records || {})
+  end
+
+  # Hostnames naming the DNS operator, from the zone and from the registry
+  #
+  # @return [Array<String>]
+  def nameserver_hostnames
+    registry = Array(nameservers).map { |ns| ns.to_s.downcase.chomp(".") }
+
+    (Report::DnsSweepService.nameserver_hostnames(dns_records || {}) + registry)
+      .compact_blank.uniq
+  end
+
+  # Every hostname the zone points at, registry nameservers included
+  #
+  # @return [Array<String>]
+  def resolved_hostnames
+    (serving_hostnames + nameserver_hostnames).uniq
+  end
+
+  # The mail hosts for the domain, for routing the mail side of a report by hand
+  #
+  # @return [Array<String>]
+  def mail_hosts
+    Array((dns_records || {})["MX"]).filter_map do |mx|
+      (mx.is_a?(Hash) ? mx["exchange"] : mx).to_s.downcase.chomp(".").presence
+    end
+  end
+
   # Get all matched contacts
   def matched_contacts
     [ matched_registrar_contact, matched_hosting_contact ].compact.uniq
@@ -98,6 +142,8 @@ def to_summary
       nameservers: nameservers,
       a_records: a_records,
       aaaa_records: aaaa_records,
+      dns_records: dns_records,
+      mail_hosts: mail_hosts,
       hosting_provider: hosting_provider,
       domain_created_at: domain_created_at&.iso8601,
       domain_expires_at: domain_expires_at&.iso8601,

app/services/report/abuse_contact_importer.rb

@@ -122,7 +122,7 @@ def should_update?(key, current_value, new_value)
       when :notes
         # Append notes rather than replace
         false
-      when :registrar_patterns, :nameserver_patterns, :ip_ranges
+      when :registrar_patterns, :hostname_patterns, :ip_ranges
         # Merge arrays
         false
       else

app/services/report/dns_sweep_service.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+module Report
+  # Resolves the full DNS picture for a domain.
+  #
+  # A report is only useful if it reaches the party that can act on it, and no
+  # single record names that party. The addresses find a host that runs its own
+  # ranges; a CNAME names a platform whose addresses are shared anycast ones; a
+  # reverse lookup names the host when the forward records do not; MX names who
+  # carries the mail side. Sweeping the zone gives the matcher hostnames to work
+  # with and gives a human the evidence to route a report by hand.
+  #
+  # Usage:
+  #   records = Report::DnsSweepService.new.sweep("example.com")
+  #   records["CNAME"]  # => ["site.vercel-dns.com"]
+  #
+  # Not swept: DNSKEY and DS have no typed class in Resolv and say nothing about
+  # who to report to, and SRV is only meaningful under a _service._proto label
+  # rather than on the domain itself.
+  class DnsSweepService < BaseService
+    TIMEOUT = 5
+
+    # Reverse lookups cost one query per address, so cap how many are tried.
+    MAX_REVERSE_LOOKUPS = 4
+
+    RECORD_TYPES = {
+      "A" => Resolv::DNS::Resource::IN::A,
+      "AAAA" => Resolv::DNS::Resource::IN::AAAA,
+      "CNAME" => Resolv::DNS::Resource::IN::CNAME,
+      "NS" => Resolv::DNS::Resource::IN::NS,
+      "MX" => Resolv::DNS::Resource::IN::MX,
+      "TXT" => Resolv::DNS::Resource::IN::TXT,
+      "SOA" => Resolv::DNS::Resource::IN::SOA,
+      "CAA" => Resolv::DNS::Resource::IN::CAA
+    }.freeze
+
+    # Sweep every record type for a domain
+    #
+    # @param domain [String] the domain to resolve
+    # @return [Hash] records keyed by type, empty types omitted
+    def sweep(domain)
+      domain = domain.to_s.strip.downcase
+      return {} if domain.blank?
+
+      records = {}
+
+      Timeout.timeout(TIMEOUT) do
+        Resolv::DNS.open do |dns|
+          RECORD_TYPES.each do |label, resource|
+            values = resolve(dns, domain, label, resource)
+            records[label] = values if values.present?
+          end
+        end
+      end
+
+      records["PTR"] = reverse_lookups(records)
+      records.compact_blank
+    rescue StandardError => e
+      # A domain that does not resolve is still worth reporting on its
+      # registration record, so a failed sweep degrades to what was collected.
+      log_debug("DNS sweep failed for #{domain}: #{e.message}")
+      records || {}
+    end
+
+    # Hostnames that name the party serving the content.
+    #
+    # A CNAME target and a reverse lookup both name the machine or platform the
+    # page is actually served from, which is who can take it down.
+    #
+    # @param records [Hash] output of #sweep
+    # @return [Array<String>] hostnames, lowercased and without trailing dots
+    def self.serving_hostnames(records)
+      records = records.to_h.with_indifferent_access
+
+      normalize(
+        Array(records["CNAME"]) +
+        Array(records["PTR"]).map { |_address, name| name }
+      )
+    end
+
+    # Hostnames that name only the DNS operator.
+    #
+    # This is the weakest signal for who hosts a page and is tried last:
+    # github.com delegates to Route 53 but is served by GitHub, so matching on
+    # nameservers first would send the report to the wrong company.
+    #
+    # @param records [Hash] output of #sweep
+    # @return [Array<String>] hostnames, lowercased and without trailing dots
+    def self.nameserver_hostnames(records)
+      normalize(Array(records.to_h.with_indifferent_access["NS"]))
+    end
+
+    # Every hostname the sweep found, in precedence order
+    #
+    # @param records [Hash] output of #sweep
+    # @return [Array<String>]
+    def self.hostnames(records)
+      (serving_hostnames(records) + nameserver_hostnames(records)).uniq
+    end
+
+    def self.normalize(names)
+      names.compact_blank.map { |name| name.to_s.downcase.chomp(".") }.uniq
+    end
+    private_class_method :normalize
+
+    private
+
+    def resolve(dns, domain, label, resource)
+      records = dns.getresources(domain, resource)
+
+      case label
+      when "A", "AAAA" then records.map { |r| r.address.to_s }
+      when "CNAME", "NS" then records.map { |r| r.name.to_s }
+      when "MX" then records.map { |r| { "preference" => r.preference, "exchange" => r.exchange.to_s } }
+      when "TXT" then records.map { |r| r.strings.join }
+      when "SOA" then soa(records.first)
+      when "CAA" then records.map { |r| { "flags" => r.flags, "tag" => r.tag, "value" => r.value } }
+      end
+    rescue StandardError => e
+      log_debug("Could not resolve #{label} for #{domain}: #{e.message}")
+      nil
+    end
+
+    def soa(record)
+      return nil if record.nil?
+
+      {
+        "mname" => record.mname.to_s,
+        "rname" => record.rname.to_s,
+        "serial" => record.serial
+      }
+    end
+
+    # A reverse lookup often names the host outright, which is what a shared
+    # address cannot do.
+    def reverse_lookups(records)
+      addresses = (Array(records["A"]) + Array(records["AAAA"])).first(MAX_REVERSE_LOOKUPS)
+      return {} if addresses.empty?
+
+      addresses.each_with_object({}) do |address, names|
+        name = reverse_lookup(address)
+        names[address] = name if name.present?
+      end
+    end
+
+    def reverse_lookup(address)
+      Timeout.timeout(TIMEOUT) { Resolv.getname(address) }
+    rescue StandardError
+      nil
+    end
+  end
+end

app/services/report/domain_lookup_service.rb

@@ -5,7 +5,6 @@ module Report
   # Uses RDAP (preferred) with WHOIS fallback
   class DomainLookupService < BaseService
     CACHE_TTL = 24.hours
-    DNS_TIMEOUT = 5
 
     # RDAP bootstrap servers by TLD
     # See: https://data.iana.org/rdap/dns.json
@@ -38,10 +37,11 @@ def lookup(domain)
       result = lookup_rdap(domain) || lookup_whois(domain)
 
       if result
-        # The hosting provider is found from the addresses the domain serves
-        # from, not from the registration record, so resolve them here.
-        result[:a_records] = resolve_addresses(domain, Resolv::DNS::Resource::IN::A)
-        result[:aaaa_records] = resolve_addresses(domain, Resolv::DNS::Resource::IN::AAAA)
+        # The hosting provider is named by the zone, not by the registration
+        # record, so sweep it here.
+        result[:dns_records] = DnsSweepService.new(logger: logger).sweep(domain)
+        result[:a_records] = Array(result[:dns_records]["A"])
+        result[:aaaa_records] = Array(result[:dns_records]["AAAA"])
         save_lookup(domain, result)
       else
         log_info("No lookup data found for #{domain}")
@@ -55,32 +55,6 @@ def normalize_domain(domain)
       domain.to_s.downcase.strip.sub(/^www\./, "")
     end
 
-    # Resolve the addresses the domain currently serves from.
-    #
-    # Report::AbuseContact.find_for_ip matches these against the ranges a
-    # hosting provider publishes, which is how a report reaches the provider
-    # that actually serves the phishing page. Both families are resolved:
-    # DigitalOcean and its peers publish IPv6 allocations too, and an
-    # IPv6-only host would otherwise never be matched.
-    #
-    # A domain that does not resolve is still worth reporting on the
-    # registration record alone, so every failure here degrades to no addresses
-    # rather than losing the lookup.
-    #
-    # @param domain [String] normalized domain
-    # @param resource [Class] Resolv::DNS::Resource::IN::A or ::AAAA
-    # @return [Array<String>] addresses, empty when the domain does not resolve
-    def resolve_addresses(domain, resource)
-      Timeout.timeout(DNS_TIMEOUT) do
-        Resolv::DNS.open do |dns|
-          dns.getresources(domain, resource).map { |record| record.address.to_s }
-        end
-      end
-    rescue StandardError => e
-      log_debug("Could not resolve #{resource.name.demodulize} records for #{domain}: #{e.message}")
-      []
-    end
-
     def lookup_rdap(domain)
       tld = domain.split(".").last&.downcase
       rdap_url = RDAP_SERVERS[tld]
@@ -251,6 +225,7 @@ def save_lookup(domain, result)
         nameservers: result[:nameservers],
         a_records: result[:a_records] || [],
         aaaa_records: result[:aaaa_records] || [],
+        dns_records: result[:dns_records] || {},
         lookup_source: result[:lookup_source],
         looked_up_at: Time.current,
         expires_at: CACHE_TTL.from_now