From 4fa84ebd611aba984be6797a35948f80e76ec15b Mon Sep 17 00:00:00 2001 From: Jasper Mayone Date: Mon, 7 Sep 2026 13:41:35 -0400 Subject: [PATCH] feat: sweep the full DNS picture to find who to report a domain to Only A and AAAA records were resolved, which answers which addresses serve a domain but not who can take the page down. A phishing site on platform hosting is reached by a CNAME, and its address is a shared anycast one belonging to a CDN rather than to the platform serving the page. Report::DnsSweepService resolves A, AAAA, CNAME, NS, MX, TXT, SOA and CAA, plus a reverse lookup on each address, and stores the lot in dns_records. DNSKEY and DS are left out: Resolv has no typed class for them and they say nothing about who to report to. SRV is left out because it is only meaningful under a _service._proto label, not on the domain itself. Hosting is now matched strongest signal first. CNAME targets and reverse lookups name the platform or machine serving the page, so they go first; the addresses come next; nameservers come last, because they frequently name a different company. github.com delegates to Route 53 but is served by GitHub, so matching nameservers first would have reported it to Amazon. MX is deliberately excluded from matching and surfaced as mail_hosts instead, for a human to route the mail side by hand. nameserver_patterns becomes hostname_patterns, since the same globs identify a provider in a CNAME target or reverse lookup just as well as in an NS record. The old column is left in place, unread, rather than renamed: renaming breaks a rolling deploy, and strong_migrations is right to say so. A later migration can drop it once this release is out. The case report renders the whole picture, so a case that needs manual review carries the evidence to route it. Claude-Session: https://claude.ai/code/session_01UJuHu239wz4x62pLNt651V --- .../admin/report/abuse_contacts_controller.rb | 2 +- app/models/report/abuse_contact.rb | 49 ++++-- app/models/report/domain_lookup.rb | 56 ++++++- app/services/report/abuse_contact_importer.rb | 2 +- app/services/report/dns_sweep_service.rb | 152 ++++++++++++++++++ app/services/report/domain_lookup_service.rb | 37 +---- app/views/reports/case_report.html.erb | 66 ++++++++ ...dd_dns_records_to_report_domain_lookups.rb | 17 ++ ...tname_patterns_to_report_abuse_contacts.rb | 29 ++++ db/schema.rb | 2 + db/seeds/abuse_contacts.rb | 28 ++-- test/models/report/domain_lookup_test.rb | 109 ++++++++++++- .../services/report/dns_sweep_service_test.rb | 75 +++++++++ 13 files changed, 561 insertions(+), 63 deletions(-) create mode 100644 app/services/report/dns_sweep_service.rb create mode 100644 db/migrate/20260907120003_add_dns_records_to_report_domain_lookups.rb create mode 100644 db/migrate/20260907120004_add_hostname_patterns_to_report_abuse_contacts.rb create mode 100644 test/services/report/dns_sweep_service_test.rb diff --git a/app/controllers/admin/report/abuse_contacts_controller.rb b/app/controllers/admin/report/abuse_contacts_controller.rb index a3d5fdf..301e520 100644 --- a/app/controllers/admin/report/abuse_contacts_controller.rb +++ b/app/controllers/admin/report/abuse_contacts_controller.rb @@ -111,7 +111,7 @@ def abuse_contact_params :notes, web_form_fields: {}, registrar_patterns: [], - nameserver_patterns: [], + hostname_patterns: [], ip_ranges: [] ) end diff --git a/app/models/report/abuse_contact.rb b/app/models/report/abuse_contact.rb index dd6eaef..392e39c 100644 --- a/app/models/report/abuse_contact.rb +++ b/app/models/report/abuse_contact.rb @@ -75,17 +75,22 @@ def find_for_registrar(registrar_name) end end - # Find contact matching nameservers - def find_for_nameservers(nameservers) - return nil if nameservers.blank? - - active.find do |contact| - patterns = contact.nameserver_patterns || [] - patterns.any? do |pattern| - nameservers.any? { |ns| File.fnmatch?(pattern, ns, File::FNM_CASEFOLD) } - end - end + # Find contact matching any hostname the domain's zone points at + # + # Nameservers, CNAME targets, reverse lookups and MX exchanges all name a + # provider the same way, so one list of globs per contact matches all of + # them. A CNAME is often the only record that names the platform serving a + # phishing page, because its addresses are shared anycast ones. + # + # @param hostnames [Array] hostnames from the zone + # @return [Report::AbuseContact, nil] + def find_for_hostnames(hostnames) + hostnames = Array(hostnames).compact_blank + return nil if hostnames.empty? + + active.by_priority.find { |contact| contact.matches_hostname?(hostnames) } end + alias_method :find_for_nameservers, :find_for_hostnames # Find contact whose published IP ranges cover any of the given addresses # @@ -118,6 +123,30 @@ def parse_ip(address) # Instance methods + # Check whether any of the given hostnames matches this contact's patterns + # + # A bare pattern also matches its own subdomains, so "digitalocean.com" + # covers "ns1.digitalocean.com" without every entry needing a glob. + # + # @param hostnames [Array] hostnames from the zone + # @return [Boolean] + def matches_hostname?(hostnames) + patterns = Array(hostname_patterns).compact_blank + return false if patterns.empty? + + names = Array(hostnames).map { |name| name.to_s.downcase.chomp(".") } + + patterns.any? do |pattern| + pattern = pattern.to_s.downcase.chomp(".") + + names.any? do |name| + File.fnmatch?(pattern, name, File::FNM_CASEFOLD) || + name == pattern || + name.end_with?(".#{pattern}") + end + end + end + # Check whether any of the given addresses falls inside this contact's ranges # # @param addresses [Array] parsed addresses diff --git a/app/models/report/domain_lookup.rb b/app/models/report/domain_lookup.rb index d680982..9d5f4d5 100644 --- a/app/models/report/domain_lookup.rb +++ b/app/models/report/domain_lookup.rb @@ -61,14 +61,25 @@ def mark_looked_up!(source: nil) # Update matched contacts based on patterns # - # Hosting falls back to the addresses the domain resolves to. A phishing site - # on a DigitalOcean droplet normally keeps its registrar's nameservers, so - # nameserver patterns alone never identify the provider serving the page. + # Hosting is matched strongest signal first: + # + # 1. CNAME and reverse lookups name the platform or machine serving the + # page. A site on platform hosting resolves to a shared anycast address + # that belongs to a CDN, so these are often the only records that name + # the party who can take it down. + # 2. The addresses name the network that owns them. + # 3. Nameservers name only the DNS operator, which is frequently a + # different company: github.com delegates to Route 53 but is served by + # GitHub, so matching here first would report to the wrong party. + # + # MX is deliberately absent. It names who carries the mail, not who serves + # the page; mail_hosts surfaces it for a human to route by hand. def match_contacts! self.matched_registrar_contact = Report::AbuseContact.find_for_registrar(registrar_name) self.matched_hosting_contact = - Report::AbuseContact.find_for_nameservers(nameservers) || - Report::AbuseContact.find_for_ip(resolved_addresses) + Report::AbuseContact.find_for_hostnames(serving_hostnames) || + Report::AbuseContact.find_for_ip(resolved_addresses) || + Report::AbuseContact.find_for_hostnames(nameserver_hostnames) self.hosting_provider = matched_hosting_contact.name if matched_hosting_contact save! end @@ -80,6 +91,39 @@ def resolved_addresses Array(a_records) + Array(aaaa_records) end + # Hostnames naming the party that serves the page + # + # @return [Array] + def serving_hostnames + Report::DnsSweepService.serving_hostnames(dns_records || {}) + end + + # Hostnames naming the DNS operator, from the zone and from the registry + # + # @return [Array] + def nameserver_hostnames + registry = Array(nameservers).map { |ns| ns.to_s.downcase.chomp(".") } + + (Report::DnsSweepService.nameserver_hostnames(dns_records || {}) + registry) + .compact_blank.uniq + end + + # Every hostname the zone points at, registry nameservers included + # + # @return [Array] + def resolved_hostnames + (serving_hostnames + nameserver_hostnames).uniq + end + + # The mail hosts for the domain, for routing the mail side of a report by hand + # + # @return [Array] + def mail_hosts + Array((dns_records || {})["MX"]).filter_map do |mx| + (mx.is_a?(Hash) ? mx["exchange"] : mx).to_s.downcase.chomp(".").presence + end + end + # Get all matched contacts def matched_contacts [ matched_registrar_contact, matched_hosting_contact ].compact.uniq @@ -98,6 +142,8 @@ def to_summary nameservers: nameservers, a_records: a_records, aaaa_records: aaaa_records, + dns_records: dns_records, + mail_hosts: mail_hosts, hosting_provider: hosting_provider, domain_created_at: domain_created_at&.iso8601, domain_expires_at: domain_expires_at&.iso8601, diff --git a/app/services/report/abuse_contact_importer.rb b/app/services/report/abuse_contact_importer.rb index 32047cd..3e7a425 100644 --- a/app/services/report/abuse_contact_importer.rb +++ b/app/services/report/abuse_contact_importer.rb @@ -122,7 +122,7 @@ def should_update?(key, current_value, new_value) when :notes # Append notes rather than replace false - when :registrar_patterns, :nameserver_patterns, :ip_ranges + when :registrar_patterns, :hostname_patterns, :ip_ranges # Merge arrays false else diff --git a/app/services/report/dns_sweep_service.rb b/app/services/report/dns_sweep_service.rb new file mode 100644 index 0000000..fa4d52a --- /dev/null +++ b/app/services/report/dns_sweep_service.rb @@ -0,0 +1,152 @@ +# frozen_string_literal: true + +module Report + # Resolves the full DNS picture for a domain. + # + # A report is only useful if it reaches the party that can act on it, and no + # single record names that party. The addresses find a host that runs its own + # ranges; a CNAME names a platform whose addresses are shared anycast ones; a + # reverse lookup names the host when the forward records do not; MX names who + # carries the mail side. Sweeping the zone gives the matcher hostnames to work + # with and gives a human the evidence to route a report by hand. + # + # Usage: + # records = Report::DnsSweepService.new.sweep("example.com") + # records["CNAME"] # => ["site.vercel-dns.com"] + # + # Not swept: DNSKEY and DS have no typed class in Resolv and say nothing about + # who to report to, and SRV is only meaningful under a _service._proto label + # rather than on the domain itself. + class DnsSweepService < BaseService + TIMEOUT = 5 + + # Reverse lookups cost one query per address, so cap how many are tried. + MAX_REVERSE_LOOKUPS = 4 + + RECORD_TYPES = { + "A" => Resolv::DNS::Resource::IN::A, + "AAAA" => Resolv::DNS::Resource::IN::AAAA, + "CNAME" => Resolv::DNS::Resource::IN::CNAME, + "NS" => Resolv::DNS::Resource::IN::NS, + "MX" => Resolv::DNS::Resource::IN::MX, + "TXT" => Resolv::DNS::Resource::IN::TXT, + "SOA" => Resolv::DNS::Resource::IN::SOA, + "CAA" => Resolv::DNS::Resource::IN::CAA + }.freeze + + # Sweep every record type for a domain + # + # @param domain [String] the domain to resolve + # @return [Hash] records keyed by type, empty types omitted + def sweep(domain) + domain = domain.to_s.strip.downcase + return {} if domain.blank? + + records = {} + + Timeout.timeout(TIMEOUT) do + Resolv::DNS.open do |dns| + RECORD_TYPES.each do |label, resource| + values = resolve(dns, domain, label, resource) + records[label] = values if values.present? + end + end + end + + records["PTR"] = reverse_lookups(records) + records.compact_blank + rescue StandardError => e + # A domain that does not resolve is still worth reporting on its + # registration record, so a failed sweep degrades to what was collected. + log_debug("DNS sweep failed for #{domain}: #{e.message}") + records || {} + end + + # Hostnames that name the party serving the content. + # + # A CNAME target and a reverse lookup both name the machine or platform the + # page is actually served from, which is who can take it down. + # + # @param records [Hash] output of #sweep + # @return [Array] hostnames, lowercased and without trailing dots + def self.serving_hostnames(records) + records = records.to_h.with_indifferent_access + + normalize( + Array(records["CNAME"]) + + Array(records["PTR"]).map { |_address, name| name } + ) + end + + # Hostnames that name only the DNS operator. + # + # This is the weakest signal for who hosts a page and is tried last: + # github.com delegates to Route 53 but is served by GitHub, so matching on + # nameservers first would send the report to the wrong company. + # + # @param records [Hash] output of #sweep + # @return [Array] hostnames, lowercased and without trailing dots + def self.nameserver_hostnames(records) + normalize(Array(records.to_h.with_indifferent_access["NS"])) + end + + # Every hostname the sweep found, in precedence order + # + # @param records [Hash] output of #sweep + # @return [Array] + def self.hostnames(records) + (serving_hostnames(records) + nameserver_hostnames(records)).uniq + end + + def self.normalize(names) + names.compact_blank.map { |name| name.to_s.downcase.chomp(".") }.uniq + end + private_class_method :normalize + + private + + def resolve(dns, domain, label, resource) + records = dns.getresources(domain, resource) + + case label + when "A", "AAAA" then records.map { |r| r.address.to_s } + when "CNAME", "NS" then records.map { |r| r.name.to_s } + when "MX" then records.map { |r| { "preference" => r.preference, "exchange" => r.exchange.to_s } } + when "TXT" then records.map { |r| r.strings.join } + when "SOA" then soa(records.first) + when "CAA" then records.map { |r| { "flags" => r.flags, "tag" => r.tag, "value" => r.value } } + end + rescue StandardError => e + log_debug("Could not resolve #{label} for #{domain}: #{e.message}") + nil + end + + def soa(record) + return nil if record.nil? + + { + "mname" => record.mname.to_s, + "rname" => record.rname.to_s, + "serial" => record.serial + } + end + + # A reverse lookup often names the host outright, which is what a shared + # address cannot do. + def reverse_lookups(records) + addresses = (Array(records["A"]) + Array(records["AAAA"])).first(MAX_REVERSE_LOOKUPS) + return {} if addresses.empty? + + addresses.each_with_object({}) do |address, names| + name = reverse_lookup(address) + names[address] = name if name.present? + end + end + + def reverse_lookup(address) + Timeout.timeout(TIMEOUT) { Resolv.getname(address) } + rescue StandardError + nil + end + end +end diff --git a/app/services/report/domain_lookup_service.rb b/app/services/report/domain_lookup_service.rb index f67a1fb..b3411fb 100644 --- a/app/services/report/domain_lookup_service.rb +++ b/app/services/report/domain_lookup_service.rb @@ -5,7 +5,6 @@ module Report # Uses RDAP (preferred) with WHOIS fallback class DomainLookupService < BaseService CACHE_TTL = 24.hours - DNS_TIMEOUT = 5 # RDAP bootstrap servers by TLD # See: https://data.iana.org/rdap/dns.json @@ -38,10 +37,11 @@ def lookup(domain) result = lookup_rdap(domain) || lookup_whois(domain) if result - # The hosting provider is found from the addresses the domain serves - # from, not from the registration record, so resolve them here. - result[:a_records] = resolve_addresses(domain, Resolv::DNS::Resource::IN::A) - result[:aaaa_records] = resolve_addresses(domain, Resolv::DNS::Resource::IN::AAAA) + # The hosting provider is named by the zone, not by the registration + # record, so sweep it here. + result[:dns_records] = DnsSweepService.new(logger: logger).sweep(domain) + result[:a_records] = Array(result[:dns_records]["A"]) + result[:aaaa_records] = Array(result[:dns_records]["AAAA"]) save_lookup(domain, result) else log_info("No lookup data found for #{domain}") @@ -55,32 +55,6 @@ def normalize_domain(domain) domain.to_s.downcase.strip.sub(/^www\./, "") end - # Resolve the addresses the domain currently serves from. - # - # Report::AbuseContact.find_for_ip matches these against the ranges a - # hosting provider publishes, which is how a report reaches the provider - # that actually serves the phishing page. Both families are resolved: - # DigitalOcean and its peers publish IPv6 allocations too, and an - # IPv6-only host would otherwise never be matched. - # - # A domain that does not resolve is still worth reporting on the - # registration record alone, so every failure here degrades to no addresses - # rather than losing the lookup. - # - # @param domain [String] normalized domain - # @param resource [Class] Resolv::DNS::Resource::IN::A or ::AAAA - # @return [Array] addresses, empty when the domain does not resolve - def resolve_addresses(domain, resource) - Timeout.timeout(DNS_TIMEOUT) do - Resolv::DNS.open do |dns| - dns.getresources(domain, resource).map { |record| record.address.to_s } - end - end - rescue StandardError => e - log_debug("Could not resolve #{resource.name.demodulize} records for #{domain}: #{e.message}") - [] - end - def lookup_rdap(domain) tld = domain.split(".").last&.downcase rdap_url = RDAP_SERVERS[tld] @@ -251,6 +225,7 @@ def save_lookup(domain, result) nameservers: result[:nameservers], a_records: result[:a_records] || [], aaaa_records: result[:aaaa_records] || [], + dns_records: result[:dns_records] || {}, lookup_source: result[:lookup_source], looked_up_at: Time.current, expires_at: CACHE_TTL.from_now diff --git a/app/views/reports/case_report.html.erb b/app/views/reports/case_report.html.erb index 4bcada4..752e3b1 100644 --- a/app/views/reports/case_report.html.erb +++ b/app/views/reports/case_report.html.erb @@ -124,6 +124,72 @@ <% end %> + <% if @domain_info.dig("dns_records", "CNAME").present? %> + + CNAME + +
    + <% @domain_info["dns_records"]["CNAME"].each do |name| %> +
  • <%= name %>
    • + <% end %> +
+ + + <% end %> + <% if @domain_info.dig("dns_records", "PTR").present? %> + + Reverse Lookups + +
    + <% @domain_info["dns_records"]["PTR"].each do |address, name| %> +
  • <%= address %> → <%= name %>
    • + <% end %> +
+ + + <% end %> + <% if @domain_info["mail_hosts"].present? %> + + Mail Hosts (MX) + +
    + <% @domain_info["mail_hosts"].each do |host| %> +
  • <%= host %>
    • + <% end %> +
+ + + <% end %> + <% if @domain_info.dig("dns_records", "TXT").present? %> + + TXT + +
    + <% @domain_info["dns_records"]["TXT"].each do |txt| %> +
  • <%= txt.truncate(200) %>
    • + <% end %> +
+ + + <% end %> + <% if @domain_info.dig("dns_records", "CAA").present? %> + + CAA + +
    + <% @domain_info["dns_records"]["CAA"].each do |caa| %> +
  • <%= caa["tag"] %> <%= caa["value"] %>
    • + <% end %> +
+ + + <% end %> + <% if @domain_info.dig("dns_records", "SOA", "mname").present? %> + + SOA + <%= @domain_info["dns_records"]["SOA"]["mname"] %> + + <% end %> <% end %> diff --git a/db/migrate/20260907120003_add_dns_records_to_report_domain_lookups.rb b/db/migrate/20260907120003_add_dns_records_to_report_domain_lookups.rb new file mode 100644 index 0000000..73101de --- /dev/null +++ b/db/migrate/20260907120003_add_dns_records_to_report_domain_lookups.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +# Only A and AAAA records were resolved, which answers "which addresses serve +# this" but not "who do I report this to". A phishing site on platform hosting +# is reached by a CNAME, and its address is a shared anycast one that belongs +# to a CDN rather than the platform serving the page. +# +# dns_records holds the whole zone picture keyed by record type, so a case +# carries the evidence a human needs to route a report by hand and the matcher +# has hostnames to work with. a_records and aaaa_records stay as they are: the +# abuse contact matcher reads them on every case, and the domain_info snapshot +# already stored on existing cases uses those keys. +class AddDnsRecordsToReportDomainLookups < ActiveRecord::Migration[8.1] + def change + add_column :report_domain_lookups, :dns_records, :jsonb, default: {} + end +end diff --git a/db/migrate/20260907120004_add_hostname_patterns_to_report_abuse_contacts.rb b/db/migrate/20260907120004_add_hostname_patterns_to_report_abuse_contacts.rb new file mode 100644 index 0000000..4b7aefb --- /dev/null +++ b/db/migrate/20260907120004_add_hostname_patterns_to_report_abuse_contacts.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +# nameserver_patterns was only ever matched against a domain's nameservers, so +# a provider was found only when the site also used that provider's DNS. The +# same globs identify a provider just as well in a CNAME target, a reverse +# lookup or an MX exchange, and those are what actually name the host of a +# phishing page. +# +# Renaming the column outright breaks a rolling deploy, because the running +# release still reads the old name. So this adds the new column, copies what is +# there, and the model moves its reads over. nameserver_patterns is left in +# place, unread, for a later migration to drop once this release is out. +class AddHostnamePatternsToReportAbuseContacts < ActiveRecord::Migration[8.1] + def up + add_column :report_abuse_contacts, :hostname_patterns, :jsonb, default: [] + + Report::AbuseContact.reset_column_information + + say_with_time "copying nameserver_patterns into hostname_patterns" do + Report::AbuseContact.unscoped.update_all( + "hostname_patterns = COALESCE(nameserver_patterns, '[]'::jsonb)" + ) + end + end + + def down + remove_column :report_abuse_contacts, :hostname_patterns + end +end diff --git a/db/schema.rb b/db/schema.rb index 75fa296..0be5440 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -577,6 +577,7 @@ t.datetime "created_at", null: false t.datetime "discarded_at" t.string "email" + t.jsonb "hostname_patterns", default: [] t.jsonb "ip_ranges", default: [] t.enum "method", null: false, enum_type: "report_contact_method" t.string "name", null: false @@ -651,6 +652,7 @@ t.jsonb "a_records", default: [] t.jsonb "aaaa_records", default: [] t.datetime "created_at", null: false + t.jsonb "dns_records", default: {} t.string "domain", null: false t.datetime "domain_created_at" t.datetime "domain_expires_at" diff --git a/db/seeds/abuse_contacts.rb b/db/seeds/abuse_contacts.rb index 0ac8bb8..76b89ef 100644 --- a/db/seeds/abuse_contacts.rb +++ b/db/seeds/abuse_contacts.rb @@ -11,45 +11,45 @@ class AbuseContacts # Registrars (priority: 10) { name: "1API", contact_type: :registrar, method: :email, email: "abuse@1api.net", priority: 10 }, { name: "Alibaba Cloud", contact_type: :registrar, method: :email, email: "infosec@service.aliyun.com", priority: 10 }, - { name: "Cloudflare", contact_type: :registrar, method: :web_form, web_form_url: "https://abuse.cloudflare.com/", priority: 10, notes: "The abuse@cloudflare.com email will not work and will redirect you to the form." }, + { name: "Cloudflare", contact_type: :registrar, method: :web_form, web_form_url: "https://abuse.cloudflare.com/", priority: 10, hostname_patterns: [ "cloudflare.com", "cloudflare.net", "pages.dev" ], notes: "The abuse@cloudflare.com email will not work and will redirect you to the form." }, { name: "Communigal Communications", contact_type: :registrar, method: :email, email: "abuse@galcomm.com", priority: 10 }, { name: "Cosmotown", contact_type: :registrar, method: :email, email: "abuse@cosmotown.com", priority: 10 }, { name: "Eranet International Limited", contact_type: :registrar, method: :email, email: "support@tnet.hk", priority: 10 }, { name: "GMO Internet Inc.d/b/a Onamae.com", contact_type: :registrar, method: :email, email: "abuse@gmo.jp", priority: 10 }, { name: "GNAME", contact_type: :registrar, method: :web_form, web_form_url: "https://www.gname.com/abuse", priority: 10, notes: "If your complaint gets denied, email complaint@gname.com." }, - { name: "GoDaddy", contact_type: :registrar, method: :web_form, web_form_url: "https://supportcenter.godaddy.com/AbuseReport", priority: 10 }, + { name: "GoDaddy", contact_type: :registrar, method: :web_form, web_form_url: "https://supportcenter.godaddy.com/AbuseReport", priority: 10, hostname_patterns: [ "domaincontrol.com", "secureserver.net", "godaddy.com" ] }, { name: "Host Arabia", contact_type: :registrar, method: :email, email: "dom@tasjeel.ae", priority: 10 }, - { name: "Hostinger", contact_type: :registrar, method: :web_form, email: "abuse@hostinger.com", web_form_url: "https://www.hostinger.com/report-abuse", priority: 10 }, + { name: "Hostinger", contact_type: :registrar, method: :web_form, email: "abuse@hostinger.com", web_form_url: "https://www.hostinger.com/report-abuse", priority: 10, hostname_patterns: [ "hostinger.com", "hostingersite.com" ] }, { name: "Kouming", contact_type: :registrar, method: :email, email: "abuse@kouming.com", priority: 10 }, { name: "MarkMonitor Inc", contact_type: :registrar, method: :email, email: "abusecomplaints@markmonitor.com", priority: 10 }, { name: "Metaregistrar BV", contact_type: :registrar, method: :email, email: "abuse@metaregistrar.com", priority: 10 }, - { name: "NameSilo", contact_type: :registrar, method: :web_form, web_form_url: "https://new.namesilo.com/phishing_report.php", priority: 10 }, + { name: "NameSilo", contact_type: :registrar, method: :web_form, web_form_url: "https://new.namesilo.com/phishing_report.php", priority: 10, hostname_patterns: [ "namesilo.com" ] }, { name: "NiceNIC.NET", contact_type: :registrar, method: :web_form, email: "abuse@nicenic.net", web_form_url: "https://nicenic.net/customer/reportabuse.php", priority: 10, notes: "Use the form for faster response. Use an alt email when contacting." }, { name: "OwnRegistrar", contact_type: :registrar, method: :email, email: "abuse@ownregistrar.com", priority: 10 }, - { name: "Porkbun", contact_type: :registrar, method: :web_form, web_form_url: "https://porkbun.com/abuse", priority: 10 }, + { name: "Porkbun", contact_type: :registrar, method: :web_form, web_form_url: "https://porkbun.com/abuse", priority: 10, hostname_patterns: [ "porkbun.com" ] }, { name: "REG.RU", contact_type: :registrar, method: :email, email: "abuse@reg.ru", priority: 10, notes: "Due to Russian laws, also contact incident@cert.gov.ru who will instruct reg.ru to suspend the domain." }, { name: "RU-CENTER", contact_type: :registrar, method: :email, email: "tld-abuse@nic.ru", priority: 10 }, { name: "Registrar.eu", contact_type: :registrar, method: :web_form, email: "support@openprovider.zendesk.com", web_form_url: "https://abuse.registrar.eu/", priority: 10 }, { name: "Regtons", contact_type: :registrar, method: :email, email: "abuse@regtons.com", priority: 10 }, { name: "Rumahweb Indonesia", contact_type: :registrar, method: :email, email: "abuse@rumahweb.co.id", priority: 10 }, { name: "Sav.com, LLC", contact_type: :registrar, method: :web_form, email: "abuse-contact@sav.com", web_form_url: "https://abuse.sav.com/", priority: 10, notes: "The email will not work, use the form." }, - { name: "Tucows", contact_type: :registrar, method: :web_form, web_form_url: "https://tucowsdomains.com/report-abuse/", priority: 10 }, + { name: "Tucows", contact_type: :registrar, method: :web_form, web_form_url: "https://tucowsdomains.com/report-abuse/", priority: 10, hostname_patterns: [ "tucows.com", "hover.com" ] }, { name: "webnic.cc", contact_type: :registrar, method: :email, email: "compliance_abuse@webnic.cc", priority: 10 }, # Hosting Providers (priority: 30) - { name: "Amazon AWS", contact_type: :hosting, method: :email, email: "abuse@amazonaws.com", priority: 30 }, - { name: "Deno Deploy", contact_type: :hosting, method: :email, email: "deploy@deno.com", priority: 30, notes: "Handles deno.dev domains." }, - { name: "DigitalOcean", contact_type: :hosting, method: :email, email: "abuse@digitalocean.com", priority: 30, accepts_xarf: true, nameserver_patterns: [ "ns1.digitalocean.com", "ns2.digitalocean.com", "ns3.digitalocean.com" ], notes: "The mailbox is processed by automated tooling that only accepts X-ARF reports, so accepts_xarf must stay on. DigitalOceanRangeSyncJob keeps ip_ranges current from https://digitalocean.com/geo/google.csv - do not edit them by hand." }, + { name: "Amazon AWS", contact_type: :hosting, method: :email, email: "abuse@amazonaws.com", priority: 30, hostname_patterns: [ "amazonaws.com", "cloudfront.net", "*awsdns-*" ] }, + { name: "Deno Deploy", contact_type: :hosting, method: :email, email: "deploy@deno.com", priority: 30, hostname_patterns: [ "deno.dev" ], notes: "Handles deno.dev domains." }, + { name: "DigitalOcean", contact_type: :hosting, method: :email, email: "abuse@digitalocean.com", priority: 30, accepts_xarf: true, hostname_patterns: [ "digitalocean.com", "digitaloceanspaces.com" ], notes: "The mailbox is processed by automated tooling that only accepts X-ARF reports, so accepts_xarf must stay on. DigitalOceanRangeSyncJob keeps ip_ranges current from https://digitalocean.com/geo/google.csv - do not edit them by hand." }, { name: "ESITED", contact_type: :hosting, method: :email, email: "net-abuse@esited.com", priority: 30 }, - { name: "Google Cloud", contact_type: :hosting, method: :email, email: "google-cloud-compliance@google.com", priority: 30 }, + { name: "Google Cloud", contact_type: :hosting, method: :email, email: "google-cloud-compliance@google.com", priority: 30, hostname_patterns: [ "googleusercontent.com", "googlehosted.com", "googledomains.com" ] }, { name: "IQWeb FZ-LLC", contact_type: :hosting, method: :email, email: "abuse@iqweb.io", priority: 30 }, { name: "MTW-AS (RU)", contact_type: :hosting, method: :email, email: "support@ruvds.com", priority: 30 }, { name: "ROUTERHOSTING / Cloudzy", contact_type: :hosting, method: :email, email: "abuse-reports@cloudzy.com", priority: 30 }, { name: "SEDO-AS (DE)", contact_type: :hosting, method: :email, email: "ripe@internetx.de", priority: 30 }, { name: "TRELLIAN-AS-AP", contact_type: :hosting, method: :email, email: "abuse@trellian.com", priority: 30 }, - { name: "Vercel", contact_type: :hosting, method: :web_form, email: "abuse@vercel.com", web_form_url: "https://vercel.com/abuse", priority: 30 }, + { name: "Vercel", contact_type: :hosting, method: :web_form, email: "abuse@vercel.com", web_form_url: "https://vercel.com/abuse", priority: 30, hostname_patterns: [ "vercel-dns.com", "vercel.app", "vercel.com" ] }, { name: "ZhouyiSat Communications", contact_type: :hosting, method: :email, email: "support@62yun.com", priority: 30 }, - { name: "000webhost.com", contact_type: :hosting, method: :web_form, web_form_url: "https://www.000webhost.com/report-abuse", priority: 30 }, + { name: "000webhost.com", contact_type: :hosting, method: :web_form, web_form_url: "https://www.000webhost.com/report-abuse", priority: 30, hostname_patterns: [ "000webhost.com", "000webhostapp.com" ] }, # Link Shorteners (priority: 50) { name: "bit.ly", contact_type: :other, method: :web_form, web_form_url: "https://bitly.com/pages/trust/report-abuse", priority: 50 }, @@ -86,8 +86,8 @@ class AbuseContacts { name: "Dropbox", contact_type: :other, method: :email, email: "abuse@dropbox.com", priority: 50 }, { name: "SugarSync", contact_type: :other, method: :email, email: "support@sugarsync.zendesk.com", priority: 50 }, { name: "OpenProvider", contact_type: :other, method: :web_form, web_form_url: "https://www.openprovider.com/company/contact-us/report-abuse", priority: 50 }, - { name: "Wix", contact_type: :other, method: :web_form, web_form_url: "http://www.wix.com/upgrade/abuse#!spam-report/c18hy", priority: 50 }, - { name: "Weebly", contact_type: :other, method: :web_form, web_form_url: "https://www.weebly.com/spam", priority: 50 }, + { name: "Wix", contact_type: :other, method: :web_form, web_form_url: "http://www.wix.com/upgrade/abuse#!spam-report/c18hy", priority: 50, hostname_patterns: [ "wixdns.net", "wixsite.com", "wix.com" ] }, + { name: "Weebly", contact_type: :other, method: :web_form, web_form_url: "https://www.weebly.com/spam", priority: 50, hostname_patterns: [ "weebly.com", "weeblysite.com" ] }, # Security Vendors (priority: 20) - These receive all high-confidence reports # API keys must be configured in credentials or via admin UI diff --git a/test/models/report/domain_lookup_test.rb b/test/models/report/domain_lookup_test.rb index 54f8f77..0f10afc 100644 --- a/test/models/report/domain_lookup_test.rb +++ b/test/models/report/domain_lookup_test.rb @@ -10,7 +10,7 @@ def lookup_for(attrs = {}) end test "a nameserver match still wins" do - host = create_test_abuse_contact(nameserver_patterns: [ "ns1.examplehost.com" ]) + host = create_test_abuse_contact(hostname_patterns: [ "ns1.examplehost.com" ]) lookup = lookup_for(nameservers: [ "ns1.examplehost.com" ]) lookup.match_contacts! @@ -45,6 +45,113 @@ def lookup_for(attrs = {}) assert_nil lookup.matched_hosting_contact end + test "a CNAME target names the platform when the address does not" do + platform = create_test_abuse_contact(hostname_patterns: [ "vercel-dns.com" ]) + # The address is a shared anycast one in nobody's published range. + lookup = lookup_for( + nameservers: [ "ns1.somewhere-else.com" ], + a_records: [ "76.76.21.21" ], + dns_records: { "CNAME" => [ "cname.vercel-dns.com." ] } + ) + + lookup.match_contacts! + + assert_equal platform, lookup.matched_hosting_contact + end + + test "a reverse lookup names the host when nothing else does" do + host = create_test_abuse_contact(hostname_patterns: [ "digitalocean.com" ]) + lookup = lookup_for( + a_records: [ "24.144.65.10" ], + dns_records: { "PTR" => { "24.144.65.10" => "droplet.digitalocean.com." } } + ) + + lookup.match_contacts! + + assert_equal host, lookup.matched_hosting_contact + end + + test "a bare pattern covers the subdomains under it" do + host = create_test_abuse_contact(hostname_patterns: [ "digitalocean.com" ]) + lookup = lookup_for(dns_records: { "CNAME" => [ "a.b.digitalocean.com" ] }) + + lookup.match_contacts! + + assert_equal host, lookup.matched_hosting_contact + end + + test "a pattern does not match a lookalike domain" do + create_test_abuse_contact(hostname_patterns: [ "digitalocean.com" ]) + lookup = lookup_for(dns_records: { "CNAME" => [ "notdigitalocean.com" ] }) + + lookup.match_contacts! + + assert_nil lookup.matched_hosting_contact + end + + test "a serving hostname is tried before the address" do + by_hostname = create_test_abuse_contact(hostname_patterns: [ "vercel-dns.com" ], priority: 30) + create_test_abuse_contact(ip_ranges: [ "76.76.0.0/16" ], priority: 30) + lookup = lookup_for( + a_records: [ "76.76.21.21" ], + dns_records: { "CNAME" => [ "cname.vercel-dns.com" ] } + ) + + lookup.match_contacts! + + assert_equal by_hostname, lookup.matched_hosting_contact + end + + # github.com delegates to Route 53 but is served by GitHub. Matching on + # nameservers first would send the report to Amazon. + test "the address beats a nameserver that names a different company" do + create_test_abuse_contact(hostname_patterns: [ "*awsdns-*" ], priority: 30) + network = create_test_abuse_contact(ip_ranges: [ "140.82.112.0/20" ], priority: 30) + lookup = lookup_for( + a_records: [ "140.82.114.3" ], + dns_records: { "NS" => [ "ns-1707.awsdns-21.co.uk" ] } + ) + + lookup.match_contacts! + + assert_equal network, lookup.matched_hosting_contact + end + + test "a nameserver still matches when nothing stronger does" do + dns_operator = create_test_abuse_contact(hostname_patterns: [ "cloudflare.com" ]) + lookup = lookup_for(dns_records: { "NS" => [ "doug.ns.cloudflare.com" ] }) + + lookup.match_contacts! + + assert_equal dns_operator, lookup.matched_hosting_contact + end + + test "a mail host never decides who hosts the page" do + create_test_abuse_contact(hostname_patterns: [ "aspmx.l.google.com" ]) + lookup = lookup_for( + dns_records: { "MX" => [ { "preference" => 1, "exchange" => "aspmx.l.google.com" } ] } + ) + + lookup.match_contacts! + + assert_nil lookup.matched_hosting_contact + end + + test "the mail hosts are surfaced for routing the mail side by hand" do + lookup = lookup_for( + dns_records: { "MX" => [ { "preference" => 10, "exchange" => "mx.example.com." } ] } + ) + + assert_equal [ "mx.example.com" ], lookup.mail_hosts + assert_equal [ "mx.example.com" ], lookup.to_summary[:mail_hosts] + end + + test "the full record set travels with the case summary" do + lookup = lookup_for(dns_records: { "TXT" => [ "v=spf1 include:_spf.google.com ~all" ] }) + + assert_equal [ "v=spf1 include:_spf.google.com ~all" ], lookup.to_summary[:dns_records]["TXT"] + end + test "an IPv6-only host is matched on its AAAA records" do host = create_test_abuse_contact(ip_ranges: [ "2604:a880::/32" ]) lookup = lookup_for(a_records: [], aaaa_records: [ "2604:a880::1" ]) diff --git a/test/services/report/dns_sweep_service_test.rb b/test/services/report/dns_sweep_service_test.rb new file mode 100644 index 0000000..3bf53ed --- /dev/null +++ b/test/services/report/dns_sweep_service_test.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +require "test_helper" + +# A phishing site on platform hosting resolves to a shared anycast address that +# belongs to a CDN, not to the platform serving the page. The CNAME and the +# reverse lookup are what name the party who can take it down, so the sweep has +# to surface them for the matcher and for a human reading the case. +class Report::DnsSweepServiceTest < ActiveSupport::TestCase + def records + { + "A" => [ "76.76.21.21" ], + "AAAA" => [ "2606:4700::1" ], + "CNAME" => [ "cname.vercel-dns.com." ], + "NS" => [ "ns1.vercel-dns.com." ], + "MX" => [ { "preference" => 10, "exchange" => "mx.example.com." } ], + "PTR" => { "76.76.21.21" => "cname.vercel-dns.com." } + } + end + + test "the serving hostnames are the ones that name who runs the page" do + serving = Report::DnsSweepService.serving_hostnames(records) + + assert_equal [ "cname.vercel-dns.com" ], serving + assert_not_includes serving, "ns1.vercel-dns.com", "a nameserver is not a serving host" + assert_not_includes serving, "mx.example.com", "a mail host is not a serving host" + end + + test "nameservers are kept apart as the weaker signal" do + assert_equal [ "ns1.vercel-dns.com" ], Report::DnsSweepService.nameserver_hostnames(records) + end + + test "the combined list puts serving hostnames first" do + hostnames = Report::DnsSweepService.hostnames(records) + + assert_operator hostnames.index("cname.vercel-dns.com"), :<, + hostnames.index("ns1.vercel-dns.com") + end + + test "trailing dots are stripped so patterns match" do + Report::DnsSweepService.hostnames(records).each do |hostname| + assert_not hostname.end_with?("."), "#{hostname} kept its trailing dot" + end + end + + test "hostnames are deduplicated" do + duplicated = { "CNAME" => [ "a.example.com." ], "NS" => [ "A.EXAMPLE.COM" ] } + + assert_equal [ "a.example.com" ], Report::DnsSweepService.hostnames(duplicated) + end + + test "an empty sweep yields no hostnames" do + assert_empty Report::DnsSweepService.hostnames({}) + end + + test "a blank domain is not queried" do + assert_empty Report::DnsSweepService.new.sweep("") + assert_empty Report::DnsSweepService.new.sweep(nil) + end + + test "the record types swept are the ones that name a party to report to" do + types = Report::DnsSweepService::RECORD_TYPES.keys + + assert_equal %w[A AAAA CNAME NS MX TXT SOA CAA], types + end + + test "a domain that does not resolve yields no records rather than raising" do + result = nil + + assert_nothing_raised do + result = Report::DnsSweepService.new.sweep("does-not-exist-#{SecureRandom.hex(8)}.invalid") + end + assert_empty result + end +end