diff --git a/app/controllers/admin/report/abuse_contacts_controller.rb b/app/controllers/admin/report/abuse_contacts_controller.rb index a3d5fdf..301e520 100644 --- a/app/controllers/admin/report/abuse_contacts_controller.rb +++ b/app/controllers/admin/report/abuse_contacts_controller.rb @@ -111,7 +111,7 @@ def abuse_contact_params :notes, web_form_fields: {}, registrar_patterns: [], - nameserver_patterns: [], + hostname_patterns: [], ip_ranges: [] ) end diff --git a/app/models/report/abuse_contact.rb b/app/models/report/abuse_contact.rb index dd6eaef..392e39c 100644 --- a/app/models/report/abuse_contact.rb +++ b/app/models/report/abuse_contact.rb @@ -75,17 +75,22 @@ def find_for_registrar(registrar_name) end end - # Find contact matching nameservers - def find_for_nameservers(nameservers) - return nil if nameservers.blank? - - active.find do |contact| - patterns = contact.nameserver_patterns || [] - patterns.any? do |pattern| - nameservers.any? { |ns| File.fnmatch?(pattern, ns, File::FNM_CASEFOLD) } - end - end + # Find contact matching any hostname the domain's zone points at + # + # Nameservers, CNAME targets, reverse lookups and MX exchanges all name a + # provider the same way, so one list of globs per contact matches all of + # them. A CNAME is often the only record that names the platform serving a + # phishing page, because its addresses are shared anycast ones. + # + # @param hostnames [Array] hostnames from the zone + # @return [Report::AbuseContact, nil] + def find_for_hostnames(hostnames) + hostnames = Array(hostnames).compact_blank + return nil if hostnames.empty? + + active.by_priority.find { |contact| contact.matches_hostname?(hostnames) } end + alias_method :find_for_nameservers, :find_for_hostnames # Find contact whose published IP ranges cover any of the given addresses # @@ -118,6 +123,30 @@ def parse_ip(address) # Instance methods + # Check whether any of the given hostnames matches this contact's patterns + # + # A bare pattern also matches its own subdomains, so "digitalocean.com" + # covers "ns1.digitalocean.com" without every entry needing a glob. + # + # @param hostnames [Array] hostnames from the zone + # @return [Boolean] + def matches_hostname?(hostnames) + patterns = Array(hostname_patterns).compact_blank + return false if patterns.empty? + + names = Array(hostnames).map { |name| name.to_s.downcase.chomp(".") } + + patterns.any? do |pattern| + pattern = pattern.to_s.downcase.chomp(".") + + names.any? do |name| + File.fnmatch?(pattern, name, File::FNM_CASEFOLD) || + name == pattern || + name.end_with?(".#{pattern}") + end + end + end + # Check whether any of the given addresses falls inside this contact's ranges # # @param addresses [Array] parsed addresses diff --git a/app/models/report/domain_lookup.rb b/app/models/report/domain_lookup.rb index d680982..9d5f4d5 100644 --- a/app/models/report/domain_lookup.rb +++ b/app/models/report/domain_lookup.rb @@ -61,14 +61,25 @@ def mark_looked_up!(source: nil) # Update matched contacts based on patterns # - # Hosting falls back to the addresses the domain resolves to. A phishing site - # on a DigitalOcean droplet normally keeps its registrar's nameservers, so - # nameserver patterns alone never identify the provider serving the page. + # Hosting is matched strongest signal first: + # + # 1. CNAME and reverse lookups name the platform or machine serving the + # page. A site on platform hosting resolves to a shared anycast address + # that belongs to a CDN, so these are often the only records that name + # the party who can take it down. + # 2. The addresses name the network that owns them. + # 3. Nameservers name only the DNS operator, which is frequently a + # different company: github.com delegates to Route 53 but is served by + # GitHub, so matching here first would report to the wrong party. + # + # MX is deliberately absent. It names who carries the mail, not who serves + # the page; mail_hosts surfaces it for a human to route by hand. def match_contacts! self.matched_registrar_contact = Report::AbuseContact.find_for_registrar(registrar_name) self.matched_hosting_contact = - Report::AbuseContact.find_for_nameservers(nameservers) || - Report::AbuseContact.find_for_ip(resolved_addresses) + Report::AbuseContact.find_for_hostnames(serving_hostnames) || + Report::AbuseContact.find_for_ip(resolved_addresses) || + Report::AbuseContact.find_for_hostnames(nameserver_hostnames) self.hosting_provider = matched_hosting_contact.name if matched_hosting_contact save! end @@ -80,6 +91,39 @@ def resolved_addresses Array(a_records) + Array(aaaa_records) end + # Hostnames naming the party that serves the page + # + # @return [Array] + def serving_hostnames + Report::DnsSweepService.serving_hostnames(dns_records || {}) + end + + # Hostnames naming the DNS operator, from the zone and from the registry + # + # @return [Array] + def nameserver_hostnames + registry = Array(nameservers).map { |ns| ns.to_s.downcase.chomp(".") } + + (Report::DnsSweepService.nameserver_hostnames(dns_records || {}) + registry) + .compact_blank.uniq + end + + # Every hostname the zone points at, registry nameservers included + # + # @return [Array] + def resolved_hostnames + (serving_hostnames + nameserver_hostnames).uniq + end + + # The mail hosts for the domain, for routing the mail side of a report by hand + # + # @return [Array] + def mail_hosts + Array((dns_records || {})["MX"]).filter_map do |mx| + (mx.is_a?(Hash) ? mx["exchange"] : mx).to_s.downcase.chomp(".").presence + end + end + # Get all matched contacts def matched_contacts [ matched_registrar_contact, matched_hosting_contact ].compact.uniq @@ -98,6 +142,8 @@ def to_summary nameservers: nameservers, a_records: a_records, aaaa_records: aaaa_records, + dns_records: dns_records, + mail_hosts: mail_hosts, hosting_provider: hosting_provider, domain_created_at: domain_created_at&.iso8601, domain_expires_at: domain_expires_at&.iso8601, diff --git a/app/services/report/abuse_contact_importer.rb b/app/services/report/abuse_contact_importer.rb index 32047cd..3e7a425 100644 --- a/app/services/report/abuse_contact_importer.rb +++ b/app/services/report/abuse_contact_importer.rb @@ -122,7 +122,7 @@ def should_update?(key, current_value, new_value) when :notes # Append notes rather than replace false - when :registrar_patterns, :nameserver_patterns, :ip_ranges + when :registrar_patterns, :hostname_patterns, :ip_ranges # Merge arrays false else diff --git a/app/services/report/dns_sweep_service.rb b/app/services/report/dns_sweep_service.rb new file mode 100644 index 0000000..fa4d52a --- /dev/null +++ b/app/services/report/dns_sweep_service.rb @@ -0,0 +1,152 @@ +# frozen_string_literal: true + +module Report + # Resolves the full DNS picture for a domain. + # + # A report is only useful if it reaches the party that can act on it, and no + # single record names that party. The addresses find a host that runs its own + # ranges; a CNAME names a platform whose addresses are shared anycast ones; a + # reverse lookup names the host when the forward records do not; MX names who + # carries the mail side. Sweeping the zone gives the matcher hostnames to work + # with and gives a human the evidence to route a report by hand. + # + # Usage: + # records = Report::DnsSweepService.new.sweep("example.com") + # records["CNAME"] # => ["site.vercel-dns.com"] + # + # Not swept: DNSKEY and DS have no typed class in Resolv and say nothing about + # who to report to, and SRV is only meaningful under a _service._proto label + # rather than on the domain itself. + class DnsSweepService < BaseService + TIMEOUT = 5 + + # Reverse lookups cost one query per address, so cap how many are tried. + MAX_REVERSE_LOOKUPS = 4 + + RECORD_TYPES = { + "A" => Resolv::DNS::Resource::IN::A, + "AAAA" => Resolv::DNS::Resource::IN::AAAA, + "CNAME" => Resolv::DNS::Resource::IN::CNAME, + "NS" => Resolv::DNS::Resource::IN::NS, + "MX" => Resolv::DNS::Resource::IN::MX, + "TXT" => Resolv::DNS::Resource::IN::TXT, + "SOA" => Resolv::DNS::Resource::IN::SOA, + "CAA" => Resolv::DNS::Resource::IN::CAA + }.freeze + + # Sweep every record type for a domain + # + # @param domain [String] the domain to resolve + # @return [Hash] records keyed by type, empty types omitted + def sweep(domain) + domain = domain.to_s.strip.downcase + return {} if domain.blank? + + records = {} + + Timeout.timeout(TIMEOUT) do + Resolv::DNS.open do |dns| + RECORD_TYPES.each do |label, resource| + values = resolve(dns, domain, label, resource) + records[label] = values if values.present? + end + end + end + + records["PTR"] = reverse_lookups(records) + records.compact_blank + rescue StandardError => e + # A domain that does not resolve is still worth reporting on its + # registration record, so a failed sweep degrades to what was collected. + log_debug("DNS sweep failed for #{domain}: #{e.message}") + records || {} + end + + # Hostnames that name the party serving the content. + # + # A CNAME target and a reverse lookup both name the machine or platform the + # page is actually served from, which is who can take it down. + # + # @param records [Hash] output of #sweep + # @return [Array] hostnames, lowercased and without trailing dots + def self.serving_hostnames(records) + records = records.to_h.with_indifferent_access + + normalize( + Array(records["CNAME"]) + + Array(records["PTR"]).map { |_address, name| name } + ) + end + + # Hostnames that name only the DNS operator. + # + # This is the weakest signal for who hosts a page and is tried last: + # github.com delegates to Route 53 but is served by GitHub, so matching on + # nameservers first would send the report to the wrong company. + # + # @param records [Hash] output of #sweep + # @return [Array] hostnames, lowercased and without trailing dots + def self.nameserver_hostnames(records) + normalize(Array(records.to_h.with_indifferent_access["NS"])) + end + + # Every hostname the sweep found, in precedence order + # + # @param records [Hash] output of #sweep + # @return [Array] + def self.hostnames(records) + (serving_hostnames(records) + nameserver_hostnames(records)).uniq + end + + def self.normalize(names) + names.compact_blank.map { |name| name.to_s.downcase.chomp(".") }.uniq + end + private_class_method :normalize + + private + + def resolve(dns, domain, label, resource) + records = dns.getresources(domain, resource) + + case label + when "A", "AAAA" then records.map { |r| r.address.to_s } + when "CNAME", "NS" then records.map { |r| r.name.to_s } + when "MX" then records.map { |r| { "preference" => r.preference, "exchange" => r.exchange.to_s } } + when "TXT" then records.map { |r| r.strings.join } + when "SOA" then soa(records.first) + when "CAA" then records.map { |r| { "flags" => r.flags, "tag" => r.tag, "value" => r.value } } + end + rescue StandardError => e + log_debug("Could not resolve #{label} for #{domain}: #{e.message}") + nil + end + + def soa(record) + return nil if record.nil? + + { + "mname" => record.mname.to_s, + "rname" => record.rname.to_s, + "serial" => record.serial + } + end + + # A reverse lookup often names the host outright, which is what a shared + # address cannot do. + def reverse_lookups(records) + addresses = (Array(records["A"]) + Array(records["AAAA"])).first(MAX_REVERSE_LOOKUPS) + return {} if addresses.empty? + + addresses.each_with_object({}) do |address, names| + name = reverse_lookup(address) + names[address] = name if name.present? + end + end + + def reverse_lookup(address) + Timeout.timeout(TIMEOUT) { Resolv.getname(address) } + rescue StandardError + nil + end + end +end diff --git a/app/services/report/domain_lookup_service.rb b/app/services/report/domain_lookup_service.rb index f67a1fb..b3411fb 100644 --- a/app/services/report/domain_lookup_service.rb +++ b/app/services/report/domain_lookup_service.rb @@ -5,7 +5,6 @@ module Report # Uses RDAP (preferred) with WHOIS fallback class DomainLookupService < BaseService CACHE_TTL = 24.hours - DNS_TIMEOUT = 5 # RDAP bootstrap servers by TLD # See: https://data.iana.org/rdap/dns.json @@ -38,10 +37,11 @@ def lookup(domain) result = lookup_rdap(domain) || lookup_whois(domain) if result - # The hosting provider is found from the addresses the domain serves - # from, not from the registration record, so resolve them here. - result[:a_records] = resolve_addresses(domain, Resolv::DNS::Resource::IN::A) - result[:aaaa_records] = resolve_addresses(domain, Resolv::DNS::Resource::IN::AAAA) + # The hosting provider is named by the zone, not by the registration + # record, so sweep it here. + result[:dns_records] = DnsSweepService.new(logger: logger).sweep(domain) + result[:a_records] = Array(result[:dns_records]["A"]) + result[:aaaa_records] = Array(result[:dns_records]["AAAA"]) save_lookup(domain, result) else log_info("No lookup data found for #{domain}") @@ -55,32 +55,6 @@ def normalize_domain(domain) domain.to_s.downcase.strip.sub(/^www\./, "") end - # Resolve the addresses the domain currently serves from. - # - # Report::AbuseContact.find_for_ip matches these against the ranges a - # hosting provider publishes, which is how a report reaches the provider - # that actually serves the phishing page. Both families are resolved: - # DigitalOcean and its peers publish IPv6 allocations too, and an - # IPv6-only host would otherwise never be matched. - # - # A domain that does not resolve is still worth reporting on the - # registration record alone, so every failure here degrades to no addresses - # rather than losing the lookup. - # - # @param domain [String] normalized domain - # @param resource [Class] Resolv::DNS::Resource::IN::A or ::AAAA - # @return [Array] addresses, empty when the domain does not resolve - def resolve_addresses(domain, resource) - Timeout.timeout(DNS_TIMEOUT) do - Resolv::DNS.open do |dns| - dns.getresources(domain, resource).map { |record| record.address.to_s } - end - end - rescue StandardError => e - log_debug("Could not resolve #{resource.name.demodulize} records for #{domain}: #{e.message}") - [] - end - def lookup_rdap(domain) tld = domain.split(".").last&.downcase rdap_url = RDAP_SERVERS[tld] @@ -251,6 +225,7 @@ def save_lookup(domain, result) nameservers: result[:nameservers], a_records: result[:a_records] || [], aaaa_records: result[:aaaa_records] || [], + dns_records: result[:dns_records] || {}, lookup_source: result[:lookup_source], looked_up_at: Time.current, expires_at: CACHE_TTL.from_now diff --git a/app/views/reports/case_report.html.erb b/app/views/reports/case_report.html.erb index 4bcada4..752e3b1 100644 --- a/app/views/reports/case_report.html.erb +++ b/app/views/reports/case_report.html.erb @@ -124,6 +124,72 @@ <% end %> + <% if @domain_info.dig("dns_records", "CNAME").present? %> + + CNAME + +
    + <% @domain_info["dns_records"]["CNAME"].each do |name| %> +
  • <%= name %>
    • + <% end %> +
+ + + <% end %> + <% if @domain_info.dig("dns_records", "PTR").present? %> + + Reverse Lookups + +
    + <% @domain_info["dns_records"]["PTR"].each do |address, name| %> +
  • <%= address %> → <%= name %>
    • + <% end %> +
+ + + <% end %> + <% if @domain_info["mail_hosts"].present? %> + + Mail Hosts (MX) + +
    + <% @domain_info["mail_hosts"].each do |host| %> +
  • <%= host %>
    • + <% end %> +
+ + + <% end %> + <% if @domain_info.dig("dns_records", "TXT").present? %> + + TXT + +
    + <% @domain_info["dns_records"]["TXT"].each do |txt| %> +
  • <%= txt.truncate(200) %>
    • + <% end %> +
+ + + <% end %> + <% if @domain_info.dig("dns_records", "CAA").present? %> + + CAA + +
    + <% @domain_info["dns_records"]["CAA"].each do |caa| %> +
  • <%= caa["tag"] %> <%= caa["value"] %>
    • + <% end %> +
+ + + <% end %> + <% if @domain_info.dig("dns_records", "SOA", "mname").present? %> + + SOA + <%= @domain_info["dns_records"]["SOA"]["mname"] %> + + <% end %> <% end %> diff --git a/db/migrate/20260907120003_add_dns_records_to_report_domain_lookups.rb b/db/migrate/20260907120003_add_dns_records_to_report_domain_lookups.rb new file mode 100644 index 0000000..73101de --- /dev/null +++ b/db/migrate/20260907120003_add_dns_records_to_report_domain_lookups.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +# Only A and AAAA records were resolved, which answers "which addresses serve +# this" but not "who do I report this to". A phishing site on platform hosting +# is reached by a CNAME, and its address is a shared anycast one that belongs +# to a CDN rather than the platform serving the page. +# +# dns_records holds the whole zone picture keyed by record type, so a case +# carries the evidence a human needs to route a report by hand and the matcher +# has hostnames to work with. a_records and aaaa_records stay as they are: the +# abuse contact matcher reads them on every case, and the domain_info snapshot +# already stored on existing cases uses those keys. +class AddDnsRecordsToReportDomainLookups < ActiveRecord::Migration[8.1] + def change + add_column :report_domain_lookups, :dns_records, :jsonb, default: {} + end +end diff --git a/db/migrate/20260907120004_add_hostname_patterns_to_report_abuse_contacts.rb b/db/migrate/20260907120004_add_hostname_patterns_to_report_abuse_contacts.rb new file mode 100644 index 0000000..4b7aefb --- /dev/null +++ b/db/migrate/20260907120004_add_hostname_patterns_to_report_abuse_contacts.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +# nameserver_patterns was only ever matched against a domain's nameservers, so +# a provider was found only when the site also used that provider's DNS. The +# same globs identify a provider just as well in a CNAME target, a reverse +# lookup or an MX exchange, and those are what actually name the host of a +# phishing page. +# +# Renaming the column outright breaks a rolling deploy, because the running +# release still reads the old name. So this adds the new column, copies what is +# there, and the model moves its reads over. nameserver_patterns is left in +# place, unread, for a later migration to drop once this release is out. +class AddHostnamePatternsToReportAbuseContacts < ActiveRecord::Migration[8.1] + def up + add_column :report_abuse_contacts, :hostname_patterns, :jsonb, default: [] + + Report::AbuseContact.reset_column_information + + say_with_time "copying nameserver_patterns into hostname_patterns" do + Report::AbuseContact.unscoped.update_all( + "hostname_patterns = COALESCE(nameserver_patterns, '[]'::jsonb)" + ) + end + end + + def down + remove_column :report_abuse_contacts, :hostname_patterns + end +end diff --git a/db/schema.rb b/db/schema.rb index 75fa296..0be5440 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -577,6 +577,7 @@ t.datetime "created_at", null: false t.datetime "discarded_at" t.string "email" + t.jsonb "hostname_patterns", default: [] t.jsonb "ip_ranges", default: [] t.enum "method", null: false, enum_type: "report_contact_method" t.string "name", null: false @@ -651,6 +652,7 @@ t.jsonb "a_records", default: [] t.jsonb "aaaa_records", default: [] t.datetime "created_at", null: false + t.jsonb "dns_records", default: {} t.string "domain", null: false t.datetime "domain_created_at" t.datetime "domain_expires_at" diff --git a/db/seeds/abuse_contacts.rb b/db/seeds/abuse_contacts.rb index 0ac8bb8..76b89ef 100644 --- a/db/seeds/abuse_contacts.rb +++ b/db/seeds/abuse_contacts.rb @@ -11,45 +11,45 @@ class AbuseContacts # Registrars (priority: 10) { name: "1API", contact_type: :registrar, method: :email, email: "abuse@1api.net", priority: 10 }, { name: "Alibaba Cloud", contact_type: :registrar, method: :email, email: "infosec@service.aliyun.com", priority: 10 }, - { name: "Cloudflare", contact_type: :registrar, method: :web_form, web_form_url: "https://abuse.cloudflare.com/", priority: 10, notes: "The abuse@cloudflare.com email will not work and will redirect you to the form." }, + { name: "Cloudflare", contact_type: :registrar, method: :web_form, web_form_url: "https://abuse.cloudflare.com/", priority: 10, hostname_patterns: [ "cloudflare.com", "cloudflare.net", "pages.dev" ], notes: "The abuse@cloudflare.com email will not work and will redirect you to the form." }, { name: "Communigal Communications", contact_type: :registrar, method: :email, email: "abuse@galcomm.com", priority: 10 }, { name: "Cosmotown", contact_type: :registrar, method: :email, email: "abuse@cosmotown.com", priority: 10 }, { name: "Eranet International Limited", contact_type: :registrar, method: :email, email: "support@tnet.hk", priority: 10 }, { name: "GMO Internet Inc.d/b/a Onamae.com", contact_type: :registrar, method: :email, email: "abuse@gmo.jp", priority: 10 }, { name: "GNAME", contact_type: :registrar, method: :web_form, web_form_url: "https://www.gname.com/abuse", priority: 10, notes: "If your complaint gets denied, email complaint@gname.com." }, - { name: "GoDaddy", contact_type: :registrar, method: :web_form, web_form_url: "https://supportcenter.godaddy.com/AbuseReport", priority: 10 }, + { name: "GoDaddy", contact_type: :registrar, method: :web_form, web_form_url: "https://supportcenter.godaddy.com/AbuseReport", priority: 10, hostname_patterns: [ "domaincontrol.com", "secureserver.net", "godaddy.com" ] }, { name: "Host Arabia", contact_type: :registrar, method: :email, email: "dom@tasjeel.ae", priority: 10 }, - { name: "Hostinger", contact_type: :registrar, method: :web_form, email: "abuse@hostinger.com", web_form_url: "https://www.hostinger.com/report-abuse", priority: 10 }, + { name: "Hostinger", contact_type: :registrar, method: :web_form, email: "abuse@hostinger.com", web_form_url: "https://www.hostinger.com/report-abuse", priority: 10, hostname_patterns: [ "hostinger.com", "hostingersite.com" ] }, { name: "Kouming", contact_type: :registrar, method: :email, email: "abuse@kouming.com", priority: 10 }, { name: "MarkMonitor Inc", contact_type: :registrar, method: :email, email: "abusecomplaints@markmonitor.com", priority: 10 }, { name: "Metaregistrar BV", contact_type: :registrar, method: :email, email: "abuse@metaregistrar.com", priority: 10 }, - { name: "NameSilo", contact_type: :registrar, method: :web_form, web_form_url: "https://new.namesilo.com/phishing_report.php", priority: 10 }, + { name: "NameSilo", contact_type: :registrar, method: :web_form, web_form_url: "https://new.namesilo.com/phishing_report.php", priority: 10, hostname_patterns: [ "namesilo.com" ] }, { name: "NiceNIC.NET", contact_type: :registrar, method: :web_form, email: "abuse@nicenic.net", web_form_url: "https://nicenic.net/customer/reportabuse.php", priority: 10, notes: "Use the form for faster response. Use an alt email when contacting." }, { name: "OwnRegistrar", contact_type: :registrar, method: :email, email: "abuse@ownregistrar.com", priority: 10 }, - { name: "Porkbun", contact_type: :registrar, method: :web_form, web_form_url: "https://porkbun.com/abuse", priority: 10 }, + { name: "Porkbun", contact_type: :registrar, method: :web_form, web_form_url: "https://porkbun.com/abuse", priority: 10, hostname_patterns: [ "porkbun.com" ] }, { name: "REG.RU", contact_type: :registrar, method: :email, email: "abuse@reg.ru", priority: 10, notes: "Due to Russian laws, also contact incident@cert.gov.ru who will instruct reg.ru to suspend the domain." }, { name: "RU-CENTER", contact_type: :registrar, method: :email, email: "tld-abuse@nic.ru", priority: 10 }, { name: "Registrar.eu", contact_type: :registrar, method: :web_form, email: "support@openprovider.zendesk.com", web_form_url: "https://abuse.registrar.eu/", priority: 10 }, { name: "Regtons", contact_type: :registrar, method: :email, email: "abuse@regtons.com", priority: 10 }, { name: "Rumahweb Indonesia", contact_type: :registrar, method: :email, email: "abuse@rumahweb.co.id", priority: 10 }, { name: "Sav.com, LLC", contact_type: :registrar, method: :web_form, email: "abuse-contact@sav.com", web_form_url: "https://abuse.sav.com/", priority: 10, notes: "The email will not work, use the form." }, - { name: "Tucows", contact_type: :registrar, method: :web_form, web_form_url: "https://tucowsdomains.com/report-abuse/", priority: 10 }, + { name: "Tucows", contact_type: :registrar, method: :web_form, web_form_url: "https://tucowsdomains.com/report-abuse/", priority: 10, hostname_patterns: [ "tucows.com", "hover.com" ] }, { name: "webnic.cc", contact_type: :registrar, method: :email, email: "compliance_abuse@webnic.cc", priority: 10 }, # Hosting Providers (priority: 30) - { name: "Amazon AWS", contact_type: :hosting, method: :email, email: "abuse@amazonaws.com", priority: 30 }, - { name: "Deno Deploy", contact_type: :hosting, method: :email, email: "deploy@deno.com", priority: 30, notes: "Handles deno.dev domains." }, - { name: "DigitalOcean", contact_type: :hosting, method: :email, email: "abuse@digitalocean.com", priority: 30, accepts_xarf: true, nameserver_patterns: [ "ns1.digitalocean.com", "ns2.digitalocean.com", "ns3.digitalocean.com" ], notes: "The mailbox is processed by automated tooling that only accepts X-ARF reports, so accepts_xarf must stay on. DigitalOceanRangeSyncJob keeps ip_ranges current from https://digitalocean.com/geo/google.csv - do not edit them by hand." }, + { name: "Amazon AWS", contact_type: :hosting, method: :email, email: "abuse@amazonaws.com", priority: 30, hostname_patterns: [ "amazonaws.com", "cloudfront.net", "*awsdns-*" ] }, + { name: "Deno Deploy", contact_type: :hosting, method: :email, email: "deploy@deno.com", priority: 30, hostname_patterns: [ "deno.dev" ], notes: "Handles deno.dev domains." }, + { name: "DigitalOcean", contact_type: :hosting, method: :email, email: "abuse@digitalocean.com", priority: 30, accepts_xarf: true, hostname_patterns: [ "digitalocean.com", "digitaloceanspaces.com" ], notes: "The mailbox is processed by automated tooling that only accepts X-ARF reports, so accepts_xarf must stay on. DigitalOceanRangeSyncJob keeps ip_ranges current from https://digitalocean.com/geo/google.csv - do not edit them by hand." }, { name: "ESITED", contact_type: :hosting, method: :email, email: "net-abuse@esited.com", priority: 30 }, - { name: "Google Cloud", contact_type: :hosting, method: :email, email: "google-cloud-compliance@google.com", priority: 30 }, + { name: "Google Cloud", contact_type: :hosting, method: :email, email: "google-cloud-compliance@google.com", priority: 30, hostname_patterns: [ "googleusercontent.com", "googlehosted.com", "googledomains.com" ] }, { name: "IQWeb FZ-LLC", contact_type: :hosting, method: :email, email: "abuse@iqweb.io", priority: 30 }, { name: "MTW-AS (RU)", contact_type: :hosting, method: :email, email: "support@ruvds.com", priority: 30 }, { name: "ROUTERHOSTING / Cloudzy", contact_type: :hosting, method: :email, email: "abuse-reports@cloudzy.com", priority: 30 }, { name: "SEDO-AS (DE)", contact_type: :hosting, method: :email, email: "ripe@internetx.de", priority: 30 }, { name: "TRELLIAN-AS-AP", contact_type: :hosting, method: :email, email: "abuse@trellian.com", priority: 30 }, - { name: "Vercel", contact_type: :hosting, method: :web_form, email: "abuse@vercel.com", web_form_url: "https://vercel.com/abuse", priority: 30 }, + { name: "Vercel", contact_type: :hosting, method: :web_form, email: "abuse@vercel.com", web_form_url: "https://vercel.com/abuse", priority: 30, hostname_patterns: [ "vercel-dns.com", "vercel.app", "vercel.com" ] }, { name: "ZhouyiSat Communications", contact_type: :hosting, method: :email, email: "support@62yun.com", priority: 30 }, - { name: "000webhost.com", contact_type: :hosting, method: :web_form, web_form_url: "https://www.000webhost.com/report-abuse", priority: 30 }, + { name: "000webhost.com", contact_type: :hosting, method: :web_form, web_form_url: "https://www.000webhost.com/report-abuse", priority: 30, hostname_patterns: [ "000webhost.com", "000webhostapp.com" ] }, # Link Shorteners (priority: 50) { name: "bit.ly", contact_type: :other, method: :web_form, web_form_url: "https://bitly.com/pages/trust/report-abuse", priority: 50 }, @@ -86,8 +86,8 @@ class AbuseContacts { name: "Dropbox", contact_type: :other, method: :email, email: "abuse@dropbox.com", priority: 50 }, { name: "SugarSync", contact_type: :other, method: :email, email: "support@sugarsync.zendesk.com", priority: 50 }, { name: "OpenProvider", contact_type: :other, method: :web_form, web_form_url: "https://www.openprovider.com/company/contact-us/report-abuse", priority: 50 }, - { name: "Wix", contact_type: :other, method: :web_form, web_form_url: "http://www.wix.com/upgrade/abuse#!spam-report/c18hy", priority: 50 }, - { name: "Weebly", contact_type: :other, method: :web_form, web_form_url: "https://www.weebly.com/spam", priority: 50 }, + { name: "Wix", contact_type: :other, method: :web_form, web_form_url: "http://www.wix.com/upgrade/abuse#!spam-report/c18hy", priority: 50, hostname_patterns: [ "wixdns.net", "wixsite.com", "wix.com" ] }, + { name: "Weebly", contact_type: :other, method: :web_form, web_form_url: "https://www.weebly.com/spam", priority: 50, hostname_patterns: [ "weebly.com", "weeblysite.com" ] }, # Security Vendors (priority: 20) - These receive all high-confidence reports # API keys must be configured in credentials or via admin UI diff --git a/test/models/report/domain_lookup_test.rb b/test/models/report/domain_lookup_test.rb index 54f8f77..0f10afc 100644 --- a/test/models/report/domain_lookup_test.rb +++ b/test/models/report/domain_lookup_test.rb @@ -10,7 +10,7 @@ def lookup_for(attrs = {}) end test "a nameserver match still wins" do - host = create_test_abuse_contact(nameserver_patterns: [ "ns1.examplehost.com" ]) + host = create_test_abuse_contact(hostname_patterns: [ "ns1.examplehost.com" ]) lookup = lookup_for(nameservers: [ "ns1.examplehost.com" ]) lookup.match_contacts! @@ -45,6 +45,113 @@ def lookup_for(attrs = {}) assert_nil lookup.matched_hosting_contact end + test "a CNAME target names the platform when the address does not" do + platform = create_test_abuse_contact(hostname_patterns: [ "vercel-dns.com" ]) + # The address is a shared anycast one in nobody's published range. + lookup = lookup_for( + nameservers: [ "ns1.somewhere-else.com" ], + a_records: [ "76.76.21.21" ], + dns_records: { "CNAME" => [ "cname.vercel-dns.com." ] } + ) + + lookup.match_contacts! + + assert_equal platform, lookup.matched_hosting_contact + end + + test "a reverse lookup names the host when nothing else does" do + host = create_test_abuse_contact(hostname_patterns: [ "digitalocean.com" ]) + lookup = lookup_for( + a_records: [ "24.144.65.10" ], + dns_records: { "PTR" => { "24.144.65.10" => "droplet.digitalocean.com." } } + ) + + lookup.match_contacts! + + assert_equal host, lookup.matched_hosting_contact + end + + test "a bare pattern covers the subdomains under it" do + host = create_test_abuse_contact(hostname_patterns: [ "digitalocean.com" ]) + lookup = lookup_for(dns_records: { "CNAME" => [ "a.b.digitalocean.com" ] }) + + lookup.match_contacts! + + assert_equal host, lookup.matched_hosting_contact + end + + test "a pattern does not match a lookalike domain" do + create_test_abuse_contact(hostname_patterns: [ "digitalocean.com" ]) + lookup = lookup_for(dns_records: { "CNAME" => [ "notdigitalocean.com" ] }) + + lookup.match_contacts! + + assert_nil lookup.matched_hosting_contact + end + + test "a serving hostname is tried before the address" do + by_hostname = create_test_abuse_contact(hostname_patterns: [ "vercel-dns.com" ], priority: 30) + create_test_abuse_contact(ip_ranges: [ "76.76.0.0/16" ], priority: 30) + lookup = lookup_for( + a_records: [ "76.76.21.21" ], + dns_records: { "CNAME" => [ "cname.vercel-dns.com" ] } + ) + + lookup.match_contacts! + + assert_equal by_hostname, lookup.matched_hosting_contact + end + + # github.com delegates to Route 53 but is served by GitHub. Matching on + # nameservers first would send the report to Amazon. + test "the address beats a nameserver that names a different company" do + create_test_abuse_contact(hostname_patterns: [ "*awsdns-*" ], priority: 30) + network = create_test_abuse_contact(ip_ranges: [ "140.82.112.0/20" ], priority: 30) + lookup = lookup_for( + a_records: [ "140.82.114.3" ], + dns_records: { "NS" => [ "ns-1707.awsdns-21.co.uk" ] } + ) + + lookup.match_contacts! + + assert_equal network, lookup.matched_hosting_contact + end + + test "a nameserver still matches when nothing stronger does" do + dns_operator = create_test_abuse_contact(hostname_patterns: [ "cloudflare.com" ]) + lookup = lookup_for(dns_records: { "NS" => [ "doug.ns.cloudflare.com" ] }) + + lookup.match_contacts! + + assert_equal dns_operator, lookup.matched_hosting_contact + end + + test "a mail host never decides who hosts the page" do + create_test_abuse_contact(hostname_patterns: [ "aspmx.l.google.com" ]) + lookup = lookup_for( + dns_records: { "MX" => [ { "preference" => 1, "exchange" => "aspmx.l.google.com" } ] } + ) + + lookup.match_contacts! + + assert_nil lookup.matched_hosting_contact + end + + test "the mail hosts are surfaced for routing the mail side by hand" do + lookup = lookup_for( + dns_records: { "MX" => [ { "preference" => 10, "exchange" => "mx.example.com." } ] } + ) + + assert_equal [ "mx.example.com" ], lookup.mail_hosts + assert_equal [ "mx.example.com" ], lookup.to_summary[:mail_hosts] + end + + test "the full record set travels with the case summary" do + lookup = lookup_for(dns_records: { "TXT" => [ "v=spf1 include:_spf.google.com ~all" ] }) + + assert_equal [ "v=spf1 include:_spf.google.com ~all" ], lookup.to_summary[:dns_records]["TXT"] + end + test "an IPv6-only host is matched on its AAAA records" do host = create_test_abuse_contact(ip_ranges: [ "2604:a880::/32" ]) lookup = lookup_for(a_records: [], aaaa_records: [ "2604:a880::1" ]) diff --git a/test/services/report/dns_sweep_service_test.rb b/test/services/report/dns_sweep_service_test.rb new file mode 100644 index 0000000..3bf53ed --- /dev/null +++ b/test/services/report/dns_sweep_service_test.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +require "test_helper" + +# A phishing site on platform hosting resolves to a shared anycast address that +# belongs to a CDN, not to the platform serving the page. The CNAME and the +# reverse lookup are what name the party who can take it down, so the sweep has +# to surface them for the matcher and for a human reading the case. +class Report::DnsSweepServiceTest < ActiveSupport::TestCase + def records + { + "A" => [ "76.76.21.21" ], + "AAAA" => [ "2606:4700::1" ], + "CNAME" => [ "cname.vercel-dns.com." ], + "NS" => [ "ns1.vercel-dns.com." ], + "MX" => [ { "preference" => 10, "exchange" => "mx.example.com." } ], + "PTR" => { "76.76.21.21" => "cname.vercel-dns.com." } + } + end + + test "the serving hostnames are the ones that name who runs the page" do + serving = Report::DnsSweepService.serving_hostnames(records) + + assert_equal [ "cname.vercel-dns.com" ], serving + assert_not_includes serving, "ns1.vercel-dns.com", "a nameserver is not a serving host" + assert_not_includes serving, "mx.example.com", "a mail host is not a serving host" + end + + test "nameservers are kept apart as the weaker signal" do + assert_equal [ "ns1.vercel-dns.com" ], Report::DnsSweepService.nameserver_hostnames(records) + end + + test "the combined list puts serving hostnames first" do + hostnames = Report::DnsSweepService.hostnames(records) + + assert_operator hostnames.index("cname.vercel-dns.com"), :<, + hostnames.index("ns1.vercel-dns.com") + end + + test "trailing dots are stripped so patterns match" do + Report::DnsSweepService.hostnames(records).each do |hostname| + assert_not hostname.end_with?("."), "#{hostname} kept its trailing dot" + end + end + + test "hostnames are deduplicated" do + duplicated = { "CNAME" => [ "a.example.com." ], "NS" => [ "A.EXAMPLE.COM" ] } + + assert_equal [ "a.example.com" ], Report::DnsSweepService.hostnames(duplicated) + end + + test "an empty sweep yields no hostnames" do + assert_empty Report::DnsSweepService.hostnames({}) + end + + test "a blank domain is not queried" do + assert_empty Report::DnsSweepService.new.sweep("") + assert_empty Report::DnsSweepService.new.sweep(nil) + end + + test "the record types swept are the ones that name a party to report to" do + types = Report::DnsSweepService::RECORD_TYPES.keys + + assert_equal %w[A AAAA CNAME NS MX TXT SOA CAA], types + end + + test "a domain that does not resolve yields no records rather than raising" do + result = nil + + assert_nothing_raised do + result = Report::DnsSweepService.new.sweep("does-not-exist-#{SecureRandom.hex(8)}.invalid") + end + assert_empty result + end +end