diff --git a/app/controllers/admin/report/abuse_contacts_controller.rb b/app/controllers/admin/report/abuse_contacts_controller.rb index 05ef661..a3d5fdf 100644 --- a/app/controllers/admin/report/abuse_contacts_controller.rb +++ b/app/controllers/admin/report/abuse_contacts_controller.rb @@ -105,6 +105,7 @@ def abuse_contact_params :organization, :trusted_reporter, :accepts_bulk, + :accepts_xarf, :priority, :active, :notes, diff --git a/app/controllers/xarf_controller.rb b/app/controllers/xarf_controller.rb index af120e5..c8373ae 100644 --- a/app/controllers/xarf_controller.rb +++ b/app/controllers/xarf_controller.rb @@ -2,7 +2,7 @@ # The public XARF utility. # -# Anyone can paste a link and get a XARF v4 report back. Sending that report to +# Anyone can paste a link and get an X-ARF report back. Sending that report to # the registrar and hosting provider needs an account, because that path puts # mail in someone else's inbox under our name. class XarfController < ApplicationController diff --git a/app/jobs/digital_ocean_range_sync_job.rb b/app/jobs/digital_ocean_range_sync_job.rb new file mode 100644 index 0000000..0db6ae5 --- /dev/null +++ b/app/jobs/digital_ocean_range_sync_job.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +# Refreshes the IP ranges on the DigitalOcean abuse contact. +# DigitalOcean reallocates blocks, so a stale list makes the report pipeline +# miss sites it should report. +class DigitalOceanRangeSyncJob < ApplicationJob + queue_as QUEUE_MAINTENANCE + + def perform + Rails.logger.info("[DigitalOceanRangeSyncJob] Starting DigitalOcean IP range sync...") + + result = Report::DigitalOceanRangeService.new.sync + + if result[:success] + Rails.logger.info("[DigitalOceanRangeSyncJob] Sync complete: #{result[:ranges]} ranges stored") + else + Rails.logger.error("[DigitalOceanRangeSyncJob] Sync failed: #{result[:error]}") + end + end +end diff --git a/app/mailers/report/abuse_report_mailer.rb b/app/mailers/report/abuse_report_mailer.rb index 11ccae8..fc708e3 100644 --- a/app/mailers/report/abuse_report_mailer.rb +++ b/app/mailers/report/abuse_report_mailer.rb @@ -10,17 +10,73 @@ def abuse_report @contact = params[:contact] @payload = (@submission.payload.presence || @submission.build_payload).with_indifferent_access - # No env_subject - reports are always sent to real external contacts - # regardless of environment (only legit phishing domains are reported) - mail( + @contact.accepts_xarf? ? xarf_report_mail : standard_report_mail + end + + private + + def mail_headers + { to: @contact.email, cc: @case.email_address, # case_xxx@cases.phish.directory for reply threading reply_to: [ "support@phish.directory", @case.email_address ], + # No env_subject - reports are always sent to real external contacts + # regardless of environment (only legit phishing domains are reported) subject: "[Automated] [Phishing Report] #{@case.domain_name} - Case #{@case.case_number}" - ) + } + end + + def standard_report_mail + mail(**mail_headers) + end + + # Build the report the way https://github.com/abusix/xarf specifies for + # SMTP: an RFC 5965 feedback report whose third part carries the machine + # readable document. + # + # multipart/report; report-type=feedback-report + # text/plain the human readable report + # message/feedback-report Feedback-Type: xarf, so an ARF parser stops + # here and an X-ARF parser reads the next part + # application/json xarf.json, the report itself + # + # The HTML part is left out on purpose. RFC 6522 puts the human readable + # part first, and a lone text/plain keeps the structure identical to the + # published example, which is what these parsers are written against. + def xarf_report_mail + message = mail(**mail_headers) do |format| + format.text { render "abuse_report" } + end + + message.add_part(feedback_report_part) + message.attachments["xarf.json"] = { + mime_type: "application/json", + content: JSON.pretty_generate( + Xarf::ReportGenerator.new.generate_for_submission(@submission) + ) + } + + # add_part set a boundary while wrapping the body, so reuse it rather + # than letting the new Content-Type drop it. + message.content_type = + %(multipart/report; report-type=feedback-report; boundary="#{message.body.boundary}") + + message + end + + def feedback_report_part + Mail::Part.new do + content_type "message/feedback-report" + content_disposition "inline" + body [ + "Feedback-Type: xarf", + "User-Agent: phish.directory/#{ENV.fetch('RELEASE_VERSION', '1.0.0')}", + "Version: 1" + ].join("\r\n") + end end end end diff --git a/app/models/report/abuse_contact.rb b/app/models/report/abuse_contact.rb index 7ba88d7..dd6eaef 100644 --- a/app/models/report/abuse_contact.rb +++ b/app/models/report/abuse_contact.rb @@ -61,6 +61,7 @@ class Report::AbuseContact < ApplicationRecord scope :security_vendors, -> { where(contact_type: "security_vendor") } scope :trusted, -> { where(trusted_reporter: true) } scope :by_priority, -> { order(priority: :asc) } + scope :with_ip_ranges, -> { where("jsonb_array_length(ip_ranges) > 0") } # Class methods class << self @@ -86,13 +87,49 @@ def find_for_nameservers(nameservers) end end + # Find contact whose published IP ranges cover any of the given addresses + # + # Nameserver patterns only identify a host when the site also uses that + # host's DNS, which phishing sites usually do not. Matching the addresses a + # domain actually resolves to is what finds the provider serving the page. + # + # @param addresses [Array] IP addresses the domain resolves to + # @return [Report::AbuseContact, nil] + def find_for_ip(addresses) + ips = Array(addresses).filter_map { |address| parse_ip(address) } + return nil if ips.empty? + + active.with_ip_ranges.by_priority.find { |contact| contact.covers_ip?(ips) } + end + # Get all contacts that should always receive reports def always_report_to active.trusted.by_priority end + + private + + def parse_ip(address) + IPAddr.new(address.to_s) + rescue IPAddr::InvalidAddressError + nil + end end # Instance methods + + # Check whether any of the given addresses falls inside this contact's ranges + # + # @param addresses [Array] parsed addresses + # @return [Boolean] + def covers_ip?(addresses) + return false if cidr_ranges.empty? + + Array(addresses).any? do |address| + cidr_ranges.any? { |range| range.include?(address) } + end + end + def operational? active? && kept? end @@ -113,6 +150,15 @@ def display_name private + # DigitalOcean publishes around 1200 ranges, so parse them once per record. + def cidr_ranges + @cidr_ranges ||= Array(ip_ranges).filter_map do |range| + IPAddr.new(range.to_s) + rescue IPAddr::InvalidAddressError + nil + end + end + def update_response_stats! acknowledged = submissions.where.not(acknowledged_at: nil) return if acknowledged.empty? diff --git a/app/models/report/domain_lookup.rb b/app/models/report/domain_lookup.rb index 6c63e2c..d680982 100644 --- a/app/models/report/domain_lookup.rb +++ b/app/models/report/domain_lookup.rb @@ -60,12 +60,26 @@ def mark_looked_up!(source: nil) end # Update matched contacts based on patterns + # + # Hosting falls back to the addresses the domain resolves to. A phishing site + # on a DigitalOcean droplet normally keeps its registrar's nameservers, so + # nameserver patterns alone never identify the provider serving the page. def match_contacts! self.matched_registrar_contact = Report::AbuseContact.find_for_registrar(registrar_name) - self.matched_hosting_contact = Report::AbuseContact.find_for_nameservers(nameservers) + self.matched_hosting_contact = + Report::AbuseContact.find_for_nameservers(nameservers) || + Report::AbuseContact.find_for_ip(resolved_addresses) + self.hosting_provider = matched_hosting_contact.name if matched_hosting_contact save! end + # Every address the domain resolves to, both families + # + # @return [Array] + def resolved_addresses + Array(a_records) + Array(aaaa_records) + end + # Get all matched contacts def matched_contacts [ matched_registrar_contact, matched_hosting_contact ].compact.uniq @@ -82,6 +96,8 @@ def to_summary registrar_name: registrar_name, registrar_abuse_email: registrar_abuse_email, nameservers: nameservers, + a_records: a_records, + aaaa_records: aaaa_records, hosting_provider: hosting_provider, domain_created_at: domain_created_at&.iso8601, domain_expires_at: domain_expires_at&.iso8601, diff --git a/app/services/report/digital_ocean_range_service.rb b/app/services/report/digital_ocean_range_service.rb new file mode 100644 index 0000000..9988c13 --- /dev/null +++ b/app/services/report/digital_ocean_range_service.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module Report + # Keeps the DigitalOcean abuse contact's IP ranges current. + # + # DigitalOcean publishes its allocations as a CSV whose first column is the + # CIDR block. Report::AbuseContact.find_for_ip matches a phishing domain's A + # records against those blocks, which is how the pipeline decides that + # DigitalOcean is the host to notify. + # + # Usage: + # Report::DigitalOceanRangeService.new.sync + # + class DigitalOceanRangeService < BaseService + RANGES_URL = "https://digitalocean.com/geo/google.csv" + CONTACT_NAME = "DigitalOcean" + + def sync + cidrs = parse_cidrs(fetch_ranges) + + if cidrs.empty? + log_error("Range list was empty", ServiceError.new("no usable CIDR blocks")) + return { success: false, error: "No IP ranges returned" } + end + + contact = Report::AbuseContact.find_by(name: CONTACT_NAME) + + unless contact + return { success: false, error: "#{CONTACT_NAME} abuse contact is not seeded" } + end + + contact.update!(ip_ranges: cidrs) + log_info("Stored #{cidrs.size} DigitalOcean IP ranges") + + { success: true, ranges: cidrs.size } + rescue ServiceError => e + { success: false, error: e.message } + end + + private + + def fetch_ranges + conn = connection(base_url: RANGES_URL, headers: { "Accept" => "text/csv" }) + get(conn, "").to_s + end + + # Rows look like "5.101.96.0/21,NL,NL-NH,Amsterdam,1098 XH". Anything that + # is not a CIDR block is dropped rather than stored and matched against + # later. + def parse_cidrs(body) + body.each_line.filter_map do |line| + cidr = line.split(",").first.to_s.strip + next if cidr.blank? + + IPAddr.new(cidr) + cidr + rescue IPAddr::InvalidAddressError + nil + end + end + end +end diff --git a/app/services/report/domain_lookup_service.rb b/app/services/report/domain_lookup_service.rb index 588a8c7..f67a1fb 100644 --- a/app/services/report/domain_lookup_service.rb +++ b/app/services/report/domain_lookup_service.rb @@ -5,6 +5,7 @@ module Report # Uses RDAP (preferred) with WHOIS fallback class DomainLookupService < BaseService CACHE_TTL = 24.hours + DNS_TIMEOUT = 5 # RDAP bootstrap servers by TLD # See: https://data.iana.org/rdap/dns.json @@ -37,6 +38,10 @@ def lookup(domain) result = lookup_rdap(domain) || lookup_whois(domain) if result + # The hosting provider is found from the addresses the domain serves + # from, not from the registration record, so resolve them here. + result[:a_records] = resolve_addresses(domain, Resolv::DNS::Resource::IN::A) + result[:aaaa_records] = resolve_addresses(domain, Resolv::DNS::Resource::IN::AAAA) save_lookup(domain, result) else log_info("No lookup data found for #{domain}") @@ -50,6 +55,32 @@ def normalize_domain(domain) domain.to_s.downcase.strip.sub(/^www\./, "") end + # Resolve the addresses the domain currently serves from. + # + # Report::AbuseContact.find_for_ip matches these against the ranges a + # hosting provider publishes, which is how a report reaches the provider + # that actually serves the phishing page. Both families are resolved: + # DigitalOcean and its peers publish IPv6 allocations too, and an + # IPv6-only host would otherwise never be matched. + # + # A domain that does not resolve is still worth reporting on the + # registration record alone, so every failure here degrades to no addresses + # rather than losing the lookup. + # + # @param domain [String] normalized domain + # @param resource [Class] Resolv::DNS::Resource::IN::A or ::AAAA + # @return [Array] addresses, empty when the domain does not resolve + def resolve_addresses(domain, resource) + Timeout.timeout(DNS_TIMEOUT) do + Resolv::DNS.open do |dns| + dns.getresources(domain, resource).map { |record| record.address.to_s } + end + end + rescue StandardError => e + log_debug("Could not resolve #{resource.name.demodulize} records for #{domain}: #{e.message}") + [] + end + def lookup_rdap(domain) tld = domain.split(".").last&.downcase rdap_url = RDAP_SERVERS[tld] @@ -218,6 +249,8 @@ def save_lookup(domain, result) domain_created_at: result[:domain_created_at], domain_expires_at: result[:domain_expires_at], nameservers: result[:nameservers], + a_records: result[:a_records] || [], + aaaa_records: result[:aaaa_records] || [], lookup_source: result[:lookup_source], looked_up_at: Time.current, expires_at: CACHE_TTL.from_now diff --git a/app/services/xarf/category_mapper.rb b/app/services/xarf/category_mapper.rb index b3905e1..569984f 100644 --- a/app/services/xarf/category_mapper.rb +++ b/app/services/xarf/category_mapper.rb @@ -1,16 +1,16 @@ # frozen_string_literal: true module Xarf - # Maps between phish.directory classifications and XARF v4 categories/types + # Maps between phish.directory classifications and the X-ARF schema 3 + # taxonomy published at https://github.com/abusix/xarf. # - # XARF v4 Categories (7): - # - connection: Network-level attacks (login_attack, port_scan, ddos, etc.) - # - content: Malicious/harmful content (phishing, malware, fraud, etc.) - # - copyright: IP infringement (copyright, p2p, cyberlocker, etc.) - # - infrastructure: Compromised systems (botnet, compromised_server) - # - messaging: Spam and bulk messaging (spam, bulk_messaging) - # - reputation: Threat intel and blocklists (blocklist, threat_intelligence) - # - vulnerability: Security issues (cve, open, misconfiguration) + # Schema 3 splits a report into a ReportClass and a ReportType. There are + # three classes and one type per schema file: + # + # Content Phishing, Malware, Copyright, Trademark, ChildAbuse, Botnet + # Activity Spam, DOS, PortScan, LoginAttack, Exploit, WebCrawler, + # Harassment, PotentiallyCompromisedAccount, Malware (RPZ) + # Vulnerability OpenService # # phish.directory Classifications: # - phishing: Confirmed phishing sites @@ -20,182 +20,121 @@ module Xarf # - protected: Protected domains (whitelisted) # class CategoryMapper - # XARF v4 categories - XARF_CATEGORIES = %w[ - connection - content - copyright - infrastructure - messaging - reputation - vulnerability - ].freeze - - # XARF v4 types organized by category - XARF_TYPES = { - connection: %w[ - login_attack - port_scan - ddos - infected_host - reconnaissance - scraping - sql_injection - vuln_scanning - ].freeze, - content: %w[ - phishing - malware - csam - csem - exposed_data - brand_infringement - fraud - remote_compromise - suspicious_registration - ].freeze, - copyright: %w[ - copyright - p2p - cyberlocker - ugc_platform - link_site - usenet + REPORT_CLASSES = %w[Content Activity Vulnerability].freeze + + # Types by class, taken from the schema 3 files. Malware appears twice + # because the RPZ schema reports it as Activity with an RPZ-Rewrite + # subtype, while the malware schema reports it as Content. + REPORT_TYPES = { + "Content" => %w[Phishing Malware Copyright Trademark ChildAbuse Botnet].freeze, + "Activity" => %w[ + Spam + DOS + PortScan + LoginAttack + Exploit + PotentiallyCompromisedAccount + WebCrawler + Harassment + Malware ].freeze, - infrastructure: %w[ - botnet - compromised_server - ].freeze, - messaging: %w[ - spam - bulk_messaging - ].freeze, - reputation: %w[ - blocklist - threat_intelligence - ].freeze, - vulnerability: %w[ - cve - open - misconfiguration - ].freeze + "Vulnerability" => %w[OpenService].freeze }.freeze - # Mapping from phish.directory classification to XARF category/type + # Mapping from phish.directory classification to X-ARF class/type. + # + # Schema 3 has no type for an unconfirmed site, so "suspicious" is reported + # as Phishing too. ReporterSeverity and the notes carry the confidence, so + # the receiving desk still sees that the finding is not confirmed. CLASSIFICATION_TO_XARF = { - "phishing" => { category: "content", type: "phishing" }, - "suspicious" => { category: "content", type: "suspicious_registration" }, - "clean" => nil, # Clean domains don't need XARF reports - "unknown" => nil, # Unknown domains don't have enough info for XARF + "phishing" => { report_class: "Content", report_type: "Phishing" }, + "suspicious" => { report_class: "Content", report_type: "Phishing" }, + "clean" => nil, # Clean domains don't need X-ARF reports + "unknown" => nil, # Unknown domains don't have enough info for X-ARF "protected" => nil # Protected domains shouldn't be reported }.freeze - # Mapping from XARF type to phish.directory classification + # Mapping from X-ARF type to phish.directory classification. + # nil means the type is real but outside what this directory classifies. XARF_TYPE_TO_CLASSIFICATION = { - # Content types - "phishing" => "phishing", - "malware" => "phishing", - "fraud" => "phishing", - "brand_infringement" => "suspicious", - "suspicious_registration" => "suspicious", - "exposed_data" => "suspicious", - "remote_compromise" => "phishing", - "csam" => "phishing", - "csem" => "phishing", - - # Connection types (usually infrastructure-level, map to suspicious) - "login_attack" => "suspicious", - "port_scan" => "suspicious", - "ddos" => "suspicious", - "infected_host" => "phishing", - "reconnaissance" => "suspicious", - "scraping" => "suspicious", - "sql_injection" => "phishing", - "vuln_scanning" => "suspicious", - - # Infrastructure types - "botnet" => "phishing", - "compromised_server" => "phishing", - - # Messaging types - "spam" => "suspicious", - "bulk_messaging" => "suspicious", - - # Reputation types (informational) - "blocklist" => "suspicious", - "threat_intelligence" => "suspicious", - - # Copyright types (not typically phishing) - "copyright" => nil, - "p2p" => nil, - "cyberlocker" => nil, - "ugc_platform" => nil, - "link_site" => nil, - "usenet" => nil, - - # Vulnerability types - "cve" => "suspicious", - "open" => "suspicious", - "misconfiguration" => "suspicious" + "Phishing" => "phishing", + "Malware" => "phishing", + "Botnet" => "phishing", + "Exploit" => "phishing", + "Trademark" => "suspicious", + "Spam" => "suspicious", + "LoginAttack" => "suspicious", + "PortScan" => "suspicious", + "DOS" => "suspicious", + "WebCrawler" => "suspicious", + "PotentiallyCompromisedAccount" => "suspicious", + "OpenService" => "suspicious", + "Copyright" => nil, + "ChildAbuse" => nil, + "Harassment" => nil }.freeze - # Confidence score adjustments based on XARF type - # Higher values = more confidence the mapping is accurate + # Confidence score for an incoming type to classification mapping. + # Higher values = more confidence the mapping is accurate. XARF_TYPE_CONFIDENCE = { - "phishing" => 1.0, - "malware" => 0.95, - "fraud" => 0.9, - "infected_host" => 0.85, - "botnet" => 0.85, - "compromised_server" => 0.8, - "brand_infringement" => 0.7, - "suspicious_registration" => 0.6, - "spam" => 0.5, - "blocklist" => 0.6, - "threat_intelligence" => 0.7 + "Phishing" => 1.0, + "Malware" => 0.95, + "Botnet" => 0.85, + "Exploit" => 0.8, + "Trademark" => 0.7, + "Spam" => 0.5, + "LoginAttack" => 0.5, + "PortScan" => 0.5, + "DOS" => 0.5, + "WebCrawler" => 0.4, + "PotentiallyCompromisedAccount" => 0.6, + "OpenService" => 0.5 }.freeze DEFAULT_CONFIDENCE = 0.5 + # Severity thresholds. ReporterSeverity is a closed low/medium/high enum. + HIGH_CONFIDENCE = 0.9 + MEDIUM_CONFIDENCE = 0.7 + class << self - # Convert phish.directory classification to XARF category/type + # Convert phish.directory classification to X-ARF class/type # # @param classification [String] phish.directory classification - # @param subtype [String, nil] optional subtype for more specific mapping - # @return [Hash, nil] { category:, type: } or nil if not mappable - def to_xarf(classification, subtype: nil) + # @param report_type [String, nil] optional override for a more specific type + # @return [Hash, nil] { report_class:, report_type: } or nil if not mappable + def to_xarf(classification, report_type: nil) mapping = CLASSIFICATION_TO_XARF[classification.to_s] return nil if mapping.nil? - # Allow subtype override for more specific mappings - if subtype && valid_xarf_type?(subtype) - { category: category_for_type(subtype), type: subtype } + if report_type && valid_report_type?(report_type) + { report_class: classes_for_type(report_type).first, report_type: report_type.to_s } else mapping end end - # Convert XARF category/type to phish.directory classification + # Convert X-ARF class/type to phish.directory classification # - # @param category [String] XARF category - # @param type [String] XARF type + # @param report_class [String] X-ARF ReportClass + # @param report_type [String] X-ARF ReportType # @return [String, nil] phish.directory classification or nil - def from_xarf(category, type) - return nil unless valid_xarf_category?(category) - return nil unless valid_xarf_type?(type) + def from_xarf(report_class, report_type) + return nil unless valid_report_class?(report_class) + return nil unless valid_report_type?(report_type) - XARF_TYPE_TO_CLASSIFICATION[type.to_s] + XARF_TYPE_TO_CLASSIFICATION[report_type.to_s] end - # Get confidence score for XARF type to classification mapping + # Get confidence score for an X-ARF type to classification mapping # - # @param type [String] XARF type + # @param report_type [String] X-ARF ReportType # @return [Float] confidence score (0.0 - 1.0) - def confidence_for_type(type) - XARF_TYPE_CONFIDENCE.fetch(type.to_s, DEFAULT_CONFIDENCE) + def confidence_for_type(report_type) + XARF_TYPE_CONFIDENCE.fetch(report_type.to_s, DEFAULT_CONFIDENCE) end - # Check if a classification is reportable via XARF + # Check if a classification is reportable via X-ARF # # @param classification [String] phish.directory classification # @return [Boolean] @@ -203,128 +142,110 @@ def reportable?(classification) CLASSIFICATION_TO_XARF[classification.to_s].present? end - # Get the XARF category for a given type + # Get every ReportClass a type may appear under # - # @param type [String] XARF type - # @return [String, nil] XARF category or nil - def category_for_type(type) - XARF_TYPES.each do |category, types| - return category.to_s if types.include?(type.to_s) - end - nil + # @param report_type [String] X-ARF ReportType + # @return [Array] classes, most common first + def classes_for_type(report_type) + REPORT_TYPES.select { |_klass, types| types.include?(report_type.to_s) }.keys end - # Validate XARF category + # Validate a ReportClass # - # @param category [String] XARF category + # @param report_class [String] # @return [Boolean] - def valid_xarf_category?(category) - XARF_CATEGORIES.include?(category.to_s) + def valid_report_class?(report_class) + REPORT_CLASSES.include?(report_class.to_s) end - # Validate XARF type + # Validate a ReportType # - # @param type [String] XARF type + # @param report_type [String] # @return [Boolean] - def valid_xarf_type?(type) - XARF_TYPES.values.flatten.include?(type.to_s) + def valid_report_type?(report_type) + REPORT_TYPES.values.flatten.include?(report_type.to_s) end - # Get all XARF types for a category + # Check that a type is allowed under a class # - # @param category [String, Symbol] XARF category + # @param report_class [String] + # @param report_type [String] + # @return [Boolean] + def type_in_class?(report_class, report_type) + REPORT_TYPES.fetch(report_class.to_s, []).include?(report_type.to_s) + end + + # Get all types for a class + # + # @param report_class [String] # @return [Array] list of types - def types_for_category(category) - XARF_TYPES[category.to_sym] || [] + def types_for_class(report_class) + REPORT_TYPES.fetch(report_class.to_s, []) end - # Map verdict object to XARF category/type with full context + # Map a verdict onto an X-ARF class/type with full context # # @param verdict [Verdict] verdict record - # @return [Hash] { category:, type:, confidence:, reportable: } + # @return [Hash] { report_class:, report_type:, confidence:, reportable: } def map_verdict(verdict) return { reportable: false } if verdict.nil? - xarf_mapping = to_xarf(verdict.classification) + mapping = to_xarf(verdict.classification) + return { reportable: false } unless mapping - if xarf_mapping - { - category: xarf_mapping[:category], - type: xarf_mapping[:type], - confidence: verdict.confidence_score || DEFAULT_CONFIDENCE, - reportable: true - } - else - { reportable: false } + mapping.merge( + confidence: verdict.confidence_score || DEFAULT_CONFIDENCE, + reportable: true + ) + end + + # Severity for a confidence score, as the ReporterSeverity enum + # + # @param confidence [Float, nil] + # @return [String] "low", "medium" or "high" + def severity_for_confidence(confidence) + case confidence.to_f + when HIGH_CONFIDENCE.. then "high" + when MEDIUM_CONFIDENCE...HIGH_CONFIDENCE then "medium" + else "low" end end - # Get descriptive info about a XARF type + # Describe an X-ARF type # - # @param type [String] XARF type - # @return [Hash] { category:, description:, severity: } - def type_info(type) - category = category_for_type(type) - return nil unless category + # @param report_type [String] X-ARF ReportType + # @return [Hash, nil] { report_class:, report_type:, description: } + def type_info(report_type) + classes = classes_for_type(report_type) + return nil if classes.empty? { - category: category, - type: type, - description: type_description(type), - severity: type_severity(type) + report_class: classes.first, + report_type: report_type.to_s, + description: type_description(report_type) } end private - def type_description(type) + def type_description(report_type) { - # Content types - "phishing" => "Fraudulent attempt to obtain sensitive information", - "malware" => "Malicious software distribution", - "fraud" => "Deceptive practices for financial gain", - "brand_infringement" => "Unauthorized use of brand identity", - "suspicious_registration" => "Domain registered with suspicious patterns", - "exposed_data" => "Sensitive data exposure", - "remote_compromise" => "Remote system compromise", - "csam" => "Child sexual abuse material", - "csem" => "Child sexual exploitation material", - - # Connection types - "login_attack" => "Brute force or credential stuffing attack", - "port_scan" => "Network port scanning activity", - "ddos" => "Distributed denial of service attack", - "infected_host" => "Compromised host exhibiting malicious behavior", - "reconnaissance" => "Information gathering for potential attack", - "scraping" => "Unauthorized data scraping", - "sql_injection" => "SQL injection attack attempt", - "vuln_scanning" => "Vulnerability scanning activity", - - # Infrastructure types - "botnet" => "Part of a botnet command and control", - "compromised_server" => "Server showing signs of compromise", - - # Messaging types - "spam" => "Unsolicited bulk messaging", - "bulk_messaging" => "High-volume messaging campaign", - - # Reputation types - "blocklist" => "Listed on security blocklist", - "threat_intelligence" => "Identified in threat intelligence feed" - }.fetch(type.to_s, "Unknown abuse type") - end - - def type_severity(type) - case type.to_s - when "phishing", "malware", "csam", "csem", "remote_compromise", "botnet" - "critical" - when "fraud", "infected_host", "compromised_server", "sql_injection" - "high" - when "brand_infringement", "login_attack", "ddos", "spam" - "medium" - else - "low" - end + "Phishing" => "Fraudulent attempt to obtain sensitive information", + "Malware" => "Malicious software distribution", + "Copyright" => "Copyright infringing content", + "Trademark" => "Unauthorized use of brand identity", + "ChildAbuse" => "Child sexual abuse material", + "Botnet" => "Botnet command and control", + "Spam" => "Unsolicited bulk messaging", + "DOS" => "Denial of service attack", + "PortScan" => "Network port scanning activity", + "LoginAttack" => "Brute force or credential stuffing attack", + "Exploit" => "Attempt to exploit a vulnerability", + "PotentiallyCompromisedAccount" => "Account showing signs of compromise", + "WebCrawler" => "Unwanted automated crawling", + "Harassment" => "Harassment of an individual", + "OpenService" => "Service exposed that should not be reachable" + }.fetch(report_type.to_s, "Unknown abuse type") end end end diff --git a/app/services/xarf/public_report_service.rb b/app/services/xarf/public_report_service.rb index 04ab336..6fafd91 100644 --- a/app/services/xarf/public_report_service.rb +++ b/app/services/xarf/public_report_service.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Xarf - # Turns something a person pasted into a browser into a XARF v4 report. + # Turns something a person pasted into a browser into an X-ARF report. # # The point raised against XARF is that it asks an ordinary reporter to # hand-write JSON, which gatekeeps who is able to report abuse at all. This diff --git a/app/services/xarf/report_generator.rb b/app/services/xarf/report_generator.rb index dbb29f1..f9895c1 100644 --- a/app/services/xarf/report_generator.rb +++ b/app/services/xarf/report_generator.rb @@ -1,7 +1,12 @@ # frozen_string_literal: true module Xarf - # Generates XARF v4 compliant reports from phish.directory data + # Generates X-ARF reports from phish.directory data. + # + # Output follows schema 3 of https://github.com/abusix/xarf, which is the + # newest published schema. It is PascalCase and nests everything under + # ReporterInfo and Report. Abuse desks with automated tooling validate + # against it, so the same document serves the admin UI and the wire. # # Usage: # generator = Xarf::ReportGenerator.new @@ -15,16 +20,20 @@ module Xarf # # Generate from a verdict # report = generator.generate_for_verdict(verdict, source_type: :domain, source: domain) # + # # Generate the document that goes out as an email's xarf.json attachment + # report = generator.generate_for_submission(submission) + # # # Get JSON # report.to_json # class ReportGenerator - XARF_VERSION = "4.0.0" + SCHEMA_VERSION = "3" DEFAULT_REPORTER = { org: "phish.directory", - contact: "abuse@phish.directory", - domain: "phish.directory" + domain: "phish.directory", + email: "reports@phish.directory", + contact_name: "phish.directory Automated Reporting" }.freeze attr_reader :reporter @@ -33,11 +42,11 @@ def initialize(reporter: nil) @reporter = reporter || DEFAULT_REPORTER end - # Generate a XARF report for a Phish::Domain record + # Generate an X-ARF report for a Phish::Domain record # # @param domain [Phish::Domain] domain record # @param options [Hash] additional options - # @return [Hash] XARF v4 compliant report + # @return [Hash] X-ARF schema 3 report def generate_for_domain(domain, **options) raise ArgumentError, "Domain required" if domain.nil? @@ -49,22 +58,18 @@ def generate_for_domain(domain, **options) end build_report( - source_identifier: domain.domain, - source_type: :domain, - category: mapping[:category], - type: mapping[:type], - confidence: mapping[:confidence], + source_url: "https://#{domain.domain}", + mapping: mapping, verdict: verdict, - record: domain, **options ) end - # Generate a XARF report for a Phish::Url record + # Generate an X-ARF report for a Phish::Url record # # @param url [Phish::Url] URL record # @param options [Hash] additional options - # @return [Hash] XARF v4 compliant report + # @return [Hash] X-ARF schema 3 report def generate_for_url(url, **options) raise ArgumentError, "URL required" if url.nil? @@ -76,25 +81,20 @@ def generate_for_url(url, **options) end build_report( - source_identifier: url.domain || url.url, - source_type: :url, - category: mapping[:category], - type: mapping[:type], - confidence: mapping[:confidence], + source_url: url.url, + mapping: mapping, verdict: verdict, - record: url, - url: url.url, **options ) end - # Generate a XARF report for a Verdict with a specified source + # Generate an X-ARF report for a Verdict with a specified source # # @param verdict [Verdict] verdict record # @param source_type [Symbol] :domain or :url # @param source [String] the source identifier # @param options [Hash] additional options - # @return [Hash] XARF v4 compliant report + # @return [Hash] X-ARF schema 3 report def generate_for_verdict(verdict, source_type:, source:, **options) raise ArgumentError, "Verdict required" if verdict.nil? raise ArgumentError, "Source required" if source.blank? @@ -105,22 +105,44 @@ def generate_for_verdict(verdict, source_type:, source:, **options) return { error: "Verdict classification not reportable via XARF" } end + source_url = source_type.to_sym == :url ? source : "https://#{source}" + + build_report(source_url: source_url, mapping: mapping, verdict: verdict, **options) + end + + # Generate the document that goes out as an abuse report's xarf.json. + # + # A submission carries case context the other entry points do not have: the + # case number, the address replies thread back to, and the addresses the + # domain resolved to when the case was opened. + # + # @param submission [Report::Submission] submission record + # @param options [Hash] additional options + # @return [Hash] X-ARF schema 3 report + def generate_for_submission(submission, **options) + raise ArgumentError, "Submission required" if submission.nil? + + report_case = submission.case + payload = (submission.payload.presence || submission.build_payload).with_indifferent_access + build_report( - source_identifier: source, - source_type: source_type, - category: mapping[:category], - type: mapping[:type], - confidence: mapping[:confidence], - verdict: verdict, + source_url: payload[:url].presence || "https://#{payload[:domain] || report_case.domain_name}", + mapping: submission_mapping(payload), + source_ip: case_source_ip(report_case), + detected_at: payload[:detected_at], + sources: payload[:sources], + case_reference: payload[:case_reference] || report_case.case_number, + # Replies to this address thread back onto the case. + contact_email: report_case.email_address, **options ) end - # Generate bulk XARF reports for multiple domains + # Generate bulk X-ARF reports for multiple domains # # @param domains [Array] array of domain records # @param options [Hash] additional options - # @return [Array] array of XARF reports + # @return [Array] array of X-ARF reports def generate_bulk_for_domains(domains, **options) domains.filter_map do |domain| report = generate_for_domain(domain, **options) @@ -128,11 +150,11 @@ def generate_bulk_for_domains(domains, **options) end end - # Generate bulk XARF reports for multiple URLs + # Generate bulk X-ARF reports for multiple URLs # # @param urls [Array] array of URL records # @param options [Hash] additional options - # @return [Array] array of XARF reports + # @return [Array] array of X-ARF reports def generate_bulk_for_urls(urls, **options) urls.filter_map do |url| report = generate_for_url(url, **options) @@ -142,270 +164,112 @@ def generate_bulk_for_urls(urls, **options) # Export reports to NDJSON format (one JSON per line) # - # @param reports [Array] array of XARF reports + # @param reports [Array] array of X-ARF reports # @return [String] NDJSON formatted string def to_ndjson(reports) - reports.map { |r| r.to_json }.join("\n") + reports.map(&:to_json).join("\n") end private - def build_report(source_identifier:, source_type:, category:, type:, confidence:, verdict:, record: nil, **options) - report = { - xarf_version: XARF_VERSION, - report_id: generate_uuid, - timestamp: Time.current.iso8601, - reporter: format_contact(reporter, include_type: true), - sender: format_contact(reporter), - source_identifier: source_identifier, - category: category, - type: type, - severity: determine_severity(type, confidence), - description: build_description(type, source_identifier, verdict) + def build_report(source_url:, mapping:, verdict: nil, source_ip: nil, detected_at: nil, + sources: nil, case_reference: nil, contact_email: nil, **options) + confidence = mapping[:confidence] + source_names = format_sources(sources || verdict&.sources_list) + + { + "Version" => SCHEMA_VERSION, + "ReporterInfo" => reporter_info(contact_email), + "Disclosure" => true, + "Report" => { + "ReportClass" => mapping[:report_class], + "ReportType" => mapping[:report_type], + "Date" => format_date(detected_at || verdict&.created_at), + "SourceUrl" => source_url, + "SourceIp" => normalize_ip(source_ip), + "Ongoing" => true, + "ReporterCaseID" => case_reference, + "ReporterSeverity" => CategoryMapper.severity_for_confidence(confidence), + "ReporterNotes" => notes(confidence, source_names, case_reference, contact_email), + "Custom" => custom_fields(confidence, source_names, case_reference), + "Samples" => options[:samples].presence + }.compact } - - # Add confidence if available - report[:confidence] = confidence.round(2) if confidence - - # Add URL if provided - report[:url] = options[:url] if options[:url].present? - - # Add type-specific fields - add_phishing_fields(report, verdict, record, options) if type == "phishing" - add_fraud_fields(report, verdict, record, options) if type == "fraud" - - # Add evidence (always include, even if empty array for spec compliance) - evidence = build_evidence(verdict, record, options) - report[:evidence] = evidence - - # Add tags - tags = build_tags(verdict, record, source_type) - report[:tags] = tags if tags.any? - - # Add optional fields - add_optional_fields(report, verdict, record, options) - - report end - def format_contact(contact, include_type: false) - result = { - org: contact[:org], - contact: contact[:contact], - domain: contact[:domain] - } - result[:type] = "automated" if include_type - result + # ReporterInfo forbids properties outside this set, so nothing else goes in. + def reporter_info(contact_email) + { + "ReporterType" => "Org", + "ReporterOrg" => reporter[:org], + "ReporterOrgDomain" => reporter[:domain], + "ReporterOrgEmail" => reporter[:email], + "ReporterContactName" => reporter[:contact_name], + "ReporterContactEmail" => contact_email || reporter[:email] + }.compact end - def build_description(type, source_identifier, verdict) - confidence_text = if verdict&.confidence_score - "#{(verdict.confidence_score * 100).round}% confidence" - else - "unconfirmed" - end + def submission_mapping(payload) + mapping = CategoryMapper.to_xarf(payload[:classification]) || + CategoryMapper::CLASSIFICATION_TO_XARF["phishing"] - sources_count = verdict&.sources_list&.count || 0 - sources_text = sources_count > 0 ? "detected by #{sources_count} source#{'s' if sources_count > 1}" : "" - - case type - when "phishing" - base = "Phishing site identified at #{source_identifier}" - [ base, confidence_text, sources_text ].reject(&:blank?).join(" - ") - when "suspicious_registration" - "Suspicious domain registration: #{source_identifier} - #{confidence_text}" - when "malware" - "Malware distribution identified at #{source_identifier} - #{confidence_text}" - when "fraud" - "Fraudulent activity identified at #{source_identifier} - #{confidence_text}" - else - "Abuse report for #{source_identifier} - #{confidence_text}" - end - end - - def generate_uuid - SecureRandom.uuid + mapping.merge(confidence: payload[:confidence].to_f) end - def determine_severity(type, confidence) - # Severity based on type and confidence - # critical: immediate threat requiring urgent action - # high: significant threat - # medium: moderate threat - # low: minor or informational - base_severity = case type - when "phishing", "malware", "fraud" - confidence && confidence >= 0.8 ? "high" : "medium" - when "suspicious_registration" - "medium" - else - "low" - end + # SourceIp only accepts an IP literal. Prefer IPv4: a report is more useful + # to a host when it names the address most of the traffic reached. + def case_source_ip(report_case) + info = report_case.domain_info || {} - # Elevate to critical for high-confidence phishing/malware - if %w[phishing malware].include?(type) && confidence && confidence >= 0.95 - "critical" - else - base_severity - end + Array(info["a_records"]).first || Array(info["aaaa_records"]).first end - def add_phishing_fields(report, verdict, record, options) - # Target brand from metadata or options - target_brand = options[:target_brand] || - verdict&.metadata_hash&.dig("target_brand") - report[:target_brand] = target_brand if target_brand.present? - - # Cloned site - cloned_site = options[:cloned_site] || - verdict&.metadata_hash&.dig("cloned_site") - report[:cloned_site] = cloned_site if cloned_site.present? - - # Credential fields if known - credential_fields = options[:credential_fields] || - verdict&.metadata_hash&.dig("credential_fields") - report[:credential_fields] = credential_fields if credential_fields.present? - - # Phishing kit identification - phishing_kit = options[:phishing_kit] || - verdict&.metadata_hash&.dig("phishing_kit") - report[:phishing_kit] = phishing_kit if phishing_kit.present? - - # Lure type - lure_type = options[:lure_type] || - verdict&.metadata_hash&.dig("lure_type") - report[:lure_type] = lure_type if lure_type.present? - end + def normalize_ip(address) + return nil if address.blank? - def add_fraud_fields(report, verdict, record, options) - # Similar to phishing but with fraud-specific context - target_brand = options[:target_brand] || - verdict&.metadata_hash&.dig("target_brand") - report[:target_brand] = target_brand if target_brand.present? + IPAddr.new(address.to_s).to_s + rescue IPAddr::InvalidAddressError + nil end - # Build evidence array for XARF report - # - # XARF Evidence can include (per spec): - # - Screenshots of phishing pages (image/png, image/jpeg) - # - Email headers and content (message/rfc822, text/plain) - # - HTTP response data (application/json, text/html) - # - DNS records (application/json) - # - WHOIS data (text/plain) - # - Malware samples (application/octet-stream) - with caution - # - Log files (text/plain, application/json) - # - API responses from detection services - # - # Each evidence item should have: - # - content_type: MIME type - # - payload: Base64-encoded content - # - description: Human-readable description - # - hashes: Array of integrity hashes (sha256:xxx, sha512:xxx, md5:xxx) - # - # TODO: Momento (screenshot capturing service) will provide: - # - Screenshots of phishing pages at time of detection - # - Visual evidence for XARF reports - # - def build_evidence(verdict, record, options) - evidence = [] - - # Add detection service responses as evidence - # These are the raw results from services like VirusTotal, Google Safe Browsing, etc. - if verdict&.sources_list&.any? - sources_json = verdict.sources_list.to_json - evidence << { - type: "detection_results", - description: "Detection service responses from #{verdict.sources_list.map { |s| s['name'] || s[:name] }.compact.join(', ')}", - hash: Digest::SHA256.hexdigest(sources_json), - hash_algorithm: "sha256" - } - end - - # Add verdict metadata as evidence if present - if verdict&.metadata_hash&.any? - metadata_json = verdict.metadata_hash.to_json - evidence << { - type: "metadata", - description: "Additional detection metadata", - hash: Digest::SHA256.hexdigest(metadata_json), - hash_algorithm: "sha256" - } - end - - # Add screenshot if provided (from Momento or manual upload) - if options[:screenshot].present? - evidence << { - type: "screenshot", - description: "Screenshot of malicious content", - hash: options[:screenshot_hash], - hash_algorithm: "sha256" - }.compact - end - - # Add custom evidence items - if options[:evidence].is_a?(Array) - evidence.concat(options[:evidence]) - end - - evidence + def format_date(timestamp) + Time.parse(timestamp.to_s).utc.iso8601 + rescue ArgumentError, TypeError + Time.current.utc.iso8601 end - def build_tags(verdict, record, source_type) - tags = [] - - # Add source type tag - tags << "phishdirectory:source:#{source_type}" - - # Add classification tag - if verdict&.classification - tags << "phishdirectory:classification:#{verdict.classification}" - end - - # Add confidence level tag - if verdict&.confidence_score - confidence_level = case verdict.confidence_score - when 0.8..1.0 then "high" - when 0.5...0.8 then "medium" - else "low" + def format_sources(sources) + names = Array(sources).filter_map do |source| + if source.is_a?(Hash) + source["service"] || source[:service] || source["name"] || source[:name] + else + source.to_s.presence end - tags << "phishdirectory:confidence:#{confidence_level}" - end - - # Add TLD tag for domains - if record.respond_to?(:tld) && record.tld.present? - tags << "phishdirectory:tld:#{record.tld.name}" - end - - # Add source service tags - verdict&.sources_list&.each do |source| - source_name = source["name"] || source[:name] - tags << "phishdirectory:detected_by:#{source_name}" if source_name end - tags.uniq + names.any? ? names.join(", ") : "phish.directory aggregated threat intelligence" end - def add_optional_fields(report, verdict, record, options) - # Reporter reference ID (our internal ID) - if record&.respond_to?(:public_id) - report[:reporter_reference_id] = record.public_id - elsif verdict&.respond_to?(:public_id) - report[:reporter_reference_id] = verdict.public_id - end + def notes(confidence, source_names, case_reference, contact_email) + notes = "Phishing site detected by phish.directory with " \ + "#{confidence_percent(confidence)}% confidence. " \ + "Detection sources: #{source_names}." + notes += " Case reference: #{case_reference}." if case_reference.present? + notes += " Reply to #{contact_email} with any update on your investigation." if contact_email.present? + notes + end - # Priority based on confidence - if verdict&.confidence_score - report[:priority] = case verdict.confidence_score - when 0.9..1.0 then "high" - when 0.7...0.9 then "medium" - else "low" - end - end + # Custom only accepts string and integer values. + def custom_fields(confidence, source_names, case_reference) + { + "CaseReference" => case_reference, + "Confidence" => confidence_percent(confidence), + "DetectionSources" => source_names + }.compact + end - # Custom fields from options - if options[:reporter_custom_fields].is_a?(Hash) - report[:reporter_custom_fields] = options[:reporter_custom_fields] - end + def confidence_percent(confidence) + (confidence.to_f * 100).round end end end diff --git a/app/services/xarf/report_parser.rb b/app/services/xarf/report_parser.rb index 795e193..5dc54ef 100644 --- a/app/services/xarf/report_parser.rb +++ b/app/services/xarf/report_parser.rb @@ -1,7 +1,10 @@ # frozen_string_literal: true module Xarf - # Parses incoming XARF v4 reports and extracts relevant data + # Parses incoming X-ARF reports and extracts relevant data. + # + # Accepts schema 3 of https://github.com/abusix/xarf, the same schema + # Xarf::ReportGenerator emits. # # Usage: # parser = Xarf::ReportParser.new(json_string_or_hash) @@ -9,25 +12,17 @@ module Xarf # result = parser.parse # # result contains normalized data for creating/updating records # else - # parser.errors # => ["Missing required field: report_id", ...] + # parser.errors # => ["Missing required field: Version", ...] # end # class ReportParser - XARF_VERSION_PATTERN = /\A4\.\d+\.\d+\z/ - UUID_PATTERN = /\A[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\z/i - - REQUIRED_FIELDS = %w[ - xarf_version - report_id - timestamp - reporter - sender - source_identifier - category - type - ].freeze - - REQUIRED_CONTACT_FIELDS = %w[org contact domain].freeze + SUPPORTED_VERSIONS = %w[3].freeze + + REQUIRED_FIELDS = %w[Version ReporterInfo Disclosure Report].freeze + REQUIRED_REPORT_FIELDS = %w[ReportClass ReportType Date].freeze + + # ReporterInfo requires these unless the reporter is a natural person. + REQUIRED_REPORTER_FIELDS = %w[ReporterOrg ReporterOrgDomain ReporterOrgEmail].freeze attr_reader :raw_data, :errors @@ -37,7 +32,7 @@ def initialize(data) @raw_data = normalize_input(data) end - # Validate the XARF report structure + # Validate the X-ARF report structure # # @return [Boolean] true if valid def valid? @@ -52,7 +47,7 @@ def parsed? @parsed end - # Parse the XARF report and extract relevant data + # Parse the X-ARF report and extract relevant data # # @return [Hash] normalized data for record creation # @raise [InvalidReportError] if report is invalid @@ -62,45 +57,42 @@ def parse @parsed = true { - report_id: raw_data["report_id"], - xarf_version: raw_data["xarf_version"], - timestamp: parse_timestamp(raw_data["timestamp"]), - category: raw_data["category"], - type: raw_data["type"], - source_identifier: raw_data["source_identifier"], - reporter: parse_contact(raw_data["reporter"]), - sender: parse_contact(raw_data["sender"]), + version: raw_data["Version"], + timestamp: parse_timestamp(report["Date"]), + report_class: report_class, + report_type: report_type, + report_subtype: report["ReportSubType"], + case_id: report["ReporterCaseID"], + severity: report["ReporterSeverity"], + notes: report["ReporterNotes"], + reporter: parse_reporter(raw_data["ReporterInfo"]), + disclosure: raw_data["Disclosure"], classification: derive_classification, confidence: derive_confidence, urls: extract_urls, domains: extract_domains, ip_addresses: extract_ip_addresses, - evidence: extract_evidence, + samples: extract_samples, metadata: extract_metadata, - tags: raw_data["tags"] || [], raw: raw_data } end - # Get the XARF category + # The Report object, which holds everything about the event itself # - # @return [String, nil] - def category - raw_data["category"] + # @return [Hash] + def report + raw_data["Report"].is_a?(Hash) ? raw_data["Report"] : {} end - # Get the XARF type - # # @return [String, nil] - def type - raw_data["type"] + def report_class + report["ReportClass"] end - # Get the source identifier (IP, domain, etc.) - # # @return [String, nil] - def source_identifier - raw_data["source_identifier"] + def report_type + report["ReportType"] end # Get URLs from the report @@ -141,243 +133,169 @@ def validate @validated = true validate_required_fields - validate_xarf_version - validate_report_id + validate_version + validate_report validate_timestamp - validate_category_and_type - validate_contacts + validate_class_and_type + validate_reporter end def validate_required_fields REQUIRED_FIELDS.each do |field| - if raw_data[field].blank? - errors << "Missing required field: #{field}" - end + # Disclosure is a boolean, so false is present but blank. + next if field == "Disclosure" && [ true, false ].include?(raw_data[field]) + + errors << "Missing required field: #{field}" if raw_data[field].blank? end end - def validate_xarf_version - version = raw_data["xarf_version"] - return if version.blank? # Already caught by required fields + def validate_version + version = raw_data["Version"] + return if version.blank? - unless version.match?(XARF_VERSION_PATTERN) - errors << "Invalid xarf_version format: expected 4.x.x" + unless SUPPORTED_VERSIONS.include?(version.to_s) + errors << "Unsupported Version: expected one of #{SUPPORTED_VERSIONS.join(', ')}" end end - def validate_report_id - report_id = raw_data["report_id"] - return if report_id.blank? + def validate_report + return errors << "Report must be an object" unless raw_data["Report"].is_a?(Hash) + + REQUIRED_REPORT_FIELDS.each do |field| + errors << "Missing required field: Report.#{field}" if report[field].blank? + end - unless report_id.match?(UUID_PATTERN) - errors << "Invalid report_id format: expected UUID v4" + # A report is anchored to an origin by either an address or a URL. + if report["SourceIp"].blank? && report["SourceUrl"].blank? + errors << "Report requires either SourceIp or SourceUrl" end end def validate_timestamp - timestamp = raw_data["timestamp"] + timestamp = report["Date"] return if timestamp.blank? Time.iso8601(timestamp) rescue ArgumentError - errors << "Invalid timestamp format: expected ISO 8601" + errors << "Invalid Report.Date format: expected ISO 8601" end - def validate_category_and_type - category = raw_data["category"] - type = raw_data["type"] + def validate_class_and_type + return if report_class.blank? || report_type.blank? - return if category.blank? || type.blank? - - unless CategoryMapper.valid_xarf_category?(category) - errors << "Invalid category: #{category}" + unless CategoryMapper.valid_report_class?(report_class) + errors << "Invalid ReportClass: #{report_class}" end - unless CategoryMapper.valid_xarf_type?(type) - errors << "Invalid type: #{type}" + unless CategoryMapper.valid_report_type?(report_type) + errors << "Invalid ReportType: #{report_type}" + return end - expected_category = CategoryMapper.category_for_type(type) - if expected_category && expected_category != category - errors << "Type '#{type}' does not belong to category '#{category}'" + unless CategoryMapper.type_in_class?(report_class, report_type) + errors << "ReportType '#{report_type}' does not belong to ReportClass '#{report_class}'" end end - def validate_contacts - %w[reporter sender].each do |contact_type| - contact = raw_data[contact_type] - next if contact.blank? - - unless contact.is_a?(Hash) - errors << "#{contact_type} must be an object" - next - end - - REQUIRED_CONTACT_FIELDS.each do |field| - if contact[field].blank? - errors << "Missing #{contact_type}.#{field}" - end - end + def validate_reporter + reporter = raw_data["ReporterInfo"] + return if reporter.blank? + + return errors << "ReporterInfo must be an object" unless reporter.is_a?(Hash) + + # Contact details are optional when the reporter is a natural person. + return if reporter["ReporterType"] == "Person" + + REQUIRED_REPORTER_FIELDS.each do |field| + errors << "Missing ReporterInfo.#{field}" if reporter[field].blank? end end def parse_timestamp(timestamp_str) return nil if timestamp_str.blank? + Time.iso8601(timestamp_str) rescue ArgumentError nil end - def parse_contact(contact) - return nil if contact.blank? + def parse_reporter(reporter) + return nil if reporter.blank? { - organization: contact["org"], - email: contact["contact"], - domain: contact["domain"] - } + type: reporter["ReporterType"], + organization: reporter["ReporterOrg"], + domain: reporter["ReporterOrgDomain"], + email: reporter["ReporterOrgEmail"], + contact_name: reporter["ReporterContactName"], + contact_email: reporter["ReporterContactEmail"], + contact_phone: reporter["ReporterContactPhone"] + }.compact end def derive_classification - CategoryMapper.from_xarf(raw_data["category"], raw_data["type"]) + CategoryMapper.from_xarf(report_class, report_type) end def derive_confidence - base_confidence = CategoryMapper.confidence_for_type(raw_data["type"]) - - # Adjust based on report confidence if provided - if raw_data["confidence"].present? - report_confidence = raw_data["confidence"].to_f.clamp(0.0, 1.0) - # Weighted average: 70% type confidence, 30% report confidence - (base_confidence * 0.7) + (report_confidence * 0.3) - else - base_confidence - end + CategoryMapper.confidence_for_type(report_type) end def extract_urls urls = [] - - # Direct URL field - urls << raw_data["url"] if raw_data["url"].present? - - # Redirect chain - if raw_data["redirect_chain"].is_a?(Array) - urls.concat(raw_data["redirect_chain"]) - end - - # Submission URL (for phishing) - urls << raw_data["submission_url"] if raw_data["submission_url"].present? - - # From evidence items - evidence_items = raw_data["evidence"] || [] - evidence_items.each do |item| - if item["content_type"]&.include?("url") && item["payload"].present? - decoded = decode_payload(item["payload"]) - urls << decoded if decoded.present? && decoded.match?(%r{\Ahttps?://}) - end - end - + urls << report["SourceUrl"] if report["SourceUrl"].present? urls.compact.uniq end def extract_domains - domains = [] - - # From source_identifier if it's a domain - source = raw_data["source_identifier"] - if source.present? && !ip_address?(source) - domains << normalize_domain(source) - end - - # From cloned_site field (target of phishing) - domains << raw_data["cloned_site"] if raw_data["cloned_site"].present? - - # From target_brand (might be a domain) - target = raw_data["target_brand"] - if target.present? && target.include?(".") - domains << normalize_domain(target) - end - - # Extract domains from URLs - urls.each do |url| - domain = extract_domain_from_url(url) - domains << domain if domain.present? - end - - domains.compact.uniq + extract_urls.filter_map { |url| extract_domain_from_url(url) }.uniq end def extract_ip_addresses - ips = [] - - source = raw_data["source_identifier"] - ips << source if source.present? && ip_address?(source) - - # From evidence or additional fields - if raw_data["additional_ip_addresses"].is_a?(Array) - raw_data["additional_ip_addresses"].each do |ip| - ips << ip if ip_address?(ip) - end - end - - ips.compact.uniq + [ report["SourceIp"], report["DestinationIp"], report["AttackerIp"] ] + .compact_blank + .select { |address| ip_address?(address) } + .uniq end - def extract_evidence - evidence_items = raw_data["evidence"] || [] + def extract_samples + Array(report["Samples"]).filter_map do |sample| + next unless sample.is_a?(Hash) - evidence_items.map do |item| { - content_type: item["content_type"], - description: item["description"], - hashes: item["hashes"] || [], - payload_size: item["payload"]&.length - } + content_type: sample["ContentType"], + description: sample["Description"], + base64_encoded: sample["Base64Encoded"], + file_name: sample["FileName"], + payload_size: sample["Payload"]&.length + }.compact end end def extract_metadata { - target_brand: raw_data["target_brand"], - cloned_site: raw_data["cloned_site"], - phishing_kit: raw_data["phishing_kit"], - credential_fields: raw_data["credential_fields"], - lure_type: raw_data["lure_type"], - detection_evasion: raw_data["detection_evasion"], - reporter_reference_id: raw_data["reporter_reference_id"], - priority: raw_data["priority"], - legacy_xarf_version: raw_data["legacy_xarf_version"] + report_subtype: report["ReportSubType"], + case_id: report["ReporterCaseID"], + severity: report["ReporterSeverity"], + ongoing: report["Ongoing"], + threat_actor: report["ThreatActor"], + source_port: report["SourcePort"], + asn: report["ASN"], + custom: report["Custom"] }.compact end - def decode_payload(payload) - Base64.decode64(payload) - rescue StandardError - nil - end - def ip_address?(str) return false if str.blank? - # IPv4 - return true if str.match?(/\A\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\z/) - - # IPv6 (simplified check) - str.include?(":") && str.match?(/\A[0-9a-f:]+\z/i) - end - - def normalize_domain(domain) - domain = domain.to_s.strip.downcase - domain = domain.sub(%r{\Ahttps?://}, "") - domain = domain.split("/").first - domain = domain.split(":").first - domain + IPAddr.new(str.to_s) + true + rescue IPAddr::InvalidAddressError + false end def extract_domain_from_url(url) - uri = URI.parse(url) - uri.host&.downcase + URI.parse(url).host&.downcase rescue URI::InvalidURIError nil end diff --git a/app/views/admin/phish_domains/show.html.erb b/app/views/admin/phish_domains/show.html.erb index b83ebc3..e8818c5 100644 --- a/app/views/admin/phish_domains/show.html.erb +++ b/app/views/admin/phish_domains/show.html.erb @@ -106,7 +106,7 @@
-

XARF v4 Report

+

X-ARF Report

Standardized abuse report format

<% if @xarf_report %> diff --git a/app/views/admin/phish_urls/show.html.erb b/app/views/admin/phish_urls/show.html.erb index fca1b98..4b53e44 100644 --- a/app/views/admin/phish_urls/show.html.erb +++ b/app/views/admin/phish_urls/show.html.erb @@ -98,7 +98,7 @@
-

XARF v4 Report

+

X-ARF Report

Standardized abuse report format

<% if @xarf_report %> diff --git a/app/views/admin/report/abuse_contacts/_form.html.erb b/app/views/admin/report/abuse_contacts/_form.html.erb index 06bd8cf..524c2bd 100644 --- a/app/views/admin/report/abuse_contacts/_form.html.erb +++ b/app/views/admin/report/abuse_contacts/_form.html.erb @@ -64,6 +64,12 @@
+
+ <%= f.check_box :accepts_xarf, class: "w-4 h-4 rounded border-surface-300 bg-surface-200 text-accent focus:ring-accent focus:ring-offset-0" %> + <%= f.label :accepts_xarf, "Send X-ARF attachment", class: "ml-2 text-sm text-surface-700" %> + For mailboxes processed by automated tooling that requires a machine readable report. +
+
<%= f.label :notes, class: "block text-xs font-medium text-surface-600 mb-1.5" %> <%= f.text_area :notes, rows: 3, class: "w-full px-3 py-2.5 bg-surface-200 border border-surface-300/50 rounded-md text-surface-900 placeholder-surface-500 text-sm focus:outline-none focus:border-accent focus:ring-1 focus:ring-accent/30 transition-colors resize-none", placeholder: "Additional notes..." %> diff --git a/app/views/docs/index.html.erb b/app/views/docs/index.html.erb index 07561ea..02a7b31 100644 --- a/app/views/docs/index.html.erb +++ b/app/views/docs/index.html.erb @@ -40,7 +40,7 @@ <%= link_to xarf_path, class: "bg-pd-secondary border border-slate-700/20 rounded-2xl p-6 hover:border-cyan-accent/30 transition-all group" do %>
🧾

XARF Generator

-

Paste a phishing link and get a XARF v4 report you can send to any provider.

+

Paste a phishing link and get an X-ARF report you can send to any provider.

<% end %> <%= link_to docs_page_path(page: 'trusted-sources'), class: "bg-pd-secondary border border-slate-700/20 rounded-2xl p-6 hover:border-cyan-accent/30 transition-all group" do %> diff --git a/app/views/reports/case_report.html.erb b/app/views/reports/case_report.html.erb index c7b9df1..4bcada4 100644 --- a/app/views/reports/case_report.html.erb +++ b/app/views/reports/case_report.html.erb @@ -112,6 +112,18 @@ <% end %> + <% if @domain_info["aaaa_records"].present? %> + + AAAA Records + +
    + <% @domain_info["aaaa_records"].each do |ip| %> +
  • <%= ip %>
    • + <% end %> +
+ + + <% end %> <% end %> diff --git a/app/views/xarf/_report.html.erb b/app/views/xarf/_report.html.erb index b5ab3d9..9f6d570 100644 --- a/app/views/xarf/_report.html.erb +++ b/app/views/xarf/_report.html.erb @@ -30,7 +30,7 @@ <%# The report itself. %>
-

XARF v4 JSON

+

X-ARF JSON