From d360b6338ff84204a204c4c35568c7d64ce53e95 Mon Sep 17 00:00:00 2001 From: Jasper Mayone Date: Mon, 7 Sep 2026 11:10:59 -0400 Subject: [PATCH] feat: add hybrid analysis datasource Adds Phish::HybridAnalysisService, which searches the Falcon Sandbox public API v2 for sandbox reports that mention a domain or a URL. The service only reads. Submission needs an elevated key, so a restricted free key gets 403 for it. A domain search matches every report whose sample contacted the domain, so shared hosts collect malicious reports without being phishing. Domain answers are capped below URL answers, and a domain needs most of its reports to agree before the verdict is phishing rather than suspicious. The free key allows 5 requests per minute, so the service caches for 12 hours and stays out of AggregatorService::DEFAULT_SERVICES. Hybrid Analysis rejects a JSON body, so BaseService#connection now takes a request_encoding option for form encoded requests. Claude-Session: https://claude.ai/code/session_01DMcyASE3TtpFxNmDKzNxjY --- CLAUDE.md | 2 + README.md | 3 + app/services/phish/base_service.rb | 16 +- app/services/phish/hybrid_analysis_service.rb | 226 +++++++++++++ app/services/phish/service_factory.rb | 3 +- .../phish/hybrid_analysis_service_test.rb | 310 ++++++++++++++++++ 6 files changed, 557 insertions(+), 3 deletions(-) create mode 100644 app/services/phish/hybrid_analysis_service.rb create mode 100644 test/services/phish/hybrid_analysis_service_test.rb diff --git a/CLAUDE.md b/CLAUDE.md index 93c17cf..0443c6e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -149,6 +149,8 @@ urlscan: api_key: urldna: api_key: +hybrid_analysis: + api_key: # Optional - scoring weights (see credentials for values) scoring: diff --git a/README.md b/README.md index 11838e2..960e720 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,9 @@ urlscan: urldna: api_key: +hybrid_analysis: + api_key: + # Scoring configuration (optional - stored encrypted for security) scoring: min_confidence: diff --git a/app/services/phish/base_service.rb b/app/services/phish/base_service.rb index a6d6d0e..8c0a803 100644 --- a/app/services/phish/base_service.rb +++ b/app/services/phish/base_service.rb @@ -28,6 +28,9 @@ class AuthenticationError < ServiceError; end DEFAULT_TIMEOUT = 30 DEFAULT_OPEN_TIMEOUT = 10 + # Request body encodings #connection knows how to install + REQUEST_ENCODINGS = %i[json url_encoded].freeze + # Built once. OpenSSL 3 turns on CRL checking by default and the # distribution points are frequently unreachable, which turns a working # certificate into a connection failure. Peer verification stays on. @@ -59,7 +62,16 @@ def service_name protected # Create a Faraday connection with standard configuration - def connection(base_url:, timeout: DEFAULT_TIMEOUT, headers: {}) + # + # @param request_encoding [Symbol] How request bodies are encoded, :json or + # :url_encoded. Most vendors take JSON, but some (Hybrid Analysis) only + # accept application/x-www-form-urlencoded and answer a JSON body with a + # 400 error. + def connection(base_url:, timeout: DEFAULT_TIMEOUT, headers: {}, request_encoding: :json) + unless REQUEST_ENCODINGS.include?(request_encoding) + raise ArgumentError, "Unsupported request encoding: #{request_encoding}" + end + Faraday.new(url: base_url) do |conn| conn.options.timeout = timeout conn.options.open_timeout = DEFAULT_OPEN_TIMEOUT @@ -70,7 +82,7 @@ def connection(base_url:, timeout: DEFAULT_TIMEOUT, headers: {}) headers.each { |key, value| conn.headers[key] = value } # Request middleware - conn.request :json + conn.request request_encoding # Response middleware conn.response :json, content_type: /\bjson$/ diff --git a/app/services/phish/hybrid_analysis_service.rb b/app/services/phish/hybrid_analysis_service.rb new file mode 100644 index 0000000..00ef221 --- /dev/null +++ b/app/services/phish/hybrid_analysis_service.rb @@ -0,0 +1,226 @@ +# frozen_string_literal: true + +module Phish + # Hybrid Analysis (Falcon Sandbox) Public API v2 + # https://hybrid-analysis.com/docs/api/v2 + # + # We use the search endpoint only. It answers "which sandbox reports mention + # this indicator", which every account level can call. Submitting samples or + # URLs for a new scan needs an elevated key, and a restricted key gets 403 + # for it, so this service never submits. + # + # Rate Limits (restricted self-signed key, the level a free account starts at): + # 5 requests per minute + # 200 requests per hour + # + # Report verdicts: malicious, suspicious, no specific threat, whitelisted. + # A report can also carry no verdict at all, which we count as unknown. + # + # Because of the per minute limit this service is not in + # AggregatorService::DEFAULT_SERVICES. Use it on demand, the same way + # Pulsedive is used. + # + class HybridAnalysisService < BaseService + BASE_URL = "https://www.hybrid-analysis.com/api/v2/" + + rate_limit :minute, requests: 5, period: 1.minute + rate_limit :hourly, requests: 200, period: 1.hour + + # Sandbox reports do not change after they finish, so the answer for an + # indicator is stable. Cache hard: the minute limit is the real constraint. + CACHE_TTL = 12.hours + + # Report verdicts, as the API spells them + MALICIOUS = "malicious" + SUSPICIOUS = "suspicious" + NO_SPECIFIC_THREAT = "no specific threat" + WHITELISTED = "whitelisted" + + # A busy domain can match thousands of reports. We only score the page the + # API gives us, and record the full count in the details. + MAX_REPORTS_SCORED = 25 + + # A search hit is not a conviction, and the two search terms do not carry + # the same weight. A url search matches reports for that exact URL, so a + # malicious report is about the thing we asked about. A domain search + # matches every report whose sample contacted the domain, so shared + # hosting, URL shorteners and CDNs collect malicious reports without being + # phishing themselves. Domain answers are therefore capped lower, and a + # domain needs most of its reports to agree before we call it phishing + # rather than suspicious. + URL_CONFIDENCE_CEILING = 0.9 + DOMAIN_CONFIDENCE_CEILING = 0.7 + DOMAIN_MALICIOUS_RATIO = 0.5 + + # A malicious report is worth reporting even when the ratio and the threat + # score are both low, so keep the answer above the aggregator's default + # 0.3 confidence floor. + MIN_MALICIOUS_CONFIDENCE = 0.35 + + # Confidence for the verdicts that need no arithmetic + SUSPICIOUS_CONFIDENCE = 0.5 + WHITELISTED_CONFIDENCE = 0.8 + NO_SPECIFIC_THREAT_CONFIDENCE = 0.6 + + def check_domain(domain) + lookup(term: :domain, value: normalize_domain(domain)) + end + + def check_url(url) + lookup(term: :url, value: normalize_url(url)) + end + + private + + def lookup(term:, value:) + log_info("Checking #{term}: #{value}") + + cached = read_cache(term, value) + return cached if cached + + with_rate_limit do + response = search(term, value) + result = parse_search_response(response, term: term, value: value) + write_cache(term, value, result) + result + end + rescue RateLimitable::RateLimitExceeded => e + raise RateLimitError.new("#{service_name} rate limit exceeded", retry_after: e.retry_after) + end + + def search(term, value) + response = post(authenticated_connection, "search/terms", term => value) + response.is_a?(Hash) ? response : {} + end + + def credentials + Rails.application.credentials.hybrid_analysis || {} + end + + def api_key + credentials[:api_key] + end + + def authenticated_connection + raise AuthenticationError, "Hybrid Analysis API key not configured" unless api_key + + connection( + base_url: BASE_URL, + headers: { "api-key" => api_key }, + request_encoding: :url_encoded + ) + end + + def parse_search_response(response, term:, value:) + reports = Array(response["result"]) + return build_no_reports_result(term, value) if reports.empty? + + scored = reports.first(MAX_REPORTS_SCORED) + counts = tally_verdicts(scored) + threat_score = scored.filter_map { |report| report["threat_score"] }.max + verdict, confidence = classify(counts, threat_score: threat_score, term: term) + + build_result( + verdict: verdict, + confidence: confidence, + details: { + term => value, + search_term: term, + reports_total: response["count"] || reports.size, + reports_scored: scored.size, + verdict_counts: counts, + max_threat_score: threat_score, + families: families(scored), + reports: summarize(scored), + source: "hybrid_analysis" + } + ) + end + + def build_no_reports_result(term, value) + log_info("No Hybrid Analysis reports for #{term}: #{value}") + + build_result( + verdict: "unknown", + confidence: 0.0, + details: { + term => value, + search_term: term, + reports_total: 0, + not_found: true, + source: "hybrid_analysis" + } + ) + end + + def tally_verdicts(reports) + reports.each_with_object(Hash.new(0)) do |report, counts| + counts[report["verdict"].to_s.downcase.presence || "unknown"] += 1 + end + end + + def classify(counts, threat_score:, term:) + total = counts.values.sum + malicious = counts[MALICIOUS] + + if malicious.positive? + ratio = malicious.to_f / total + confidence = malicious_confidence(ratio, threat_score, term) + + # A minority of malicious reports on a domain is a lead, not a verdict. + return [ "suspicious", confidence ] if term == :domain && ratio < DOMAIN_MALICIOUS_RATIO + + return [ "phishing", confidence ] + end + + return [ "suspicious", SUSPICIOUS_CONFIDENCE ] if counts[SUSPICIOUS].positive? + return [ "clean", WHITELISTED_CONFIDENCE ] if counts[WHITELISTED].positive? + return [ "clean", NO_SPECIFIC_THREAT_CONFIDENCE ] if counts[NO_SPECIFIC_THREAT].positive? + + # Reports exist but none of them reached a verdict. + [ "unknown", 0.0 ] + end + + # How much of the evidence points one way, refined by how bad the worst + # report was. threat_score runs 0 to 100. + def malicious_confidence(ratio, threat_score, term) + ceiling = term == :domain ? DOMAIN_CONFIDENCE_CEILING : URL_CONFIDENCE_CEILING + score = threat_score.to_f.clamp(0.0, 100.0) / 100.0 + raw = ((0.6 * ratio) + (0.4 * score)) * ceiling + + raw.clamp(MIN_MALICIOUS_CONFIDENCE, ceiling).round(2) + end + + def families(reports) + reports.filter_map { |report| report["vx_family"].presence }.uniq + end + + def summarize(reports) + reports.map do |report| + { + sha256: report["sha256"], + submit_name: report["submit_name"], + verdict: report["verdict"], + threat_score: report["threat_score"], + threat_level: report["threat_level"], + av_detect: report["av_detect"], + vx_family: report["vx_family"], + analysis_start_time: report["analysis_start_time"], + environment_description: report["environment_description"] + } + end + end + + def cache_key(term, value) + "hybrid_analysis:#{term}:#{Digest::SHA256.hexdigest(value.to_s)}" + end + + def read_cache(term, value) + Rails.cache.read(cache_key(term, value)) + end + + def write_cache(term, value, result) + Rails.cache.write(cache_key(term, value), result, expires_in: CACHE_TTL) + end + end +end diff --git a/app/services/phish/service_factory.rb b/app/services/phish/service_factory.rb index eed7e56..36408cb 100644 --- a/app/services/phish/service_factory.rb +++ b/app/services/phish/service_factory.rb @@ -17,7 +17,8 @@ class ServiceFactory ipqualityscore: "Phish::IpqualityscoreService", pulsedive: "Phish::PulsediveService", checkphish: "Phish::CheckphishService", - urldna: "Phish::UrldnaService" + urldna: "Phish::UrldnaService", + hybrid_analysis: "Phish::HybridAnalysisService" }.freeze class << self diff --git a/test/services/phish/hybrid_analysis_service_test.rb b/test/services/phish/hybrid_analysis_service_test.rb new file mode 100644 index 0000000..2cc3b9c --- /dev/null +++ b/test/services/phish/hybrid_analysis_service_test.rb @@ -0,0 +1,310 @@ +# frozen_string_literal: true + +require "test_helper" + +class Phish::HybridAnalysisServiceTest < ActiveSupport::TestCase + # The real service reads the API key from credentials, which the test + # environment does not carry. + class TestableHybridAnalysisService < Phish::HybridAnalysisService + private + + def credentials + { api_key: "test_api_key" } + end + end + + # Credentials are absent in this environment, but do not depend on that. + class UnconfiguredHybridAnalysisService < Phish::HybridAnalysisService + private + + def credentials + {} + end + end + + setup do + @service = TestableHybridAnalysisService.new + end + + test "service_name returns hybrid_analysis" do + assert_equal "hybrid_analysis", Phish::HybridAnalysisService.new.service_name + end + + test "the factory builds the service" do + assert_instance_of Phish::HybridAnalysisService, Phish::ServiceFactory.build(:hybrid_analysis) + end + + test "check_domain returns phishing when most reports are malicious" do + stub_search("domain", "malicious.com", [ + report(verdict: "malicious", threat_score: 100, vx_family: "Phishing"), + report(verdict: "malicious", threat_score: 80), + report(verdict: "no specific threat", threat_score: 5) + ]) + + result = @service.check_domain("malicious.com") + + assert_equal "phishing", result[:verdict] + # ratio 0.67, threat score 1.0, capped at the domain ceiling of 0.7 + assert_equal 0.56, result[:confidence] + assert_equal "hybrid_analysis", result[:details][:source] + assert_equal "malicious.com", result[:details][:domain] + assert_equal 3, result[:details][:reports_scored] + assert_equal 2, result[:details][:verdict_counts]["malicious"] + assert_equal [ "Phishing" ], result[:details][:families] + end + + test "check_domain returns suspicious when a minority of reports are malicious" do + stub_search("domain", "shared-host.com", [ + report(verdict: "malicious", threat_score: 90), + report(verdict: "no specific threat", threat_score: 0), + report(verdict: "no specific threat", threat_score: 0), + report(verdict: "whitelisted", threat_score: 0) + ]) + + result = @service.check_domain("shared-host.com") + + assert_equal "suspicious", result[:verdict] + assert result[:confidence] < Phish::HybridAnalysisService::DOMAIN_CONFIDENCE_CEILING + end + + test "check_domain never answers above the domain ceiling" do + stub_search("domain", "all-bad.com", [ + report(verdict: "malicious", threat_score: 100), + report(verdict: "malicious", threat_score: 100) + ]) + + result = @service.check_domain("all-bad.com") + + assert_equal "phishing", result[:verdict] + assert_equal Phish::HybridAnalysisService::DOMAIN_CONFIDENCE_CEILING, result[:confidence] + end + + test "check_url returns phishing with a higher ceiling than a domain" do + stub_search("url", "https://evil.com/login", [ + report(verdict: "malicious", threat_score: 100, vx_family: "Phishing") + ]) + + result = @service.check_url("https://evil.com/login") + + assert_equal "phishing", result[:verdict] + assert_equal Phish::HybridAnalysisService::URL_CONFIDENCE_CEILING, result[:confidence] + assert_equal "https://evil.com/login", result[:details][:url] + end + + test "check_url returns suspicious for a lone malicious report only on domains" do + stub_search("url", "https://mixed.com/page", [ + report(verdict: "malicious", threat_score: 10), + report(verdict: "no specific threat", threat_score: 0), + report(verdict: "no specific threat", threat_score: 0) + ]) + + result = @service.check_url("https://mixed.com/page") + + # A url search matches the exact URL, so the minority rule does not apply. + assert_equal "phishing", result[:verdict] + end + + test "malicious verdicts stay above the aggregator confidence floor" do + stub_search("url", "https://weak.com/page", [ + report(verdict: "malicious", threat_score: 0), + report(verdict: "no specific threat", threat_score: 0), + report(verdict: "no specific threat", threat_score: 0), + report(verdict: "no specific threat", threat_score: 0), + report(verdict: "no specific threat", threat_score: 0), + report(verdict: "no specific threat", threat_score: 0), + report(verdict: "no specific threat", threat_score: 0), + report(verdict: "no specific threat", threat_score: 0), + report(verdict: "no specific threat", threat_score: 0), + report(verdict: "no specific threat", threat_score: 0) + ]) + + result = @service.check_url("https://weak.com/page") + + assert_equal Phish::HybridAnalysisService::MIN_MALICIOUS_CONFIDENCE, result[:confidence] + end + + test "check_domain returns suspicious when reports are suspicious" do + stub_search("domain", "odd.com", [ + report(verdict: "suspicious", threat_score: 40) + ]) + + result = @service.check_domain("odd.com") + + assert_equal "suspicious", result[:verdict] + assert_equal 0.5, result[:confidence] + end + + test "check_domain returns clean for a whitelisted domain" do + stub_search("domain", "google.com", [ + report(verdict: "whitelisted", threat_score: 0) + ]) + + result = @service.check_domain("google.com") + + assert_equal "clean", result[:verdict] + assert_equal 0.8, result[:confidence] + end + + test "check_domain returns clean when reports found no specific threat" do + stub_search("domain", "boring.com", [ + report(verdict: "no specific threat", threat_score: 0) + ]) + + result = @service.check_domain("boring.com") + + assert_equal "clean", result[:verdict] + assert_equal 0.6, result[:confidence] + end + + test "check_domain returns unknown when reports carry no verdict" do + stub_search("domain", "pending.com", [ + report(verdict: nil, threat_score: nil) + ]) + + result = @service.check_domain("pending.com") + + assert_equal "unknown", result[:verdict] + assert_equal 0.0, result[:confidence] + end + + test "check_domain returns unknown when nothing matches" do + stub_search("domain", "unheard-of.com", []) + + result = @service.check_domain("unheard-of.com") + + assert_equal "unknown", result[:verdict] + assert_equal 0.0, result[:confidence] + assert result[:details][:not_found] + assert_equal 0, result[:details][:reports_total] + end + + test "check_domain scores at most MAX_REPORTS_SCORED reports" do + reports = Array.new(40) { report(verdict: "malicious", threat_score: 100) } + stub_search("domain", "busy.com", reports, count: 4_000) + + result = @service.check_domain("busy.com") + + assert_equal Phish::HybridAnalysisService::MAX_REPORTS_SCORED, result[:details][:reports_scored] + assert_equal 4_000, result[:details][:reports_total] + assert_equal Phish::HybridAnalysisService::MAX_REPORTS_SCORED, result[:details][:reports].size + end + + test "check_domain normalizes the domain before searching" do + stub_search("domain", "example.com", [ report(verdict: "whitelisted") ]) + + result = @service.check_domain("https://EXAMPLE.COM/path?query=1") + + assert_equal "clean", result[:verdict] + assert_requested :post, search_url, body: { "domain" => "example.com" }, times: 1 + end + + test "the request is form encoded and carries the api key" do + stub_search("domain", "example.com", [ report(verdict: "whitelisted") ]) + + @service.check_domain("example.com") + + assert_requested :post, search_url, times: 1 do |request| + request.headers["Api-Key"] == "test_api_key" && + request.headers["Content-Type"].start_with?("application/x-www-form-urlencoded") && + request.body == "domain=example.com" + end + end + + test "repeated lookups are served from the cache" do + original_cache = Rails.cache + Rails.cache = ActiveSupport::Cache::MemoryStore.new + + stub_search("domain", "cached.com", [ report(verdict: "malicious", threat_score: 100) ]) + + first = @service.check_domain("cached.com") + second = @service.check_domain("cached.com") + + assert_equal "phishing", first[:verdict] + assert_equal first[:details][:reports], second[:details][:reports] + assert_requested :post, search_url, times: 1 + ensure + Rails.cache = original_cache + end + + test "a missing api key raises an authentication error" do + assert_raises Phish::BaseService::AuthenticationError do + UnconfiguredHybridAnalysisService.new.check_domain("example.com") + end + end + + test "a 429 response raises a rate limit error carrying retry_after" do + stub_request(:post, search_url).to_return( + status: 429, + body: { message: "Exceeded maximum API requests per minute(5)" }.to_json, + headers: { "Content-Type" => "application/json", "Retry-After" => "30" } + ) + + error = assert_raises Phish::BaseService::RateLimitError do + @service.check_domain("throttled.com") + end + + assert_equal 30, error.retry_after + end + + test "a 403 response raises an authentication error" do + stub_request(:post, search_url).to_return( + status: 403, + body: { message: "Forbidden" }.to_json, + headers: { "Content-Type" => "application/json" } + ) + + assert_raises Phish::BaseService::AuthenticationError do + @service.check_domain("forbidden.com") + end + end + + test "exhausting the local rate limit raises a rate limit error" do + original_cache = Rails.cache + Rails.cache = ActiveSupport::Cache::MemoryStore.new + + stub_request(:post, search_url).to_return( + status: 200, + body: { count: 0, result: [] }.to_json, + headers: { "Content-Type" => "application/json" } + ) + + # The minute limit is 5 requests, and each domain is a distinct cache key. + 5.times { |i| @service.check_domain("limit-#{i}.com") } + + assert_raises Phish::BaseService::RateLimitError do + @service.check_domain("limit-6.com") + end + ensure + Rails.cache = original_cache + end + + private + + def search_url + "https://www.hybrid-analysis.com/api/v2/search/terms" + end + + def report(verdict: "malicious", threat_score: 100, vx_family: nil) + { + "sha256" => SecureRandom.hex(32), + "submit_name" => "sample.exe", + "verdict" => verdict, + "threat_score" => threat_score, + "threat_level" => 2, + "av_detect" => "50", + "vx_family" => vx_family, + "analysis_start_time" => "2026-01-01T00:00:00+00:00", + "environment_description" => "Windows 10 64 bit" + } + end + + def stub_search(term, value, reports, count: nil) + stub_request(:post, search_url) + .with(body: { term => value }) + .to_return( + status: 200, + body: { count: count || reports.size, search_terms: [ { id: term, value: value } ], result: reports }.to_json, + headers: { "Content-Type" => "application/json" } + ) + end +end