From 9faaf2c0d8c0e6e6c36d3ab6fc616d3705882e4d Mon Sep 17 00:00:00 2001 From: Hans Lemuet Date: Mon, 12 Oct 2026 16:36:24 +0200 Subject: [PATCH] perf: look up disposable domains without building a Set of them ValidEmail2.disposable_emails loads the ~180,000 disposable domains into a Set: one String object per domain plus its hash entry, about 35 MB of memory once the first email is checked. The file is sorted (see pull_mailchecker_emails.rb), so lookups now go through ValidEmail2.disposable_domains, a SortedDomainList keeping the file as a single String (2.6 MB) searched by bisection. disposable_emails still returns the same Set, built on demand. Once it is loaded, disposable_domains returns it, so code adding domains to it or removing them keeps working. --- lib/valid_email2.rb | 9 +++++++ lib/valid_email2/address.rb | 4 +-- lib/valid_email2/sorted_domain_list.rb | 36 ++++++++++++++++++++++++++ spec/benchmark_spec.rb | 8 ++++++ spec/sorted_domain_list_spec.rb | 21 +++++++++++++++ spec/valid_email2_spec.rb | 20 ++++++++++++++ 6 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 lib/valid_email2/sorted_domain_list.rb create mode 100644 spec/sorted_domain_list_spec.rb diff --git a/lib/valid_email2.rb b/lib/valid_email2.rb index 9622d93a..a40bb8af 100644 --- a/lib/valid_email2.rb +++ b/lib/valid_email2.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require "valid_email2/email_validator" +require "valid_email2/sorted_domain_list" module ValidEmail2 DENY_LIST_FILE = "config/deny_listed_email_domains.yml" @@ -8,10 +9,18 @@ module ValidEmail2 DISPOSABLE_FILE = File.expand_path('../config/disposable_email_domains.txt', __dir__) class << self + # The disposable domains as a Set, which takes about 14 MB. Lookups use disposable_domains, which + # returns this Set once it is loaded, so domains added to it or removed from it still count. def disposable_emails @disposable_emails ||= load_file(DISPOSABLE_FILE) end + # The disposable domains lookups use: the Set of disposable_emails if it was loaded, otherwise + # the sorted file searched by bisection, which takes about 3 MB. + def disposable_domains + @disposable_emails || (@sorted_disposable_domains ||= SortedDomainList.new(DISPOSABLE_FILE)) + end + def deny_list @deny_list ||= load_if_exists(DENY_LIST_FILE) || Set.new end diff --git a/lib/valid_email2/address.rb b/lib/valid_email2/address.rb index 0a7a7b81..fef00fa3 100644 --- a/lib/valid_email2/address.rb +++ b/lib/valid_email2/address.rb @@ -85,7 +85,7 @@ def disposable? end def disposable_domain? - valid? && domain_is_in?(address.domain, ValidEmail2.disposable_emails) + valid? && domain_is_in?(address.domain, ValidEmail2.disposable_domains) end def allow_listed? @@ -121,7 +121,7 @@ def valid_domain_label?(label) end def disposable_mx_server? - mx_server_is_in?(ValidEmail2.disposable_emails) + mx_server_is_in?(ValidEmail2.disposable_domains) end # TODO: (PS) keep this for backward compatibility with the test setup described in the reamde diff --git a/lib/valid_email2/sorted_domain_list.rb b/lib/valid_email2/sorted_domain_list.rb new file mode 100644 index 00000000..79fb7520 --- /dev/null +++ b/lib/valid_email2/sorted_domain_list.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module ValidEmail2 + # A list of domains read from a sorted file, one domain per line, and kept as a single String + # searched by bisection. A Set of the same domains takes several times the memory: each domain + # becomes a String object of its own, plus its hash table entry. + class SortedDomainList + NEWLINE = "\n".b.freeze + + def initialize(path) + @list = File.binread(path).freeze + end + + def include?(domain) + domain = domain.to_s.b + low = 0 + high = @list.bytesize + + while low < high + middle = (low + high) / 2 + start = (@list.rindex(NEWLINE, middle - 1) || -1) + 1 + stop = @list.index(NEWLINE, start) || @list.bytesize + line = @list.byteslice(start, stop - start) + return true if line == domain + + if line < domain + low = stop + 1 + else + high = start + end + end + + false + end + end +end diff --git a/spec/benchmark_spec.rb b/spec/benchmark_spec.rb index f2fd5a4c..625a061c 100644 --- a/spec/benchmark_spec.rb +++ b/spec/benchmark_spec.rb @@ -16,6 +16,14 @@ expect { address.disposable_domain? }.to perform_under(0.0001).sec.sample(10).times end + it "disposable_domain? has acceptable lookup performance without the Set" do + address = ValidEmail2::Address.new("test@example.com") + list = ValidEmail2::SortedDomainList.new(ValidEmail2::DISPOSABLE_FILE) + allow(ValidEmail2).to receive(:disposable_domains).and_return(list) + + expect { address.disposable_domain? }.to perform_under(0.0001).sec.sample(10).times + end + it "disposable_mx_server? has acceptable lookup performance" do address = ValidEmail2::Address.new("test@gmail.com") diff --git a/spec/sorted_domain_list_spec.rb b/spec/sorted_domain_list_spec.rb new file mode 100644 index 00000000..6c4a87f9 --- /dev/null +++ b/spec/sorted_domain_list_spec.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +require "spec_helper" + +describe ValidEmail2::SortedDomainList do + let(:domains) { File.readlines(ValidEmail2::DISPOSABLE_FILE, chomp: true) } + let(:list) { described_class.new(ValidEmail2::DISPOSABLE_FILE) } + + # The bisection relies on it; pull_mailchecker_emails.rb writes it so. + it "searches a sorted file without duplicates" do + expect(domains.each_cons(2).all? { |a, b| a.b < b.b }).to be(true) + end + + it "finds every domain of the file" do + expect(domains.reject { |domain| list.include?(domain) }).to be_empty + end + + it "finds no domain missing from the file" do + expect(%w[gmail.com example.com 0 zzzzzzzzzz.zz].select { |domain| list.include?(domain) }).to be_empty + end +end diff --git a/spec/valid_email2_spec.rb b/spec/valid_email2_spec.rb index 70cca55a..cef552bc 100644 --- a/spec/valid_email2_spec.rb +++ b/spec/valid_email2_spec.rb @@ -311,6 +311,26 @@ def set_allow_list end end + describe ".disposable_domains" do + around do |example| + loaded = ValidEmail2.instance_variable_get(:@disposable_emails) + ValidEmail2.instance_variable_set(:@disposable_emails, nil) + example.run + ValidEmail2.instance_variable_set(:@disposable_emails, loaded) + end + + it "checks a disposable domain without loading the Set of disposable_emails" do + expect(ValidEmail2::Address.new("foo@mailinator.com").disposable_domain?).to be(true) + expect(ValidEmail2.instance_variable_get(:@disposable_emails)).to be_nil + end + + it "uses the Set of disposable_emails once it is loaded, with the domains added to it" do + ValidEmail2.disposable_emails << "not-yet-listed.example" + + expect(ValidEmail2::Address.new("foo@not-yet-listed.example").disposable_domain?).to be(true) + end + end + describe "with deny list validation" do it "is valid if the domain is not deny-listed" do user = TestUserDisallowDenyListed.new(email: "foo@gmail.com")