diff --git a/lib/valid_email2.rb b/lib/valid_email2.rb index 9622d93..a40bb8a 100644 --- a/lib/valid_email2.rb +++ b/lib/valid_email2.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require "valid_email2/email_validator" +require "valid_email2/sorted_domain_list" module ValidEmail2 DENY_LIST_FILE = "config/deny_listed_email_domains.yml" @@ -8,10 +9,18 @@ module ValidEmail2 DISPOSABLE_FILE = File.expand_path('../config/disposable_email_domains.txt', __dir__) class << self + # The disposable domains as a Set, which takes about 14 MB. Lookups use disposable_domains, which + # returns this Set once it is loaded, so domains added to it or removed from it still count. def disposable_emails @disposable_emails ||= load_file(DISPOSABLE_FILE) end + # The disposable domains lookups use: the Set of disposable_emails if it was loaded, otherwise + # the sorted file searched by bisection, which takes about 3 MB. + def disposable_domains + @disposable_emails || (@sorted_disposable_domains ||= SortedDomainList.new(DISPOSABLE_FILE)) + end + def deny_list @deny_list ||= load_if_exists(DENY_LIST_FILE) || Set.new end diff --git a/lib/valid_email2/address.rb b/lib/valid_email2/address.rb index 0a7a7b8..fef00fa 100644 --- a/lib/valid_email2/address.rb +++ b/lib/valid_email2/address.rb @@ -85,7 +85,7 @@ def disposable? end def disposable_domain? - valid? && domain_is_in?(address.domain, ValidEmail2.disposable_emails) + valid? && domain_is_in?(address.domain, ValidEmail2.disposable_domains) end def allow_listed? @@ -121,7 +121,7 @@ def valid_domain_label?(label) end def disposable_mx_server? - mx_server_is_in?(ValidEmail2.disposable_emails) + mx_server_is_in?(ValidEmail2.disposable_domains) end # TODO: (PS) keep this for backward compatibility with the test setup described in the reamde diff --git a/lib/valid_email2/sorted_domain_list.rb b/lib/valid_email2/sorted_domain_list.rb new file mode 100644 index 0000000..79fb752 --- /dev/null +++ b/lib/valid_email2/sorted_domain_list.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module ValidEmail2 + # A list of domains read from a sorted file, one domain per line, and kept as a single String + # searched by bisection. A Set of the same domains takes several times the memory: each domain + # becomes a String object of its own, plus its hash table entry. + class SortedDomainList + NEWLINE = "\n".b.freeze + + def initialize(path) + @list = File.binread(path).freeze + end + + def include?(domain) + domain = domain.to_s.b + low = 0 + high = @list.bytesize + + while low < high + middle = (low + high) / 2 + start = (@list.rindex(NEWLINE, middle - 1) || -1) + 1 + stop = @list.index(NEWLINE, start) || @list.bytesize + line = @list.byteslice(start, stop - start) + return true if line == domain + + if line < domain + low = stop + 1 + else + high = start + end + end + + false + end + end +end diff --git a/spec/benchmark_spec.rb b/spec/benchmark_spec.rb index f2fd5a4..625a061 100644 --- a/spec/benchmark_spec.rb +++ b/spec/benchmark_spec.rb @@ -16,6 +16,14 @@ expect { address.disposable_domain? }.to perform_under(0.0001).sec.sample(10).times end + it "disposable_domain? has acceptable lookup performance without the Set" do + address = ValidEmail2::Address.new("test@example.com") + list = ValidEmail2::SortedDomainList.new(ValidEmail2::DISPOSABLE_FILE) + allow(ValidEmail2).to receive(:disposable_domains).and_return(list) + + expect { address.disposable_domain? }.to perform_under(0.0001).sec.sample(10).times + end + it "disposable_mx_server? has acceptable lookup performance" do address = ValidEmail2::Address.new("test@gmail.com") diff --git a/spec/sorted_domain_list_spec.rb b/spec/sorted_domain_list_spec.rb new file mode 100644 index 0000000..6c4a87f --- /dev/null +++ b/spec/sorted_domain_list_spec.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +require "spec_helper" + +describe ValidEmail2::SortedDomainList do + let(:domains) { File.readlines(ValidEmail2::DISPOSABLE_FILE, chomp: true) } + let(:list) { described_class.new(ValidEmail2::DISPOSABLE_FILE) } + + # The bisection relies on it; pull_mailchecker_emails.rb writes it so. + it "searches a sorted file without duplicates" do + expect(domains.each_cons(2).all? { |a, b| a.b < b.b }).to be(true) + end + + it "finds every domain of the file" do + expect(domains.reject { |domain| list.include?(domain) }).to be_empty + end + + it "finds no domain missing from the file" do + expect(%w[gmail.com example.com 0 zzzzzzzzzz.zz].select { |domain| list.include?(domain) }).to be_empty + end +end diff --git a/spec/valid_email2_spec.rb b/spec/valid_email2_spec.rb index 70cca55..cef552b 100644 --- a/spec/valid_email2_spec.rb +++ b/spec/valid_email2_spec.rb @@ -311,6 +311,26 @@ def set_allow_list end end + describe ".disposable_domains" do + around do |example| + loaded = ValidEmail2.instance_variable_get(:@disposable_emails) + ValidEmail2.instance_variable_set(:@disposable_emails, nil) + example.run + ValidEmail2.instance_variable_set(:@disposable_emails, loaded) + end + + it "checks a disposable domain without loading the Set of disposable_emails" do + expect(ValidEmail2::Address.new("foo@mailinator.com").disposable_domain?).to be(true) + expect(ValidEmail2.instance_variable_get(:@disposable_emails)).to be_nil + end + + it "uses the Set of disposable_emails once it is loaded, with the domains added to it" do + ValidEmail2.disposable_emails << "not-yet-listed.example" + + expect(ValidEmail2::Address.new("foo@not-yet-listed.example").disposable_domain?).to be(true) + end + end + describe "with deny list validation" do it "is valid if the domain is not deny-listed" do user = TestUserDisallowDenyListed.new(email: "foo@gmail.com")