perf: look up disposable domains without building a Set of them

lib/valid_email2.rb

@@ -1,17 +1,26 @@
 # frozen_string_literal: true
 
 require "valid_email2/email_validator"
+require "valid_email2/sorted_domain_list"
 
 module ValidEmail2
   DENY_LIST_FILE  = "config/deny_listed_email_domains.yml"
   ALLOW_LIST_FILE  = "config/allow_listed_email_domains.yml"
   DISPOSABLE_FILE = File.expand_path('../config/disposable_email_domains.txt', __dir__)
 
   class << self
+    # The disposable domains as a Set, which takes about 14 MB. Lookups use disposable_domains, which
+    # returns this Set once it is loaded, so domains added to it or removed from it still count.
     def disposable_emails
       @disposable_emails ||= load_file(DISPOSABLE_FILE)
     end
 
+    # The disposable domains lookups use: the Set of disposable_emails if it was loaded, otherwise
+    # the sorted file searched by bisection, which takes about 3 MB.
+    def disposable_domains
+      @disposable_emails || (@sorted_disposable_domains ||= SortedDomainList.new(DISPOSABLE_FILE))
+    end
+
     def deny_list
       @deny_list ||= load_if_exists(DENY_LIST_FILE) || Set.new
     end

lib/valid_email2/address.rb

@@ -85,7 +85,7 @@ def disposable?
     end
 
     def disposable_domain?
-      valid? && domain_is_in?(address.domain, ValidEmail2.disposable_emails)
+      valid? && domain_is_in?(address.domain, ValidEmail2.disposable_domains)
     end
 
     def allow_listed?
@@ -121,7 +121,7 @@ def valid_domain_label?(label)
     end
 
     def disposable_mx_server?
-      mx_server_is_in?(ValidEmail2.disposable_emails)
+      mx_server_is_in?(ValidEmail2.disposable_domains)
     end
 
     # TODO: (PS) keep this for backward compatibility with the test setup described in the reamde

lib/valid_email2/sorted_domain_list.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module ValidEmail2
+  # A list of domains read from a sorted file, one domain per line, and kept as a single String
+  # searched by bisection. A Set of the same domains takes several times the memory: each domain
+  # becomes a String object of its own, plus its hash table entry.
+  class SortedDomainList
+    NEWLINE = "\n".b.freeze
+
+    def initialize(path)
+      @list = File.binread(path).freeze
+    end
+
+    def include?(domain)
+      domain = domain.to_s.b
+      low = 0
+      high = @list.bytesize
+
+      while low < high
+        middle = (low + high) / 2
+        start = (@list.rindex(NEWLINE, middle - 1) || -1) + 1
+        stop = @list.index(NEWLINE, start) || @list.bytesize
+        line = @list.byteslice(start, stop - start)
+        return true if line == domain
+
+        if line < domain
+          low = stop + 1
+        else
+          high = start
+        end
+      end
+
+      false
+    end
+  end
+end

spec/benchmark_spec.rb

@@ -16,6 +16,14 @@
     expect { address.disposable_domain? }.to perform_under(0.0001).sec.sample(10).times
   end
 
+  it "disposable_domain? has acceptable lookup performance without the Set" do
+    address = ValidEmail2::Address.new("test@example.com")
+    list = ValidEmail2::SortedDomainList.new(ValidEmail2::DISPOSABLE_FILE)
+    allow(ValidEmail2).to receive(:disposable_domains).and_return(list)
+
+    expect { address.disposable_domain? }.to perform_under(0.0001).sec.sample(10).times
+  end
+
   it "disposable_mx_server? has acceptable lookup performance" do
     address = ValidEmail2::Address.new("test@gmail.com")
 

spec/sorted_domain_list_spec.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe ValidEmail2::SortedDomainList do
+  let(:domains) { File.readlines(ValidEmail2::DISPOSABLE_FILE, chomp: true) }
+  let(:list) { described_class.new(ValidEmail2::DISPOSABLE_FILE) }
+
+  # The bisection relies on it; pull_mailchecker_emails.rb writes it so.
+  it "searches a sorted file without duplicates" do
+    expect(domains.each_cons(2).all? { |a, b| a.b < b.b }).to be(true)
+  end
+
+  it "finds every domain of the file" do
+    expect(domains.reject { |domain| list.include?(domain) }).to be_empty
+  end
+
+  it "finds no domain missing from the file" do
+    expect(%w[gmail.com example.com 0 zzzzzzzzzz.zz].select { |domain| list.include?(domain) }).to be_empty
+  end
+end

spec/valid_email2_spec.rb

@@ -311,6 +311,26 @@ def set_allow_list
     end
   end
 
+  describe ".disposable_domains" do
+    around do |example|
+      loaded = ValidEmail2.instance_variable_get(:@disposable_emails)
+      ValidEmail2.instance_variable_set(:@disposable_emails, nil)
+      example.run
+      ValidEmail2.instance_variable_set(:@disposable_emails, loaded)
+    end
+
+    it "checks a disposable domain without loading the Set of disposable_emails" do
+      expect(ValidEmail2::Address.new("foo@mailinator.com").disposable_domain?).to be(true)
+      expect(ValidEmail2.instance_variable_get(:@disposable_emails)).to be_nil
+    end
+
+    it "uses the Set of disposable_emails once it is loaded, with the domains added to it" do
+      ValidEmail2.disposable_emails << "not-yet-listed.example"
+
+      expect(ValidEmail2::Address.new("foo@not-yet-listed.example").disposable_domain?).to be(true)
+    end
+  end
+
   describe "with deny list validation" do
     it "is valid if the domain is not deny-listed" do
       user = TestUserDisallowDenyListed.new(email: "foo@gmail.com")