Add content addressable gems support

.rubocop.yml

@@ -14,6 +14,7 @@ AllCops:
     - tmp/**/*
     - lib/rubygems/vendor/**/*
     - lib/bundler/vendor/**/*
+    - spec/support/vendor/**/*
   CacheRootDirectory: tmp/rubocop
   MaxFilesInCache: 5000
 

Manifest.txt

@@ -356,6 +356,7 @@ lib/rubygems/compact_index_client/http_fetcher.rb
 lib/rubygems/compact_index_client/parser.rb
 lib/rubygems/compact_index_client/updater.rb
 lib/rubygems/config_file.rb
+lib/rubygems/content_address.rb
 lib/rubygems/cooldown.rb
 lib/rubygems/cooldown_option.rb
 lib/rubygems/cooldown_settings.rb

Rakefile

@@ -164,6 +164,46 @@ namespace :vendor do
       error_message: "Vendored gems are out of sync. Please update the vendored lib patches."
     )
   end
+
+  # Pinned upstream revision of rubygems/rubygems.org that the vendored
+  # compact_index copy is generated from. Bump this (or pass COMPACT_INDEX_REF)
+  # and re-run the task to refresh.
+  COMPACT_INDEX_REF = "30703392778df8f2fb63d4cc1f45a64ffbeb7629"
+  COMPACT_INDEX_FILES = %w[
+    lib/compact_index.rb
+    lib/compact_index/dependency.rb
+    lib/compact_index/gem.rb
+    lib/compact_index/gem_version.rb
+    lib/compact_index/versions_file.rb
+  ].freeze
+
+  desc "Vendor spec-suite compact_index from rubygems.org (COMPACT_INDEX_REF to override ref)"
+  task :compact_index do
+    require "open-uri"
+    require "fileutils"
+
+    ref = ENV["COMPACT_INDEX_REF"] || COMPACT_INDEX_REF
+    dest_root = File.expand_path("spec/support/vendor/compact_index", __dir__)
+
+    COMPACT_INDEX_FILES.each do |path|
+      url = "https://raw.githubusercontent.com/rubygems/rubygems.org/#{ref}/#{path}"
+      contents = URI.parse(url).open(&:read).gsub("CompactIndex", "VendoredCompactIndex")
+
+      target = File.join(dest_root, path)
+      FileUtils.mkdir_p(File.dirname(target))
+      File.write(target, contents)
+    end
+
+    puts "Vendored compact_index from rubygems.org@#{ref} into #{dest_root}"
+  end
+
+  desc "Check vendored compact_index is up to date"
+  task compact_index_check: :compact_index do
+    Spec::Rubygems.check_source_control_changes(
+      success_message: "Vendored compact_index is in sync",
+      error_message: "Vendored compact_index is out of sync. Run `rake vendor:compact_index`."
+    )
+  end
 end
 
 namespace :rubocop do

lib/bundler/checksum.rb

@@ -187,83 +187,98 @@ def inspect
       # However, if the new checksum is from a different source, we register like normal.
       # This ensures a mismatch error where there are multiple top level sources
       # that contain the same gem with different checksums.
+      # The store is keyed by full name rather than lock name because a
+      # content-addressable build and the ordinary platform build of the same
+      # name, version, and platform share a lock name while being different
+      # files with different checksums.
       def replace(spec, checksum)
         return unless checksum
 
-        lock_name = spec.lock_name
+        full_name = spec.full_name
         @store_mutex.synchronize do
-          existing = fetch_checksum(lock_name, checksum.algo)
+          existing = fetch_checksum(full_name, checksum.algo)
           if !existing || existing.same_source?(checksum)
-            store_checksum(lock_name, checksum)
+            store_checksum(full_name, checksum)
           else
-            merge_checksum(lock_name, checksum, existing)
+            merge_checksum(full_name, checksum, existing, spec.lock_name)
           end
         end
       end
 
       def missing?(spec)
-        @store[spec.lock_name].nil?
+        @store[spec.full_name].nil?
       end
 
       def empty?(spec)
         return false unless spec.source.is_a?(Bundler::Source::Rubygems)
 
-        @store[spec.lock_name].empty?
+        @store[spec.full_name].empty?
       end
 
       def register(spec, checksum)
-        register_checksum(spec.lock_name, checksum)
+        register_checksum(spec.full_name, checksum, spec.lock_name)
       end
 
       def merge!(other)
-        other.store.each do |lock_name, checksums|
+        other.store.each do |full_name, checksums|
           checksums.each do |_algo, checksum|
-            register_checksum(lock_name, checksum)
+            register_checksum(full_name, checksum)
           end
         end
       end
 
       def to_lock(spec)
         lock_name = spec.lock_name
-        checksums = @store[lock_name]
-        if checksums&.any?
-          "#{lock_name} #{checksums.values.map(&:to_lock).sort.join(",")}"
+        checksums = checksums_to_lock(platform_full_name(spec))
+        if checksums
+          "#{lock_name} #{checksums}"
         else
           lock_name
         end
       end
 
+      def checksums_to_lock(full_name)
+        checksums = @store[full_name]
+        return unless checksums&.any?
+
+        checksums.values.map(&:to_lock).sort.join(",")
+      end
+
       private
 
-      def register_checksum(lock_name, checksum)
+      def platform_full_name(spec)
+        Gem::NameTuple.new(spec.name, spec.version, spec.platform).full_name
+      end
+
+      def register_checksum(full_name, checksum, display_name = full_name)
         @store_mutex.synchronize do
           if checksum
-            existing = fetch_checksum(lock_name, checksum.algo)
+            existing = fetch_checksum(full_name, checksum.algo)
             if existing
-              merge_checksum(lock_name, checksum, existing)
+              merge_checksum(full_name, checksum, existing, display_name)
             else
-              store_checksum(lock_name, checksum)
+              store_checksum(full_name, checksum)
             end
           else
-            init_checksum(lock_name)
+            init_checksum(full_name)
           end
         end
       end
 
-      def merge_checksum(lock_name, checksum, existing)
-        existing.merge!(checksum) || raise(ChecksumMismatchError.new(lock_name, existing, checksum))
+      def merge_checksum(full_name, checksum, existing, display_name = full_name)
+        existing.merge!(checksum) || raise(ChecksumMismatchError.new(display_name, existing, checksum))
       end
 
-      def store_checksum(lock_name, checksum)
-        init_checksum(lock_name)[checksum.algo] = checksum
+      def store_checksum(full_name, checksum)
+        init_checksum(full_name)[checksum.algo] = checksum
       end
 
-      def init_checksum(lock_name)
-        @store[lock_name] ||= {}
+      def init_checksum(full_name)
+        @store[full_name] ||= {}
       end
 
-      def fetch_checksum(lock_name, algo)
-        @store[lock_name]&.fetch(algo, nil)
+      def fetch_checksum(full_name, algo)
+        @store[full_name]&.fetch(algo, nil)
       end
     end
   end

lib/bundler/endpoint_specification.rb

@@ -5,24 +5,33 @@ module Bundler
   class EndpointSpecification < Gem::Specification
     include MatchRemoteMetadata
 
-    attr_reader :name, :version, :platform, :checksum, :created_at
+    attr_reader :name, :version, :platform, :checksum, :created_at, :content_address
     attr_writer :dependencies
     attr_accessor :remote, :locked_platform
 
-    def initialize(name, version, platform, spec_fetcher, dependencies, metadata = nil)
+    def initialize(name, version, suffix, spec_fetcher, dependencies, metadata = nil)
       super()
       @name         = name
       @version      = Gem::Version.create version
-      @platform     = Gem::Platform.new(platform)
       @spec_fetcher = spec_fetcher
       @dependencies = nil
       @unbuilt_dependencies = dependencies
+      @content_address = nil
+      @required_platform = nil
 
       @loaded_from          = nil
       @remote_specification = nil
       @locked_platform = nil
 
       parse_metadata(metadata)
+
+      if Gem::ContentAddress.content_addressed_row?(suffix, @required_platform, @required_ruby_version)
+        @content_address = suffix
+        @platform = @required_platform
+        @required_rubygems_version ||= Gem::Requirement.default
+      else
+        @platform = Gem::Platform.new(suffix)
+      end
     end
 
     def insecurely_materialized?
@@ -147,11 +156,13 @@ def inspect
     private
 
     def _remote_specification
-      @_remote_specification ||= @spec_fetcher.fetch_spec([@name, @version, @platform])
+      suffix = @content_address || @platform
+      @_remote_specification ||= @spec_fetcher.fetch_spec([@name, @version, suffix])
     end
 
     def local_specification_path
-      "#{base_dir}/specifications/#{full_name}.gemspec"
+      File.join(Gem::SpecificationRecord.specification_dir_for(self, base_dir),
+                "#{full_name}.gemspec")
     end
 
     def parse_metadata(data)
@@ -183,6 +194,8 @@ def parse_metadata(data)
           @required_ruby_version = Gem::Requirement.new(v)
         when "created_at"
           @created_at = parse_created_at(v.is_a?(Array) ? v.last : v)&.freeze
+        when "platform"
+          @required_platform = required_platform_from(Array(v).last)
         end
       end
     rescue StandardError => e
@@ -215,5 +228,12 @@ def parse_created_at(value)
     def build_dependency(name, requirements)
       Dependency.new(name, requirements)
     end
+
+    def required_platform_from(value)
+      value = value.to_s
+      return if value.empty?
+
+      Gem::Platform.new(value)
+    end
   end
 end

lib/bundler/fetcher.rb

@@ -177,13 +177,13 @@ def specs_with_retry(gem_names, source)
     def specs(gem_names, source)
       index = Bundler::Index.new
 
-      fetch_specs(gem_names).each do |name, version, platform, dependencies, metadata|
+      fetch_specs(gem_names).each do |name, version, suffix, dependencies, metadata|
         spec = if dependencies
-          EndpointSpecification.new(name, version, platform, self, dependencies, metadata).tap do |es|
+          EndpointSpecification.new(name, version, suffix, self, dependencies, metadata).tap do |es|
             source.checksum_store.replace(es, es.checksum)
           end
         else
-          RemoteSpecification.new(name, version, platform, self)
+          RemoteSpecification.new(name, version, suffix, self)
         end
         spec.source = source
         spec.remote = @remote

lib/bundler/lazy_specification.rb

@@ -8,7 +8,7 @@ class LazySpecification
     include MatchPlatform
     include ForcePlatform
 
-    attr_reader :name, :version, :platform, :materialization
+    attr_reader :name, :version, :platform, :materialization, :content_address
     attr_accessor :source, :remote, :force_ruby_platform, :dependencies, :required_ruby_version, :required_rubygems_version
     attr_accessor :overrides
 
@@ -27,21 +27,22 @@ class LazySpecification
     alias_method :runtime_dependencies, :dependencies
 
     def self.from_spec(s)
-      lazy_spec = new(s.name, s.version, s.platform, s.source)
+      lazy_spec = new(s.name, s.version, s.platform, s.source, content_address: s.content_address)
       lazy_spec.dependencies = s.runtime_dependencies
       lazy_spec.required_ruby_version = s.required_ruby_version
       lazy_spec.required_rubygems_version = s.required_rubygems_version
       lazy_spec.overrides = s.overrides if s.is_a?(LazySpecification)
       lazy_spec
     end
 
-    def initialize(name, version, platform, source = nil, **materialization_options)
+    def initialize(name, version, platform, source = nil, content_address: nil, **materialization_options)
       @name          = name
       @version       = version
       @dependencies  = []
       @required_ruby_version = Gem::Requirement.default
       @required_rubygems_version = Gem::Requirement.default
       @platform = platform || Gem::Platform::RUBY
+      @content_address = content_address
 
       @original_source = source
       @source = source
@@ -65,7 +66,9 @@ def source_changed?
     end
 
     def full_name
-      @full_name ||= if platform == Gem::Platform::RUBY
+      @full_name ||= if Gem::ContentAddress.content_addressed?(self, validate_ruby_abi: false)
+        "#{@name}-#{@version}-#{@content_address}"
+      elsif platform == Gem::Platform::RUBY
         "#{@name}-#{@version}"
       else
         "#{@name}-#{@version}-#{platform}"
@@ -81,7 +84,7 @@ def lock_name
     end
 
     def name_tuple
-      Gem::NameTuple.new(@name, @version, @platform)
+      Gem::NameTuple.new(@name, @version, @platform, content_address: @content_address)
     end
 
     def ==(other)
@@ -116,6 +119,16 @@ def satisfies?(dependency)
       @name == dependency.name && effective_requirement.satisfied_by?(Gem::Version.new(@version))
     end
 
+    ##
+    # Assigns the content address parsed from the lockfile's CONTENT
+    # ADDRESSES section. The full name embeds the content address, so its
+    # memoization must be invalidated.
+
+    def content_address=(value)
+      @content_address = value
+      @full_name = nil
+    end
+
     def to_lock
       out = String.new
       out << "    #{lock_name}\n"
@@ -192,6 +205,8 @@ def use_exact_resolved_specifications?
     # Used for legacy lockfiles and as a fallback when the exact locked spec
     # is incompatible. Falls back to frozen bundle behavior if none match.
     def resolve_best_platform(specs, locked_platforms: nil)
+      specs = MatchPlatform.select_all_content_address_match(specs, content_address)
+
       find_compatible_platform_spec(specs, locked_platforms: locked_platforms) || frozen_bundle_fallback(specs)
     end