From 47fc8bf7df667808bbd659100d9a88c7de6d8370 Mon Sep 17 00:00:00 2001 From: Harriet Oughton Date: Tue, 22 Sep 2026 18:11:36 +0100 Subject: [PATCH 1/2] Update gem build flags to include --content-addressable to build a content addressable gem Co-authored-by: Gira Chawda Co-authored-by: Jenny Shen --- spec/commands/clean_spec.rb | 1 + spec/install/gemfile/content_addressable_spec.rb | 1 + 2 files changed, 2 insertions(+) diff --git a/spec/commands/clean_spec.rb b/spec/commands/clean_spec.rb index e60bdc215964..4efeef8795a6 100644 --- a/spec/commands/clean_spec.rb +++ b/spec/commands/clean_spec.rb @@ -413,6 +413,7 @@ def should_not_have_gems(*gems) end it "does not remove gemspecs for content-addressed gems in the bundle", :compact_index, rubygems: ">= 4.1.0.a" do + skip "Gem::ContentAddress not available" if ruby_core? skip "A prerelease Ruby does not satisfy the ~> X.Y.0 ABI pin of content-addressed gems" if Gem.ruby_version.prerelease? simulate_platform "x86_64-linux" do diff --git a/spec/install/gemfile/content_addressable_spec.rb b/spec/install/gemfile/content_addressable_spec.rb index 0bd608118e69..eba194b1cb3e 100644 --- a/spec/install/gemfile/content_addressable_spec.rb +++ b/spec/install/gemfile/content_addressable_spec.rb @@ -2,6 +2,7 @@ RSpec.describe "bundle install with content-addressable gems", :compact_index, rubygems: ">= 4.1.0.a" do before do + skip "Gem::ContentAddress not available" if ruby_core? skip "A prerelease Ruby does not satisfy the ~> X.Y.0 ABI pin of content-addressed gems" if Gem.ruby_version.prerelease? end From fa9cb2b16dc0fadd127e38973d37be3cd68056f3 Mon Sep 17 00:00:00 2001 From: Harriet Oughton Date: Tue, 29 Sep 2026 10:49:52 -0400 Subject: [PATCH 2/2] Support content addressable gem builds / gem naming in PackageTask Adds Gem::PackageTask#content_addressable. When true, the gem task builds a content addressable gem for the single Ruby ABI identified by gem_spec.required_ruby_version, passing content_addressable through to Gem::Package.build. A content addressable gem's file name contains a hash of its contents, so it isn't known until the gem is built and can't be a Rake target. In this mode the task uses a stamp file (pkg/-.gem-built) as its Rake target, writes the built gem's path into it, and rebuilds when the referenced gem no longer exists. The staging directory is suffixed with the Ruby ABI (pkg/-) so per-ABI builds of the same full_name don't share a directory. When content_addressable is false, behaviour is unchanged. Co-authored-by: Gira Chawda Co-authored-by: Jenny Shen --- lib/rubygems/content_address.rb | 17 ++ lib/rubygems/package.rb | 19 +- lib/rubygems/package_task.rb | 81 ++++- test/rubygems/test_gem_package_task.rb | 391 ++++++++++++++++++++++++- 4 files changed, 470 insertions(+), 38 deletions(-) diff --git a/lib/rubygems/content_address.rb b/lib/rubygems/content_address.rb index 77ad66faaf3e..0f3dcaa34f57 100644 --- a/lib/rubygems/content_address.rb +++ b/lib/rubygems/content_address.rb @@ -90,6 +90,23 @@ def self.eligible?(spec, validate_ruby_abi: true) !ruby_abi_for(spec.required_ruby_version).nil? end + ## + # Raises ArgumentError unless +spec+ is eligible for content addressing + + def self.eligible!(spec) + unless platform_eligible?(spec.platform) + raise ArgumentError, + "Cannot build a content-addressable gem as no platform or a Ruby platform has been set" + end + + return unless ruby_abi_for(spec.required_ruby_version).nil? + + raise ArgumentError, + "Cannot build a content-addressable gem because required_ruby_version is set to " \ + "#{spec.required_ruby_version}. Please set required_ruby_version to \"~> X.Y.0\" " \ + "so that it identifies a single Ruby ABI." + end + ## # Whether +spec+ is content-addressed: it is eligible for content # addressing and has a valid content address set. See eligible? for diff --git a/lib/rubygems/package.rb b/lib/rubygems/package.rb index 064449e22267..a81c399805b0 100644 --- a/lib/rubygems/package.rb +++ b/lib/rubygems/package.rb @@ -401,7 +401,7 @@ def build(skip_validation = false, strict_validation = false) # +required_rubygems_version+ is constrained before building. def build_content_addressable_file(skip_validation = false, strict_validation = false) - validate_content_addressable_spec + Gem::ContentAddress.eligible!(@spec) ruby_abi = Gem::ContentAddress.ruby_abi_for(@spec.required_ruby_version) @spec.required_rubygems_version = normalized_required_rubygems_version(ruby_abi) @@ -819,23 +819,6 @@ def satisfies_rubygems_floor?(requirement, floor) end end - ## - # Validates that the spec can be built as a content-addressable gem: it - # must declare a non-Ruby platform and its +required_ruby_version+ must - # identify a single Ruby ABI. - - def validate_content_addressable_spec - return if Gem::ContentAddress.eligible?(@spec) - - unless Gem::ContentAddress.platform_eligible?(@spec.platform) - raise ArgumentError, "Cannot build a content-addressable gem as no platform or a Ruby platform has been set" - end - - raise ArgumentError, - "Cannot build a content-addressable gem because required_ruby_version is set to #{@spec.required_ruby_version}. " \ - "Please set required_ruby_version to \"~> X.Y.0\" so that it identifies a single Ruby ABI." - end - ## # Returns the full path for installing +filename+ into +destination_dir+, # which must already be resolved with File.realpath by the caller. diff --git a/lib/rubygems/package_task.rb b/lib/rubygems/package_task.rb index 398f7820ad77..fa746137ce1c 100644 --- a/lib/rubygems/package_task.rb +++ b/lib/rubygems/package_task.rb @@ -65,6 +65,10 @@ class Gem::PackageTask < Rake::PackageTask attr_accessor :gem_spec + ## + # Indicates whether this package should be built as a content-addressable gem. + attr_accessor :content_addressable + ## # Create a Gem Package task library. Automatically define the gem if a # block is given. If no block is supplied, then #define needs to be called @@ -82,6 +86,7 @@ def initialize(gem_spec) def init(gem) super gem.full_name, :noversion @gem_spec = gem + @content_addressable = false @package_files += gem_spec.files if gem_spec.files @fileutils_output = $stdout end @@ -91,33 +96,97 @@ def init(gem) # (+define+ is automatically called if a block is given to +new+). def define + Gem::ContentAddress.eligible!(gem_spec) if content_addressable + super gem_file = File.basename gem_spec.cache_file gem_path = File.join package_dir, gem_file - gem_dir = File.join package_dir, gem_spec.full_name + gem_dir = package_dir_path + + if content_addressable + build_target = stamp_file + gem_description = "content-addressable gem #{gem_spec.name}-#{gem_spec.version} " \ + "(Platform: #{gem_spec.platform}, Ruby ABI: #{content_addressable_ruby_abi})" + + discard_stale_stamp_file + else + build_target = gem_path + gem_description = "gem file #{gem_file}" + end task package: [:gem] directory package_dir directory gem_dir - desc "Build the gem file #{gem_file}" - task gem: [gem_path] + desc "Build the #{gem_description}" + task gem: [build_target] trace = Rake.application.options.trace Gem.configuration.verbose = trace - file gem_path => [package_dir, gem_dir] + @gem_spec.files do + file build_target => [package_dir, gem_dir] + @gem_spec.files do + previous_gem = recorded_gem_path if content_addressable + built_gem_path = nil + chdir(gem_dir) do - when_writing "Creating #{gem_spec.file_name}" do - built_gem_file = Gem::Package.build gem_spec + when_writing "Creating #{gem_description}" do + built_gem_file = Gem::Package.build gem_spec, content_addressable: content_addressable verbose trace do mv built_gem_file, ".." end + + if content_addressable + built_gem_path = File.join(package_dir, built_gem_file) + File.write File.join("..", File.basename(stamp_file)), built_gem_path + end + end + end + + if previous_gem && built_gem_path && previous_gem != built_gem_path + verbose trace do + rm_f previous_gem end end end end + + ## + # The name of this package. Content-addressable builds append the Ruby ABI + # so per-ABI builds of the same +full_name+ do not share a staging + # directory. + + def package_name + name = super + ruby_abi = content_addressable_ruby_abi + ruby_abi ? "#{name}-#{ruby_abi}" : name + end + + private + + def content_addressable_ruby_abi + return unless content_addressable + + Gem::ContentAddress.ruby_abi_for(gem_spec.required_ruby_version) + end + + def stamp_file + File.join package_dir, "#{package_name}.gem-built" + end + + def recorded_gem_path + return unless File.file?(stamp_file) + + path = File.read(stamp_file).strip + path unless path.empty? + end + + def discard_stale_stamp_file + return unless File.file?(stamp_file) + + path = recorded_gem_path + File.delete(stamp_file) unless path && File.file?(path) + end end diff --git a/test/rubygems/test_gem_package_task.rb b/test/rubygems/test_gem_package_task.rb index e180cc5d4ce7..a57feea07327 100644 --- a/test/rubygems/test_gem_package_task.rb +++ b/test/rubygems/test_gem_package_task.rb @@ -38,12 +38,14 @@ def test_gem_package Rake.application["package"].invoke assert_path_exist "pkg/pkgr-1.2.3.gem" + assert_path_not_exist "pkg/pkgr-1.2.3.gem-built" + assert_equal %w[x y], Dir.children("pkg/pkgr-1.2.3").sort end ensure RakeFileUtils.verbose_flag = original_rake_fileutils_verbosity end - def test_moves_filename_returned_by_build + def test_builds_content_addressable_gem_and_records_it_in_the_stamp_file original_rake_fileutils_verbosity = RakeFileUtils.verbose_flag RakeFileUtils.verbose_flag = false @@ -58,32 +60,304 @@ def test_moves_filename_returned_by_build g.summary = "summary" end - Rake.application = Rake::Application.new + Dir.chdir @tempdir do + FileUtils.touch "x" + FileUtils.touch "y" - pkg = Gem::PackageTask.new(gem) do |p| - p.package_files << "y" + Rake.application = Rake::Application.new + + pkg = Gem::PackageTask.new(gem) do |p| + p.package_files << "y" + p.content_addressable = true + end + + assert_equal %w[x y], pkg.package_files + + Rake.application["package"].invoke + + built_files = Dir["pkg/pkgr-1.2.3-*.gem"] + assert_equal 1, built_files.length + + built_gem_path = built_files.first + assert_match(%r{\Apkg/pkgr-1\.2\.3-[0-9a-f]{8}\.gem\z}, built_gem_path) + assert_path_not_exist "pkg/pkgr-1.2.3-arm64-darwin.gem" + + stamp_path = "pkg/pkgr-1.2.3-arm64-darwin-3.4.gem-built" + assert_path_exist stamp_path + assert_equal built_gem_path, File.read(stamp_path) + + assert_equal %w[x y], Dir.children("pkg/pkgr-1.2.3-arm64-darwin-3.4").sort + assert_path_not_exist "pkg/pkgr-1.2.3-arm64-darwin" + + built_spec = Gem::Package.new(built_gem_path).spec + assert_equal "3.4", built_spec.ruby_abi + assert_equal Gem::Requirement.new("~> 3.4.0"), built_spec.required_ruby_version end + ensure + RakeFileUtils.verbose_flag = original_rake_fileutils_verbosity + end - assert_equal %w[x y], pkg.package_files + def test_rebuilds_content_addressable_gem_when_stamp_points_to_missing_gem + original_rake_fileutils_verbosity = RakeFileUtils.verbose_flag + RakeFileUtils.verbose_flag = false + + gem = Gem::Specification.new do |g| + g.name = "pkgr" + g.version = "1.2.3" + g.platform = "arm64-darwin" + g.required_ruby_version = "~> 3.4.0" + + g.authors = %w[author] + g.files = %w[x] + g.summary = "summary" + end Dir.chdir @tempdir do FileUtils.touch "x" - FileUtils.touch "y" + FileUtils.mkdir_p "pkg" - built_gem_file = "pkgr-1.2.3-01234567.gem" + stamp_path = "pkg/pkgr-1.2.3-arm64-darwin-3.4.gem-built" + missing_gem_path = "pkg/pkgr-1.2.3-deadbeef.gem" + File.write stamp_path, missing_gem_path - Gem::Package.stub :build, ->(_) { - FileUtils.touch built_gem_file - built_gem_file - } do - Rake.application["package"].invoke + Rake.application = Rake::Application.new + Gem::PackageTask.new(gem) do |package| + package.content_addressable = true end + assert_path_not_exist stamp_path + + Rake.application["package"].invoke + built_files = Dir["pkg/pkgr-1.2.3-*.gem"] + assert_equal 1, built_files.length + + built_gem_path = built_files.first + assert_match(%r{\Apkg/pkgr-1\.2\.3-[0-9a-f]{8}\.gem\z}, built_gem_path) + assert_path_exist stamp_path + assert_equal built_gem_path, File.read(stamp_path) + end + ensure + RakeFileUtils.verbose_flag = original_rake_fileutils_verbosity + end + + def test_discards_empty_content_addressable_stamp_file + gem = Gem::Specification.new do |g| + g.name = "pkgr" + g.version = "1.2.3" + g.platform = "arm64-darwin" + g.required_ruby_version = "~> 3.4.0" + + g.authors = %w[author] + g.files = %w[x] + g.summary = "summary" + end + + Dir.chdir @tempdir do + FileUtils.touch "x" + FileUtils.mkdir_p "pkg" + + stamp_path = "pkg/pkgr-1.2.3-arm64-darwin-3.4.gem-built" + FileUtils.touch stamp_path + out, err = capture_output do + Rake.application = Rake::Application.new + Gem::PackageTask.new(gem) do |package| + package.content_addressable = true + end + end + + assert_path_not_exist stamp_path + assert_empty out + assert_empty err + end + end + + def test_does_not_rebuild_content_addressable_gem_when_stamp_is_up_to_date + original_rake_fileutils_verbosity = RakeFileUtils.verbose_flag + RakeFileUtils.verbose_flag = false + + gem = Gem::Specification.new do |g| + g.name = "pkgr" + g.version = "1.2.3" + g.platform = "arm64-darwin" + g.required_ruby_version = "~> 3.4.0" + + g.authors = %w[author] + g.files = %w[x] + g.summary = "summary" + end + + Dir.chdir @tempdir do + FileUtils.touch "x" + + Rake.application = Rake::Application.new + Gem::PackageTask.new(gem) do |package| + package.content_addressable = true + end + Rake.application["package"].invoke + + built_files = Dir["pkg/pkgr-1.2.3-*.gem"] assert_equal 1, built_files.length - assert_equal "pkg/pkgr-1.2.3-01234567.gem", built_files.first - assert_path_not_exist "pkg/pkgr-1.2.3-arm64-darwin.gem" + + Rake.application = Rake::Application.new + Gem::PackageTask.new(gem) do |package| + package.content_addressable = true + end + + Gem::Package.stub :build, ->(*) { flunk "expected the gem not to be rebuilt" } do + Rake.application["package"].invoke + end + + assert_equal built_files, Dir["pkg/pkgr-1.2.3-*.gem"] + assert_equal built_files.first, File.read("pkg/pkgr-1.2.3-arm64-darwin-3.4.gem-built") + end + ensure + RakeFileUtils.verbose_flag = original_rake_fileutils_verbosity + end + + def test_rebuilds_content_addressable_gem_and_removes_previous_build_when_files_change + original_rake_fileutils_verbosity = RakeFileUtils.verbose_flag + RakeFileUtils.verbose_flag = false + + gem = Gem::Specification.new do |g| + g.name = "pkgr" + g.version = "1.2.3" + g.platform = "arm64-darwin" + g.required_ruby_version = "~> 3.4.0" + + g.authors = %w[author] + g.files = %w[x] + g.summary = "summary" + end + + Dir.chdir @tempdir do + stamp_path = "pkg/pkgr-1.2.3-arm64-darwin-3.4.gem-built" + + FileUtils.touch "x" + + Rake.application = Rake::Application.new + Gem::PackageTask.new(gem) do |package| + package.content_addressable = true + end + Rake.application["package"].invoke + + first_gem_path = File.read(stamp_path) + assert_path_exist first_gem_path + + File.write "x", "changed" + newer = File.mtime(stamp_path) + 10 + File.utime newer, newer, "x" + + Rake.application = Rake::Application.new + Gem::PackageTask.new(gem) do |package| + package.content_addressable = true + end + Rake.application["package"].invoke + + second_gem_path = File.read(stamp_path) + refute_equal first_gem_path, second_gem_path + + assert_path_not_exist first_gem_path + assert_path_exist second_gem_path + assert_equal [second_gem_path], Dir["pkg/pkgr-1.2.3-*.gem"] + end + ensure + RakeFileUtils.verbose_flag = original_rake_fileutils_verbosity + end + + def test_keeps_previous_content_addressable_gem_when_rebuild_fails + original_rake_fileutils_verbosity = RakeFileUtils.verbose_flag + RakeFileUtils.verbose_flag = false + + gem = Gem::Specification.new do |g| + g.name = "pkgr" + g.version = "1.2.3" + g.platform = "arm64-darwin" + g.required_ruby_version = "~> 3.4.0" + + g.authors = %w[author] + g.files = %w[x] + g.summary = "summary" + end + + Dir.chdir @tempdir do + stamp_path = "pkg/pkgr-1.2.3-arm64-darwin-3.4.gem-built" + + FileUtils.touch "x" + + Rake.application = Rake::Application.new + Gem::PackageTask.new(gem) do |package| + package.content_addressable = true + end + Rake.application["package"].invoke + + first_gem_path = File.read(stamp_path) + assert_path_exist first_gem_path + + File.write "x", "changed" + newer = File.mtime(stamp_path) + 10 + File.utime newer, newer, "x" + + Rake.application = Rake::Application.new + Gem::PackageTask.new(gem) do |package| + package.content_addressable = true + end + + Gem::Package.stub :build, ->(*) { raise Gem::InvalidSpecificationException, "boom" } do + assert_raise(Gem::InvalidSpecificationException) do + Rake.application["package"].invoke + end + end + + assert_path_exist first_gem_path + assert_equal first_gem_path, File.read(stamp_path) + assert_equal [first_gem_path], Dir["pkg/pkgr-1.2.3-*.gem"] + end + ensure + RakeFileUtils.verbose_flag = original_rake_fileutils_verbosity + end + + def test_keeps_content_addressable_gem_when_rebuild_produces_same_content + original_rake_fileutils_verbosity = RakeFileUtils.verbose_flag + RakeFileUtils.verbose_flag = false + + gem = Gem::Specification.new do |g| + g.name = "pkgr" + g.version = "1.2.3" + g.platform = "arm64-darwin" + g.required_ruby_version = "~> 3.4.0" + + g.authors = %w[author] + g.files = %w[x] + g.summary = "summary" + end + + Dir.chdir @tempdir do + stamp_path = "pkg/pkgr-1.2.3-arm64-darwin-3.4.gem-built" + + FileUtils.touch "x" + + Rake.application = Rake::Application.new + Gem::PackageTask.new(gem) do |package| + package.content_addressable = true + end + Rake.application["package"].invoke + + first_gem_path = File.read(stamp_path) + + newer = File.mtime(stamp_path) + 10 + File.utime newer, newer, "x" + + Rake.application = Rake::Application.new + Gem::PackageTask.new(gem) do |package| + package.content_addressable = true + end + Rake.application["package"].invoke + + assert_equal first_gem_path, File.read(stamp_path) + assert_path_exist first_gem_path + assert_equal [first_gem_path], Dir["pkg/pkgr-1.2.3-*.gem"] end ensure RakeFileUtils.verbose_flag = original_rake_fileutils_verbosity @@ -157,4 +431,93 @@ def test_package_dir_path assert_equal "pkg/nokogiri-1.5.0-java", pkg.package_dir_path end + + def test_package_dir_path_with_content_addressable + gem = Gem::Specification.new do |g| + g.name = "nokogiri" + g.version = "1.5.0" + g.platform = "x86_64-linux" + g.required_ruby_version = "~> 3.4.0" + end + + pkg = Gem::PackageTask.new gem + pkg.content_addressable = true + pkg.define + + assert_equal "nokogiri-1.5.0-x86_64-linux-3.4", pkg.package_name + assert_equal "pkg/nokogiri-1.5.0-x86_64-linux-3.4", pkg.package_dir_path + assert_equal "nokogiri-1.5.0-x86_64-linux-3.4.tgz", pkg.tgz_file + end + + def test_content_addressable_package_with_need_tar + original_rake_fileutils_verbosity = RakeFileUtils.verbose_flag + RakeFileUtils.verbose_flag = false + + gem = Gem::Specification.new do |g| + g.name = "pkgr" + g.version = "1.2.3" + g.platform = "arm64-darwin" + g.required_ruby_version = "~> 3.4.0" + + g.authors = %w[author] + g.files = %w[x] + g.summary = "summary" + end + + Dir.chdir @tempdir do + FileUtils.touch "x" + + Rake.application = Rake::Application.new + + Gem::PackageTask.new(gem) do |p| + p.content_addressable = true + p.need_tar = true + end + + Rake.application["package"].invoke + + built_files = Dir["pkg/pkgr-1.2.3-*.gem"] + assert_equal 1, built_files.length + assert_equal built_files.first, File.read("pkg/pkgr-1.2.3-arm64-darwin-3.4.gem-built") + assert_path_exist "pkg/pkgr-1.2.3-arm64-darwin-3.4.tgz" + end + ensure + RakeFileUtils.verbose_flag = original_rake_fileutils_verbosity + end + + def test_content_addressable_raises_when_required_ruby_version_does_not_identify_single_abi + gem = Gem::Specification.new do |g| + g.name = "nokogiri" + g.version = "1.5.0" + g.platform = "x86_64-linux" + g.required_ruby_version = ">= 3.4" + end + + Rake.application = Rake::Application.new + + pkg = Gem::PackageTask.new gem + pkg.content_addressable = true + + error = assert_raise(ArgumentError) { pkg.define } + assert_match(/required_ruby_version is set to >= 3.4/, error.message) + assert_match(/identifies a single Ruby ABI/, error.message) + assert_empty Rake.application.tasks + end + + def test_content_addressable_raises_when_platform_is_ruby + gem = Gem::Specification.new do |g| + g.name = "pkgr" + g.version = "1.2.3" + g.required_ruby_version = "~> 3.4.0" + end + + Rake.application = Rake::Application.new + + pkg = Gem::PackageTask.new gem + pkg.content_addressable = true + + error = assert_raise(ArgumentError) { pkg.define } + assert_match(/no platform or a Ruby platform has been set/, error.message) + assert_empty Rake.application.tasks + end end if defined?(Rake::PackageTask)