feat(docs): Compress the packages with zstd

Gemfile

@@ -43,6 +43,7 @@ group :docs do
   gem 'redcarpet'
   gem 'tty-pager', require: false
   gem 'unix_utils', require: false
+  gem 'zstd-ruby', require: false
 end
 
 group :test do

Gemfile.lock

@@ -312,6 +312,7 @@ GEM
     webrick (1.9.2)
     yajl-ruby (1.4.3)
     zeitwerk (2.7.5)
+    zstd-ruby (2.0.9)
 
 PLATFORMS
   aarch64-linux
@@ -363,6 +364,7 @@ DEPENDENCIES
   unix_utils
   webrick (~> 1.9)
   yajl-ruby
+  zstd-ruby
 
 CHECKSUMS
   actionpack (8.1.3) sha256=af998cae4d47c5d581a2cc363b5c77eb718b7c4b45748d81b1887b25621c29a3
@@ -506,6 +508,7 @@ CHECKSUMS
   webrick (1.9.2) sha256=beb4a15fc474defed24a3bda4ffd88a490d517c9e4e6118c3edce59e45864131
   yajl-ruby (1.4.3) sha256=8c974d9c11ae07b0a3b6d26efea8407269b02e4138118fbe3ef0d2ec9724d1d2
   zeitwerk (2.7.5) sha256=d8da92128c09ea6ec62c949011b00ed4a20242b255293dd66bf41545398f73dd
+  zstd-ruby (2.0.9) sha256=d6e7ffb61f3b93947c3a38afe2669e11acc8e8bbf15f55dc14dd03604c0d1083
 
 RUBY VERSION
   ruby 4.0.6

docs/maintainers.md

@@ -52,6 +52,8 @@ In addition to the [publicly-documented commands](https://github.com/freeCodeCam
 
   Generates packages for one or more documentations. Those packages are intended to be uploaded to DevDocs's S3 bundle zone by maintainers via the `thor docs:upload` command, and downloaded by users via the `thor docs:download` command.
 
+  Packages are `.tar.zst` files. Documentations packaged before the switch to zstd are still published as `.tar.gz`, which `thor docs:download` falls back to; they turn into `.tar.zst` as they are re-packaged and re-uploaded. Their `.tar.gz` must be kept on the bundle zone for as long as DevDocs installations predating the switch are to be supported.
+
   Versions can be specified as such: `thor docs:package rails@5.2 node@10\ LTS`.
 
   Packages can also be automatically generated during the scraping process by passing the `--package` option to `thor docs:generate`.

lib/tasks/docs.thor

@@ -1,6 +1,15 @@
 class DocsCLI < Thor
   include Thor::Actions
 
+  # Packages are compressed with zstd, those created before the switch to it
+  # are still gzipped.
+  PACKAGE_EXTENSIONS = %w(tar.zst tar.gz).freeze
+
+  # Packaging happens once whereas the packages are downloaded over and over,
+  # which makes the slowest compression level worth its time.
+  ZSTD_COMPRESSION_LEVEL = 19
+  CHUNK_SIZE = 1024 * 1024
+
   def self.to_s
     'Docs'
   end
@@ -15,8 +24,7 @@ class DocsCLI < Thor
   option :packaged, type: :boolean
   def list
     if options[:packaged]
-      slugs = Dir[File.join(Docs.store_path, '*.tar.gz')].map { |f| File.basename(f, '.tar.gz') }
-      names = find_docs_by_slugs(slugs).map do |doc|
+      names = find_docs_by_slugs(packaged_slugs).map do |doc|
         name = if doc.version?
           "#{doc.superclass.to_s.demodulize.underscore}@#{doc.version}"
         else
@@ -161,7 +169,7 @@ class DocsCLI < Thor
 
   desc 'clean', 'Delete documentation packages and cached responses'
   def clean
-    File.delete(*Dir[File.join Docs.store_path, '*.tar.gz'])
+    File.delete(*packaged_paths)
     Docs::ResponseCache.clean
     puts 'Done'
   end
@@ -172,8 +180,7 @@ class DocsCLI < Thor
   option :rclone, type: :boolean
   def upload(*names)
     if options[:packaged]
-      slugs = Dir[File.join(Docs.store_path, '*.tar.gz')].map { |f| File.basename(f, '.tar.gz') }
-      docs = find_docs_by_slugs(slugs)
+      docs = find_docs_by_slugs(packaged_slugs)
     else
       docs = find_docs(names)
     end
@@ -187,7 +194,7 @@ class DocsCLI < Thor
         return
       end
 
-      unless File.exist?(File.join(Docs.store_path, "#{doc.path}.tar.gz"))
+      if package_path(doc).nil?
         puts "ERROR: package for '#{doc.slug}' documentation not found. Run 'thor docs:package #{doc.slug}' to create it."
         return
       end
@@ -216,7 +223,7 @@ class DocsCLI < Thor
     puts '[S3 bundle] Begin uploading.'
 
     docs.each do |doc|
-      filename = "#{doc.path}.tar.gz"
+      filename = File.basename(package_path(doc))
       puts "[S3 bundle] Uploading #{filename}..."
       cmd = "aws s3 cp #{File.join(Docs.store_path, filename)} s3://devdocs-downloads/#{filename} --profile devdocs"
       cmd << ' --dryrun' if options[:dryrun]
@@ -382,23 +389,35 @@ class DocsCLI < Thor
     if options[:rclone]
       require 'tmpdir'
       Dir.mktmpdir do |dir|
-        system("rclone copy devdocs:devdocs-downloads/#{doc.path}.tar.gz #{dir}")
-        tar_gz_path = File.join(dir, "#{File.basename(doc.path)}.tar.gz")
-        raise "rclone did not download #{doc.path}.tar.gz (not found on remote?)" unless File.exist?(tar_gz_path)
-        extract_doc(tar_gz_path, target_path)
+        extension = PACKAGE_EXTENSIONS.find do |ext|
+          system("rclone copy devdocs:devdocs-downloads/#{doc.path}.#{ext} #{dir}")
+          File.exist?(File.join(dir, "#{File.basename(doc.path)}.#{ext}"))
+        end
+        raise "rclone did not download #{doc.path} (not found on remote?)" if extension.nil?
+        extract_doc(File.join(dir, "#{File.basename(doc.path)}.#{extension}"), target_path, extension)
       end
     else
-      URI.open "https://downloads.devdocs.io/#{doc.path}.tar.gz" do |file|
-        file.close
-        extract_doc(file.path, target_path)
-        FileUtils.rm(file.path)
+      extension = PACKAGE_EXTENSIONS.find do |ext|
+        begin
+          URI.open "https://downloads.devdocs.io/#{doc.path}.#{ext}" do |file|
+            file.close
+            extract_doc(file.path, target_path, ext)
+            FileUtils.rm(file.path)
+          end
+          true
+        rescue OpenURI::HTTPError => error
+          # Packages missing from the bucket are reported as "403 Forbidden".
+          raise unless %w(403 404).include?(error.io.status.first)
+          false
+        end
       end
+      raise "#{doc.path} not found on downloads.devdocs.io" if extension.nil?
     end
   end
 
-  def extract_doc(tar_gz_path, target_path)
+  def extract_doc(archive_path, target_path, extension)
     FileUtils.mkpath(target_path)
-    tar = UnixUtils.gunzip(tar_gz_path)
+    tar = extension == 'tar.zst' ? decompress_zstd(archive_path) : UnixUtils.gunzip(archive_path)
     dir = UnixUtils.untar(tar)
     FileUtils.rm(tar)
     FileUtils.rm_rf(target_path)
@@ -410,14 +429,56 @@ class DocsCLI < Thor
 
     if File.exist?(path)
       tar = UnixUtils.tar(path)
-      gzip = UnixUtils.gzip(tar)
-      FileUtils.mv(gzip, "#{path}.tar.gz")
+      FileUtils.mv(compress_zstd(tar), "#{path}.tar.zst")
       FileUtils.rm(tar)
     else
       puts %(ERROR: can't find "#{doc.name}" documentation files.)
     end
   end
 
+  def packaged_paths
+    Dir[File.join(Docs.store_path, "*.{#{PACKAGE_EXTENSIONS.join(',')}}")]
+  end
+
+  def packaged_slugs
+    packaged_paths.map { |path| File.basename(path).sub(/\.#{Regexp.union(PACKAGE_EXTENSIONS)}\z/, '') }.uniq
+  end
+
+  def package_path(doc)
+    PACKAGE_EXTENSIONS.lazy.map { |extension| File.join(Docs.store_path, "#{doc.path}.#{extension}") }.find { |path| File.exist?(path) }
+  end
+
+  def compress_zstd(path)
+    require 'zstd-ruby'
+    stream = Zstd::StreamingCompress.new(level: ZSTD_COMPRESSION_LEVEL)
+
+    write_tmp_file(path) do |input, output|
+      output.write(stream.compress(input.read(CHUNK_SIZE))) until input.eof?
+      output.write(stream.finish)
+    end
+  end
+
+  def decompress_zstd(path)
+    require 'zstd-ruby'
+    stream = Zstd::StreamingDecompress.new
+
+    write_tmp_file(path) do |input, output|
+      output.write(stream.decompress(input.read(CHUNK_SIZE))) until input.eof?
+    end
+  end
+
+  # The packages are read and written chunk by chunk rather than at once,
+  # as they are hundreds of megabytes big.
+  def write_tmp_file(path)
+    target = UnixUtils.tmp_path(path)
+
+    File.open(target, 'wb') do |output|
+      File.open(path, 'rb') { |input| yield(input, output) }
+    end
+
+    target
+  end
+
   def generate_manifest
     Docs.generate_manifest
   end