Re-encode scraped images as WebP

.devcontainer/devcontainer.json

@@ -1,7 +1,7 @@
 {
   "image": "mcr.microsoft.com/devcontainers/ruby:4",
   "postAttachCommand": "bundle exec rackup",
-  "postCreateCommand": "bundle install",
+  "postCreateCommand": "sudo apt-get update && sudo apt-get install -y webp && bundle install",
   "mounts": [
     "source=${localEnv:HOME}/.claude,target=/home/vscode/.claude,type=bind"
   ],

.github/workflows/test.yml

@@ -11,6 +11,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+    - name: Install libwebp tools
+      run: sudo apt-get update && sudo apt-get install -y webp
     - name: Set up Ruby
       uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
       with:

Dockerfile

@@ -5,7 +5,7 @@ ENV ENABLE_SERVICE_WORKER=true
 WORKDIR /devdocs
 
 RUN apt-get update && \
-    apt-get -y install git nodejs libcurl4 libarchive-tools && \
+    apt-get -y install git nodejs libcurl4 libarchive-tools webp && \
     gem install bundler && \
     rm -rf /var/lib/apt/lists/*
 

Dockerfile-alpine

@@ -7,7 +7,7 @@ WORKDIR /devdocs
 
 COPY . /devdocs
 
-RUN apk --update add nodejs build-base libstdc++ libarchive-tools git zlib-dev libcurl && \
+RUN apk --update add nodejs build-base libstdc++ libarchive-tools git zlib-dev libcurl libwebp-tools && \
     gem install bundler && \
     bundle config set path.system true && \
     bundle config set without 'test' && \

assets/javascripts/app/config.js.erb

@@ -16,7 +16,7 @@ export const config = {
   version: <%= Time.now.to_i %>,
   release: <%= Time.now.utc.httpdate.to_json %>,
   mathml_stylesheet: '/mathml.css',
-  favicon_spritesheet: '<%= image_path('sprites/docs.png') %>',
+  favicon_spritesheet: '<%= image_path('sprites/docs.webp') %>',
   service_worker_path: '/service-worker.js',
   service_worker_enabled: <%= App.environment == :production || ENV['ENABLE_SERVICE_WORKER'] == 'true' %>,
 }

assets/stylesheets/components/_environment.scss.erb

@@ -1,3 +1,3 @@
 ._hide-in-development {
-  <%= App.environment != :production ? 'display: none;' : '' %>
+  <%= environment != :production ? 'display: none;' : '' %>
 }

assets/stylesheets/global/_icons.scss.erb

@@ -14,12 +14,12 @@
   display: block;
   width: 1rem;
   height: 1rem;
-  background-image: image-url('sprites/docs.png');
+  background-image: image-url('sprites/docs.webp');
   background-size: <%= manifest['icons_per_row'] %>rem <%= manifest['icons_per_row'] %>rem;
 }
 
 @media (-webkit-min-device-pixel-ratio: 1.5), (min-resolution: 144dpi) {
-  %doc-icon { background-image: image-url('sprites/docs@2x.png'); }
+  %doc-icon { background-image: image-url('sprites/docs@2x.webp'); }
 }
 
 html._theme-dark {

docs/filter-reference.md

@@ -82,6 +82,7 @@ The `call` method must return either `doc` or `html`, depending on the type of f
 * [`InnerHtmlFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/inner_html.rb) — converts the document to a string
 * [`CleanTextFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/clean_text.rb) — removes empty nodes
 * [`AttributionFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/attribution.rb) — appends the license info and link to the original document
+* [`ImagesFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/images.rb) — downloads and inlines images as data URIs, optimizing them and re-encoding PNGs and GIFs as lossless WebP and JPEGs as lossy WebP
 * [`TitleFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/title.rb) — prepends the document with a title (disabled by default)
 * [`EntriesFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/entries.rb) — abstract filter for extracting the page's metadata
 

docs/scraper-reference.md

@@ -109,6 +109,7 @@ Default `html_filters`:
 
 Default `text_filters`:
 
+* [`ImagesFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/images.rb) — downloads and inlines images as data URIs, optimizing them and re-encoding PNGs and GIFs as lossless WebP and JPEGs as lossy WebP
 * [`InnerHtmlFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/inner_html.rb) — converts the document to a string
 * [`CleanTextFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/clean_text.rb) — removes empty nodes
 * [`AttributionFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/attribution.rb) — appends the license info and link to the original document

lib/app.rb

@@ -56,7 +56,7 @@ class App < Sinatra::Application
                         .sort
                         .freeze
 
-    set :assets_compile, %w(*.png docs.json vendor.js unsupported.js application.css application-dark.css) + js_modules
+    set :assets_compile, %w(sprites/docs.webp sprites/docs@2x.webp docs.json vendor.js unsupported.js application.css application-dark.css) + js_modules
 
     require 'json'
     set :docs_prefix, 'docs'
@@ -267,8 +267,8 @@ def service_worker_asset_urls
         javascript_path('vendor'),
         javascript_path('unsupported'),
         stylesheet_path('application'),
-        image_path('sprites/docs.png'),
-        image_path('sprites/docs@2x.png'),
+        image_path('sprites/docs.webp'),
+        image_path('sprites/docs@2x.webp'),
       ].compact
     end
 

lib/docs/filters/core/images.rb

@@ -9,11 +9,66 @@ class ImagesFilter < Filter
 
     DEFAULT_MAX_SIZE = 120_000 # 120 kilobytes
 
+    PNG_SIGNATURE = "\x89PNG\r\n\x1a\n".b
+    GIF_SIGNATURES = ['GIF87a'.b, 'GIF89a'.b].freeze
+    JPEG_SIGNATURE = "\xff\xd8\xff".b
+
+    # WebP q=80 is roughly equivalent to JPEG q=90, and -sharp_yuv keeps the
+    # edges of the screenshots and diagrams documentation is full of crisp.
+    JPEG_QUALITY = 80
+
+    CWEBP_LOSSLESS_COMMAND = %w(cwebp -quiet -lossless -z 9 -m 6 -metadata none -o - -- -).freeze
+    CWEBP_LOSSY_COMMAND = %W(cwebp -quiet -q #{JPEG_QUALITY} -m 6 -sharp_yuv -metadata none -o - -- -).freeze
+    GIF2WEBP_COMMAND = %w(gif2webp -quiet -m 6 -metadata none -o - -- -).freeze
+
     def self.optimize_image_data(data)
       @image_optim ||= ImageOptim.new
       @image_optim.optimize_image_data(data)
     end
 
+    # Re-encodes a PNG or GIF as lossless WebP and a JPEG as lossy WebP, all of
+    # which are usually smaller. Returns nil when the data isn't an image we can
+    # convert, when the encoder isn't available, or when the result would be
+    # bigger than the original.
+    def self.convert_to_webp(data)
+      command = webp_command(data)
+      return unless command
+      webp = IO.popen(command, 'r+b', err: File::NULL) do |io|
+        io.write(data)
+        io.close_write
+        io.read
+      end
+      webp if $?.success? && !webp.empty? && webp.bytesize < data.bytesize
+    rescue SystemCallError
+      nil
+    end
+
+    def self.webp_command(data)
+      if png?(data)
+        CWEBP_LOSSLESS_COMMAND
+      elsif gif?(data)
+        # unlike cwebp, gif2webp keeps every frame of an animation
+        GIF2WEBP_COMMAND
+      elsif starts_with?(data, JPEG_SIGNATURE)
+        CWEBP_LOSSY_COMMAND
+      end
+    end
+
+    def self.png?(data)
+      return false unless starts_with?(data, PNG_SIGNATURE)
+      # cwebp silently keeps the first frame of an animated PNG
+      idat = data.index('IDAT'.b)
+      idat.nil? || !data.byteslice(0, idat).include?('acTL'.b)
+    end
+
+    def self.gif?(data)
+      GIF_SIGNATURES.any? { |signature| starts_with?(data, signature) }
+    end
+
+    def self.starts_with?(data, signature)
+      data.byteslice(0, signature.bytesize)&.b == signature
+    end
+
     def self.cache
       @cache ||= {}
     end
@@ -56,11 +111,17 @@ def call
             end
 
             image = response.body
+            mime_type = response.mime_type
 
             unless context[:optimize_images] == false
               image = self.class.optimize_image_data(image) || image
             end
 
+            if webp = self.class.convert_to_webp(image)
+              image = webp
+              mime_type = 'image/webp'
+            end
+
             size = image.bytesize
 
             if size > (context[:max_image_size] || DEFAULT_MAX_SIZE)
@@ -69,7 +130,7 @@ def call
             end
 
             image = Base64.strict_encode64(image)
-            image.prepend "data:#{response.mime_type};base64,"
+            image.prepend "data:#{mime_type};base64,"
             node['src'] = self.class.cache[src] = image
           end
         rescue => exception

lib/tasks/sprites.thor

@@ -7,14 +7,18 @@ class SpritesCLI < Thor
     require 'docs'
     require 'chunky_png'
     require 'fileutils'
-    require 'image_optim'
     require 'terminal-table'
     super
   end
 
+  # cwebp re-encodes the pixels from scratch, so the PNG ChunkyPNG hands it is
+  # only a carrier: it goes in through a pipe at the cheapest compression and
+  # never reaches the disk.
+  CWEBP_COMMAND = %w(cwebp -quiet -lossless -metadata none).freeze
+
   desc 'generate [--remove-public-icons] [--disable-optimization] [--verbose]', 'Generate the documentation icon spritesheets'
   option :remove_public_icons, type: :boolean, desc: 'Remove public/icons after generating the spritesheets'
-  option :disable_optimization, type: :boolean, desc: 'Disable optimizing the spritesheets with OptiPNG'
+  option :disable_optimization, type: :boolean, desc: "Encode the spritesheets at cwebp's fastest setting instead of its smallest"
   option :verbose, type: :boolean
   def generate
     items = get_items
@@ -41,11 +45,6 @@ class SpritesCLI < Thor
     generate_spritesheet(16, items_with_icons) {|item| item[:icon_16]}
     generate_spritesheet(32, items_with_icons) {|item| item[:icon_32]}
 
-    unless options[:disable_optimization]
-      optimize_spritesheet(get_output_path(16))
-      optimize_spritesheet(get_output_path(32))
-    end
-
     # Add Mongoose's icon details to docs without custom icons
     default_item = items_with_icons.find {|item| item[:type] == 'mongoose'}
     items_without_icons.each do |item|
@@ -173,12 +172,20 @@ class SpritesCLI < Thor
     end
 
     FileUtils.mkdir_p(File.dirname(output_path))
-    spritesheet.save(output_path)
+    save_spritesheet(spritesheet, output_path)
   end
 
-  def optimize_spritesheet(path)
-    logger.info("Optimizing spritesheet at #{path}")
-    image_optim.optimize_image!(path)
+  def save_spritesheet(spritesheet, path)
+    # -z 9 is the smallest (and slowest) lossless setting, -z 0 the fastest
+    command = [*CWEBP_COMMAND, '-z', options[:disable_optimization] ? '0' : '9', '-o', path, '--', '-']
+
+    begin
+      IO.popen(command, 'wb') {|io| io.write(spritesheet.to_blob(:fast_rgba))}
+    rescue SystemCallError
+      raise "Generating the spritesheets requires cwebp, install libwebp (Debian/Ubuntu: webp, Alpine: libwebp-tools)"
+    end
+
+    raise "cwebp failed to write #{path}" unless $?.success?
   end
 
   def save_manifest(items, icons_per_row, path)
@@ -213,7 +220,7 @@ class SpritesCLI < Thor
   end
 
   def get_output_path(size)
-    "assets/images/sprites/docs#{size == 32 ? '@2x' : ''}.png"
+    "assets/images/sprites/docs#{size == 32 ? '@2x' : ''}.webp"
   end
 
   def compile_scss_erb
@@ -222,29 +229,17 @@ class SpritesCLI < Thor
     scss_erb_files.each do |erb_path|
       scss_path = erb_path.gsub('.erb', '')
       File.open(scss_path, 'w') do |f|
-        f.write(ERB.new(File.read(erb_path)).result)
+        f.write(ERB.new(File.read(erb_path)).result(erb_binding))
         logger.info("Compiling #{erb_path} to #{scss_path}")
       end
     end
   end
 
-  def image_optim
-    @image_optim ||= ImageOptim.new(
-      :config_paths => [],
-      :advpng => false,
-      :gifsicle => false,
-      :jhead => false,
-      :jpegoptim => false,
-      :jpegrecompress => false,
-      :jpegtran => false,
-      :pngcrush => false,
-      :pngout => false,
-      :pngquant => false,
-      :svgo => false,
-      :optipng => {
-        :level => 7,
-      },
-    )
+  # The templates are compiled without the app loaded, so whatever they need
+  # has to come from here rather than from `App`.
+  def erb_binding
+    environment = (ENV['APP_ENV'] || ENV['RACK_ENV'] || 'development').to_sym
+    binding
   end
 
   def logger