diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 39403969f5..8ceea8184d 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,7 @@ { "image": "mcr.microsoft.com/devcontainers/ruby:4", "postAttachCommand": "bundle exec rackup", - "postCreateCommand": "bundle install", + "postCreateCommand": "sudo apt-get update && sudo apt-get install -y webp && bundle install", "mounts": [ "source=${localEnv:HOME}/.claude,target=/home/vscode/.claude,type=bind" ], diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 41b7a9a7c3..fda3d5c23d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,6 +11,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - name: Install libwebp tools + run: sudo apt-get update && sudo apt-get install -y webp - name: Set up Ruby uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0 with: diff --git a/Dockerfile b/Dockerfile index 8242f98717..4b9d26b64e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ ENV ENABLE_SERVICE_WORKER=true WORKDIR /devdocs RUN apt-get update && \ - apt-get -y install git nodejs libcurl4 libarchive-tools && \ + apt-get -y install git nodejs libcurl4 libarchive-tools webp && \ gem install bundler && \ rm -rf /var/lib/apt/lists/* diff --git a/Dockerfile-alpine b/Dockerfile-alpine index 864b8f587c..065464b92b 100644 --- a/Dockerfile-alpine +++ b/Dockerfile-alpine @@ -7,7 +7,7 @@ WORKDIR /devdocs COPY . /devdocs -RUN apk --update add nodejs build-base libstdc++ libarchive-tools git zlib-dev libcurl && \ +RUN apk --update add nodejs build-base libstdc++ libarchive-tools git zlib-dev libcurl libwebp-tools && \ gem install bundler && \ bundle config set path.system true && \ bundle config set without 'test' && \ diff --git a/assets/javascripts/app/config.js.erb b/assets/javascripts/app/config.js.erb index f8c8ee5598..a939a9a241 100644 --- a/assets/javascripts/app/config.js.erb +++ b/assets/javascripts/app/config.js.erb @@ -16,7 +16,7 @@ export const config = { version: <%= Time.now.to_i %>, release: <%= Time.now.utc.httpdate.to_json %>, mathml_stylesheet: '/mathml.css', - favicon_spritesheet: '<%= image_path('sprites/docs.png') %>', + favicon_spritesheet: '<%= image_path('sprites/docs.webp') %>', service_worker_path: '/service-worker.js', service_worker_enabled: <%= App.environment == :production || ENV['ENABLE_SERVICE_WORKER'] == 'true' %>, } diff --git a/assets/stylesheets/components/_environment.scss.erb b/assets/stylesheets/components/_environment.scss.erb index 5e2201c30c..74dd486659 100644 --- a/assets/stylesheets/components/_environment.scss.erb +++ b/assets/stylesheets/components/_environment.scss.erb @@ -1,3 +1,3 @@ ._hide-in-development { - <%= App.environment != :production ? 'display: none;' : '' %> + <%= environment != :production ? 'display: none;' : '' %> } diff --git a/assets/stylesheets/global/_icons.scss.erb b/assets/stylesheets/global/_icons.scss.erb index b2b22c22f1..d80f353df2 100644 --- a/assets/stylesheets/global/_icons.scss.erb +++ b/assets/stylesheets/global/_icons.scss.erb @@ -14,12 +14,12 @@ display: block; width: 1rem; height: 1rem; - background-image: image-url('sprites/docs.png'); + background-image: image-url('sprites/docs.webp'); background-size: <%= manifest['icons_per_row'] %>rem <%= manifest['icons_per_row'] %>rem; } @media (-webkit-min-device-pixel-ratio: 1.5), (min-resolution: 144dpi) { - %doc-icon { background-image: image-url('sprites/docs@2x.png'); } + %doc-icon { background-image: image-url('sprites/docs@2x.webp'); } } html._theme-dark { diff --git a/docs/filter-reference.md b/docs/filter-reference.md index fcd552d291..040814298a 100644 --- a/docs/filter-reference.md +++ b/docs/filter-reference.md @@ -82,6 +82,7 @@ The `call` method must return either `doc` or `html`, depending on the type of f * [`InnerHtmlFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/inner_html.rb) — converts the document to a string * [`CleanTextFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/clean_text.rb) — removes empty nodes * [`AttributionFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/attribution.rb) — appends the license info and link to the original document +* [`ImagesFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/images.rb) — downloads and inlines images as data URIs, optimizing them and re-encoding PNGs and GIFs as lossless WebP and JPEGs as lossy WebP * [`TitleFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/title.rb) — prepends the document with a title (disabled by default) * [`EntriesFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/entries.rb) — abstract filter for extracting the page's metadata diff --git a/docs/scraper-reference.md b/docs/scraper-reference.md index df54d8cf82..7ddb9e093a 100644 --- a/docs/scraper-reference.md +++ b/docs/scraper-reference.md @@ -109,6 +109,7 @@ Default `html_filters`: Default `text_filters`: +* [`ImagesFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/images.rb) — downloads and inlines images as data URIs, optimizing them and re-encoding PNGs and GIFs as lossless WebP and JPEGs as lossy WebP * [`InnerHtmlFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/inner_html.rb) — converts the document to a string * [`CleanTextFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/clean_text.rb) — removes empty nodes * [`AttributionFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/attribution.rb) — appends the license info and link to the original document diff --git a/lib/app.rb b/lib/app.rb index 3ec6066496..3c74344136 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -56,7 +56,7 @@ class App < Sinatra::Application .sort .freeze - set :assets_compile, %w(*.png docs.json vendor.js unsupported.js application.css application-dark.css) + js_modules + set :assets_compile, %w(sprites/docs.webp sprites/docs@2x.webp docs.json vendor.js unsupported.js application.css application-dark.css) + js_modules require 'json' set :docs_prefix, 'docs' @@ -267,8 +267,8 @@ def service_worker_asset_urls javascript_path('vendor'), javascript_path('unsupported'), stylesheet_path('application'), - image_path('sprites/docs.png'), - image_path('sprites/docs@2x.png'), + image_path('sprites/docs.webp'), + image_path('sprites/docs@2x.webp'), ].compact end diff --git a/lib/docs/filters/core/images.rb b/lib/docs/filters/core/images.rb index 5d945522b9..0b8ae38a47 100644 --- a/lib/docs/filters/core/images.rb +++ b/lib/docs/filters/core/images.rb @@ -9,11 +9,66 @@ class ImagesFilter < Filter DEFAULT_MAX_SIZE = 120_000 # 120 kilobytes + PNG_SIGNATURE = "\x89PNG\r\n\x1a\n".b + GIF_SIGNATURES = ['GIF87a'.b, 'GIF89a'.b].freeze + JPEG_SIGNATURE = "\xff\xd8\xff".b + + # WebP q=80 is roughly equivalent to JPEG q=90, and -sharp_yuv keeps the + # edges of the screenshots and diagrams documentation is full of crisp. + JPEG_QUALITY = 80 + + CWEBP_LOSSLESS_COMMAND = %w(cwebp -quiet -lossless -z 9 -m 6 -metadata none -o - -- -).freeze + CWEBP_LOSSY_COMMAND = %W(cwebp -quiet -q #{JPEG_QUALITY} -m 6 -sharp_yuv -metadata none -o - -- -).freeze + GIF2WEBP_COMMAND = %w(gif2webp -quiet -m 6 -metadata none -o - -- -).freeze + def self.optimize_image_data(data) @image_optim ||= ImageOptim.new @image_optim.optimize_image_data(data) end + # Re-encodes a PNG or GIF as lossless WebP and a JPEG as lossy WebP, all of + # which are usually smaller. Returns nil when the data isn't an image we can + # convert, when the encoder isn't available, or when the result would be + # bigger than the original. + def self.convert_to_webp(data) + command = webp_command(data) + return unless command + webp = IO.popen(command, 'r+b', err: File::NULL) do |io| + io.write(data) + io.close_write + io.read + end + webp if $?.success? && !webp.empty? && webp.bytesize < data.bytesize + rescue SystemCallError + nil + end + + def self.webp_command(data) + if png?(data) + CWEBP_LOSSLESS_COMMAND + elsif gif?(data) + # unlike cwebp, gif2webp keeps every frame of an animation + GIF2WEBP_COMMAND + elsif starts_with?(data, JPEG_SIGNATURE) + CWEBP_LOSSY_COMMAND + end + end + + def self.png?(data) + return false unless starts_with?(data, PNG_SIGNATURE) + # cwebp silently keeps the first frame of an animated PNG + idat = data.index('IDAT'.b) + idat.nil? || !data.byteslice(0, idat).include?('acTL'.b) + end + + def self.gif?(data) + GIF_SIGNATURES.any? { |signature| starts_with?(data, signature) } + end + + def self.starts_with?(data, signature) + data.byteslice(0, signature.bytesize)&.b == signature + end + def self.cache @cache ||= {} end @@ -56,11 +111,17 @@ def call end image = response.body + mime_type = response.mime_type unless context[:optimize_images] == false image = self.class.optimize_image_data(image) || image end + if webp = self.class.convert_to_webp(image) + image = webp + mime_type = 'image/webp' + end + size = image.bytesize if size > (context[:max_image_size] || DEFAULT_MAX_SIZE) @@ -69,7 +130,7 @@ def call end image = Base64.strict_encode64(image) - image.prepend "data:#{response.mime_type};base64," + image.prepend "data:#{mime_type};base64," node['src'] = self.class.cache[src] = image end rescue => exception diff --git a/lib/tasks/sprites.thor b/lib/tasks/sprites.thor index d12d314585..df415282be 100644 --- a/lib/tasks/sprites.thor +++ b/lib/tasks/sprites.thor @@ -7,14 +7,18 @@ class SpritesCLI < Thor require 'docs' require 'chunky_png' require 'fileutils' - require 'image_optim' require 'terminal-table' super end + # cwebp re-encodes the pixels from scratch, so the PNG ChunkyPNG hands it is + # only a carrier: it goes in through a pipe at the cheapest compression and + # never reaches the disk. + CWEBP_COMMAND = %w(cwebp -quiet -lossless -metadata none).freeze + desc 'generate [--remove-public-icons] [--disable-optimization] [--verbose]', 'Generate the documentation icon spritesheets' option :remove_public_icons, type: :boolean, desc: 'Remove public/icons after generating the spritesheets' - option :disable_optimization, type: :boolean, desc: 'Disable optimizing the spritesheets with OptiPNG' + option :disable_optimization, type: :boolean, desc: "Encode the spritesheets at cwebp's fastest setting instead of its smallest" option :verbose, type: :boolean def generate items = get_items @@ -41,11 +45,6 @@ class SpritesCLI < Thor generate_spritesheet(16, items_with_icons) {|item| item[:icon_16]} generate_spritesheet(32, items_with_icons) {|item| item[:icon_32]} - unless options[:disable_optimization] - optimize_spritesheet(get_output_path(16)) - optimize_spritesheet(get_output_path(32)) - end - # Add Mongoose's icon details to docs without custom icons default_item = items_with_icons.find {|item| item[:type] == 'mongoose'} items_without_icons.each do |item| @@ -173,12 +172,20 @@ class SpritesCLI < Thor end FileUtils.mkdir_p(File.dirname(output_path)) - spritesheet.save(output_path) + save_spritesheet(spritesheet, output_path) end - def optimize_spritesheet(path) - logger.info("Optimizing spritesheet at #{path}") - image_optim.optimize_image!(path) + def save_spritesheet(spritesheet, path) + # -z 9 is the smallest (and slowest) lossless setting, -z 0 the fastest + command = [*CWEBP_COMMAND, '-z', options[:disable_optimization] ? '0' : '9', '-o', path, '--', '-'] + + begin + IO.popen(command, 'wb') {|io| io.write(spritesheet.to_blob(:fast_rgba))} + rescue SystemCallError + raise "Generating the spritesheets requires cwebp, install libwebp (Debian/Ubuntu: webp, Alpine: libwebp-tools)" + end + + raise "cwebp failed to write #{path}" unless $?.success? end def save_manifest(items, icons_per_row, path) @@ -213,7 +220,7 @@ class SpritesCLI < Thor end def get_output_path(size) - "assets/images/sprites/docs#{size == 32 ? '@2x' : ''}.png" + "assets/images/sprites/docs#{size == 32 ? '@2x' : ''}.webp" end def compile_scss_erb @@ -222,29 +229,17 @@ class SpritesCLI < Thor scss_erb_files.each do |erb_path| scss_path = erb_path.gsub('.erb', '') File.open(scss_path, 'w') do |f| - f.write(ERB.new(File.read(erb_path)).result) + f.write(ERB.new(File.read(erb_path)).result(erb_binding)) logger.info("Compiling #{erb_path} to #{scss_path}") end end end - def image_optim - @image_optim ||= ImageOptim.new( - :config_paths => [], - :advpng => false, - :gifsicle => false, - :jhead => false, - :jpegoptim => false, - :jpegrecompress => false, - :jpegtran => false, - :pngcrush => false, - :pngout => false, - :pngquant => false, - :svgo => false, - :optipng => { - :level => 7, - }, - ) + # The templates are compiled without the app loaded, so whatever they need + # has to come from here rather than from `App`. + def erb_binding + environment = (ENV['APP_ENV'] || ENV['RACK_ENV'] || 'development').to_sym + binding end def logger diff --git a/test/files/image.gif b/test/files/image.gif new file mode 100644 index 0000000000..71146b2132 Binary files /dev/null and b/test/files/image.gif differ diff --git a/test/files/image.jpg b/test/files/image.jpg new file mode 100644 index 0000000000..19f813d7dd Binary files /dev/null and b/test/files/image.jpg differ diff --git a/test/lib/docs/filters/core/images_test.rb b/test/lib/docs/filters/core/images_test.rb index 2763b18cfe..acfda49e8c 100644 --- a/test/lib/docs/filters/core/images_test.rb +++ b/test/lib/docs/filters/core/images_test.rb @@ -1,6 +1,7 @@ require_relative '../../../../test_helper' require_relative '../../../../../lib/docs' require 'ostruct' +require 'chunky_png' class ImagesFilterTest < Minitest::Spec include FilterTestHelper @@ -18,6 +19,29 @@ def make_response(success: true, mime_type: 'image/png', content_length: 100, bo OpenStruct.new(success?: success, mime_type: mime_type, content_length: content_length, body: body, code: code) end + # A small gradient; lossless WebP compresses it well below the PNG. + def png_data + ChunkyPNG::Image.new(64, 64).tap do |image| + 64.times { |x| 64.times { |y| image[x, y] = ChunkyPNG::Color.rgb(x * 4, y * 4, 128) } } + end.to_blob + end + + def fixture(name) + File.binread(File.expand_path("../../../../files/#{name}", __dir__)) + end + + # Splices an acTL chunk before IDAT to mimic an animated PNG. + def apng_data + data = png_data + offset = data.index('IDAT'.b) - 4 + chunk = [8].pack('N') + 'acTL' + "\0" * 8 + [0].pack('N') + data.byteslice(0, offset) + chunk + data.byteslice(offset..-1) + end + + def image_from(src) + Base64.decode64(src.sub(/\Adata:[^,]+,/, '')) + end + def stub_request(response) stub(Docs::Request).run { |_url, &block| block.call(response) } end @@ -120,6 +144,88 @@ def stub_request(response) end end + context "with a PNG image" do + it "converts it to WebP" do + @body = IMG_BODY + data = png_data + stub_request make_response(body: data, content_length: data.bytesize) + src = filter_output.at_css('img')['src'] + assert src.start_with?('data:image/webp;base64,'), src[0, 40] + webp = image_from(src) + assert_equal 'RIFF', webp.byteslice(0, 4) + assert_equal 'WEBP', webp.byteslice(8, 4) + assert_operator webp.bytesize, :<, data.bytesize + end + + it "checks the converted size, not the PNG size, against max_image_size" do + @body = IMG_BODY + data = png_data + # no Content-Length header (chunked response): only the second check applies + context[:max_image_size] = data.bytesize - 1 + stub_request make_response(body: data, content_length: 0) + assert filter_output.at_css('img')['src'].start_with?('data:image/webp;base64,') + end + + it "keeps the PNG when the conversion doesn't pay off" do + @body = IMG_BODY + data = png_data + stub_request make_response(body: data, content_length: data.bytesize) + stub(Docs::ImagesFilter).convert_to_webp(data) { nil } + expected = "data:image/png;base64,#{Base64.strict_encode64(data)}" + assert_equal expected, filter_output.at_css('img')['src'] + end + + it "skips animated PNGs" do + assert_nil Docs::ImagesFilter.convert_to_webp(apng_data) + end + end + + context "with a GIF image" do + it "converts it to WebP" do + @body = IMG_BODY + data = fixture('image.gif') + stub_request make_response(body: data, mime_type: 'image/gif', content_length: data.bytesize) + src = filter_output.at_css('img')['src'] + assert src.start_with?('data:image/webp;base64,'), src[0, 40] + webp = image_from(src) + assert_equal 'RIFF', webp.byteslice(0, 4) + assert_equal 'WEBP', webp.byteslice(8, 4) + assert_operator webp.bytesize, :<, data.bytesize + end + + it "recognizes the GIF87a signature" do + assert Docs::ImagesFilter.convert_to_webp(fixture('image.gif').sub('GIF89a', 'GIF87a')) + end + end + + context "with a JPEG image" do + it "converts it to WebP" do + @body = IMG_BODY + data = fixture('image.jpg') + stub_request make_response(body: data, mime_type: 'image/jpeg', content_length: data.bytesize) + src = filter_output.at_css('img')['src'] + assert src.start_with?('data:image/webp;base64,'), src[0, 40] + webp = image_from(src) + assert_equal 'RIFF', webp.byteslice(0, 4) + assert_equal 'WEBP', webp.byteslice(8, 4) + assert_operator webp.bytesize, :<, data.bytesize + end + + it "encodes it lossily" do + assert_equal Docs::ImagesFilter::CWEBP_LOSSY_COMMAND, + Docs::ImagesFilter.webp_command(fixture('image.jpg')) + end + end + + context "with an image we can't convert" do + it "is left untouched" do + @body = IMG_BODY + stub_request make_response(body: 'imgdata', mime_type: 'image/bmp', content_length: 7) + expected = "data:image/bmp;base64,#{Base64.strict_encode64('imgdata')}" + assert_equal expected, filter_output.at_css('img')['src'] + end + end + context "when optimize_images is not disabled" do it "passes image data through optimize_image_data" do @body = IMG_BODY