From d7844f17e1c3e3c46a318a7758db697b2c5b1e31 Mon Sep 17 00:00:00 2001 From: Simon Legner Date: Sun, 13 Sep 2026 16:17:27 +0200 Subject: [PATCH 1/5] Convert scraped PNG images to lossless WebP Pipe PNGs through `cwebp -lossless` after image_optim, before the final size check. Roughly 15% smaller across the PNGs in the existing docs. Falls back to the PNG when the data isn't a PNG, when it's animated (cwebp would silently drop all but the first frame), when the WebP comes out bigger, or when cwebp isn't installed. WebP needs Safari 14+, Edge 18+ and iOS 14+, well below the versions the app already requires. --- .devcontainer/devcontainer.json | 2 +- .github/workflows/test.yml | 2 + Dockerfile | 2 +- Dockerfile-alpine | 2 +- docs/filter-reference.md | 1 + docs/scraper-reference.md | 1 + lib/docs/filters/core/images.rb | 33 +++++++++++- test/lib/docs/filters/core/images_test.rb | 65 +++++++++++++++++++++++ 8 files changed, 104 insertions(+), 4 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 39403969f5..8ceea8184d 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,7 @@ { "image": "mcr.microsoft.com/devcontainers/ruby:4", "postAttachCommand": "bundle exec rackup", - "postCreateCommand": "bundle install", + "postCreateCommand": "sudo apt-get update && sudo apt-get install -y webp && bundle install", "mounts": [ "source=${localEnv:HOME}/.claude,target=/home/vscode/.claude,type=bind" ], diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 41b7a9a7c3..fda3d5c23d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,6 +11,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - name: Install libwebp tools + run: sudo apt-get update && sudo apt-get install -y webp - name: Set up Ruby uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0 with: diff --git a/Dockerfile b/Dockerfile index 8242f98717..4b9d26b64e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ ENV ENABLE_SERVICE_WORKER=true WORKDIR /devdocs RUN apt-get update && \ - apt-get -y install git nodejs libcurl4 libarchive-tools && \ + apt-get -y install git nodejs libcurl4 libarchive-tools webp && \ gem install bundler && \ rm -rf /var/lib/apt/lists/* diff --git a/Dockerfile-alpine b/Dockerfile-alpine index 864b8f587c..065464b92b 100644 --- a/Dockerfile-alpine +++ b/Dockerfile-alpine @@ -7,7 +7,7 @@ WORKDIR /devdocs COPY . /devdocs -RUN apk --update add nodejs build-base libstdc++ libarchive-tools git zlib-dev libcurl && \ +RUN apk --update add nodejs build-base libstdc++ libarchive-tools git zlib-dev libcurl libwebp-tools && \ gem install bundler && \ bundle config set path.system true && \ bundle config set without 'test' && \ diff --git a/docs/filter-reference.md b/docs/filter-reference.md index fcd552d291..837b860342 100644 --- a/docs/filter-reference.md +++ b/docs/filter-reference.md @@ -82,6 +82,7 @@ The `call` method must return either `doc` or `html`, depending on the type of f * [`InnerHtmlFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/inner_html.rb) — converts the document to a string * [`CleanTextFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/clean_text.rb) — removes empty nodes * [`AttributionFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/attribution.rb) — appends the license info and link to the original document +* [`ImagesFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/images.rb) — downloads and inlines images as data URIs, optimizing them and re-encoding PNGs as lossless WebP * [`TitleFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/title.rb) — prepends the document with a title (disabled by default) * [`EntriesFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/entries.rb) — abstract filter for extracting the page's metadata diff --git a/docs/scraper-reference.md b/docs/scraper-reference.md index df54d8cf82..ec8e729f5e 100644 --- a/docs/scraper-reference.md +++ b/docs/scraper-reference.md @@ -109,6 +109,7 @@ Default `html_filters`: Default `text_filters`: +* [`ImagesFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/images.rb) — downloads and inlines images as data URIs, optimizing them and re-encoding PNGs as lossless WebP * [`InnerHtmlFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/inner_html.rb) — converts the document to a string * [`CleanTextFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/clean_text.rb) — removes empty nodes * [`AttributionFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/attribution.rb) — appends the license info and link to the original document diff --git a/lib/docs/filters/core/images.rb b/lib/docs/filters/core/images.rb index 5d945522b9..e00a112be7 100644 --- a/lib/docs/filters/core/images.rb +++ b/lib/docs/filters/core/images.rb @@ -9,11 +9,36 @@ class ImagesFilter < Filter DEFAULT_MAX_SIZE = 120_000 # 120 kilobytes + PNG_SIGNATURE = "\x89PNG\r\n\x1a\n".b + CWEBP_COMMAND = %w(cwebp -quiet -lossless -z 9 -m 6 -metadata none -o - -- -).freeze + def self.optimize_image_data(data) @image_optim ||= ImageOptim.new @image_optim.optimize_image_data(data) end + # Losslessly re-encodes a PNG as WebP, which is usually 10-50% smaller. + # Returns nil when the data isn't a PNG we can convert, when cwebp isn't + # available, or when the result would be bigger than the original. + def self.convert_to_webp(data) + return unless png?(data) + webp = IO.popen(CWEBP_COMMAND, 'r+b', err: File::NULL) do |io| + io.write(data) + io.close_write + io.read + end + webp if $?.success? && !webp.empty? && webp.bytesize < data.bytesize + rescue SystemCallError + nil + end + + def self.png?(data) + return false unless data.byteslice(0, PNG_SIGNATURE.bytesize)&.b == PNG_SIGNATURE + # cwebp silently keeps the first frame of an animated PNG + idat = data.index('IDAT'.b) + idat.nil? || !data.byteslice(0, idat).include?('acTL'.b) + end + def self.cache @cache ||= {} end @@ -56,11 +81,17 @@ def call end image = response.body + mime_type = response.mime_type unless context[:optimize_images] == false image = self.class.optimize_image_data(image) || image end + if webp = self.class.convert_to_webp(image) + image = webp + mime_type = 'image/webp' + end + size = image.bytesize if size > (context[:max_image_size] || DEFAULT_MAX_SIZE) @@ -69,7 +100,7 @@ def call end image = Base64.strict_encode64(image) - image.prepend "data:#{response.mime_type};base64," + image.prepend "data:#{mime_type};base64," node['src'] = self.class.cache[src] = image end rescue => exception diff --git a/test/lib/docs/filters/core/images_test.rb b/test/lib/docs/filters/core/images_test.rb index 2763b18cfe..f7babdfe46 100644 --- a/test/lib/docs/filters/core/images_test.rb +++ b/test/lib/docs/filters/core/images_test.rb @@ -1,6 +1,7 @@ require_relative '../../../../test_helper' require_relative '../../../../../lib/docs' require 'ostruct' +require 'chunky_png' class ImagesFilterTest < Minitest::Spec include FilterTestHelper @@ -18,6 +19,25 @@ def make_response(success: true, mime_type: 'image/png', content_length: 100, bo OpenStruct.new(success?: success, mime_type: mime_type, content_length: content_length, body: body, code: code) end + # A small gradient; lossless WebP compresses it well below the PNG. + def png_data + ChunkyPNG::Image.new(64, 64).tap do |image| + 64.times { |x| 64.times { |y| image[x, y] = ChunkyPNG::Color.rgb(x * 4, y * 4, 128) } } + end.to_blob + end + + # Splices an acTL chunk before IDAT to mimic an animated PNG. + def apng_data + data = png_data + offset = data.index('IDAT'.b) - 4 + chunk = [8].pack('N') + 'acTL' + "\0" * 8 + [0].pack('N') + data.byteslice(0, offset) + chunk + data.byteslice(offset..-1) + end + + def image_from(src) + Base64.decode64(src.sub(/\Adata:[^,]+,/, '')) + end + def stub_request(response) stub(Docs::Request).run { |_url, &block| block.call(response) } end @@ -120,6 +140,51 @@ def stub_request(response) end end + context "with a PNG image" do + it "converts it to WebP" do + @body = IMG_BODY + data = png_data + stub_request make_response(body: data, content_length: data.bytesize) + src = filter_output.at_css('img')['src'] + assert src.start_with?('data:image/webp;base64,'), src[0, 40] + webp = image_from(src) + assert_equal 'RIFF', webp.byteslice(0, 4) + assert_equal 'WEBP', webp.byteslice(8, 4) + assert_operator webp.bytesize, :<, data.bytesize + end + + it "checks the converted size, not the PNG size, against max_image_size" do + @body = IMG_BODY + data = png_data + # no Content-Length header (chunked response): only the second check applies + context[:max_image_size] = data.bytesize - 1 + stub_request make_response(body: data, content_length: 0) + assert filter_output.at_css('img')['src'].start_with?('data:image/webp;base64,') + end + + it "keeps the PNG when the conversion doesn't pay off" do + @body = IMG_BODY + data = png_data + stub_request make_response(body: data, content_length: data.bytesize) + stub(Docs::ImagesFilter).convert_to_webp(data) { nil } + expected = "data:image/png;base64,#{Base64.strict_encode64(data)}" + assert_equal expected, filter_output.at_css('img')['src'] + end + + it "skips animated PNGs" do + assert_nil Docs::ImagesFilter.convert_to_webp(apng_data) + end + end + + context "with a non-PNG image" do + it "is left untouched" do + @body = IMG_BODY + stub_request make_response(body: 'imgdata', mime_type: 'image/gif', content_length: 7) + expected = "data:image/gif;base64,#{Base64.strict_encode64('imgdata')}" + assert_equal expected, filter_output.at_css('img')['src'] + end + end + context "when optimize_images is not disabled" do it "passes image data through optimize_image_data" do @body = IMG_BODY From 19f4579cb8d30be4afc53d11067596240831afbd Mon Sep 17 00:00:00 2001 From: Simon Legner Date: Sun, 13 Sep 2026 16:19:02 +0200 Subject: [PATCH 2/5] Convert scraped GIF images to lossless WebP Route GIFs through gif2webp, sharing the size comparison and fallbacks with the PNG path. The wins are larger than for PNGs -- the GIFs in the existing docs drop to roughly a quarter of their size -- since GIF's LZW leaves a lot on the table. gif2webp keeps every frame, so animations survive the conversion. --- docs/filter-reference.md | 2 +- docs/scraper-reference.md | 2 +- lib/docs/filters/core/images.rb | 33 ++++++++++++++++++----- test/lib/docs/filters/core/images_test.rb | 29 +++++++++++++++++--- 4 files changed, 55 insertions(+), 11 deletions(-) diff --git a/docs/filter-reference.md b/docs/filter-reference.md index 837b860342..4b3ed096fc 100644 --- a/docs/filter-reference.md +++ b/docs/filter-reference.md @@ -82,7 +82,7 @@ The `call` method must return either `doc` or `html`, depending on the type of f * [`InnerHtmlFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/inner_html.rb) — converts the document to a string * [`CleanTextFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/clean_text.rb) — removes empty nodes * [`AttributionFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/attribution.rb) — appends the license info and link to the original document -* [`ImagesFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/images.rb) — downloads and inlines images as data URIs, optimizing them and re-encoding PNGs as lossless WebP +* [`ImagesFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/images.rb) — downloads and inlines images as data URIs, optimizing them and re-encoding PNGs and GIFs as lossless WebP * [`TitleFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/title.rb) — prepends the document with a title (disabled by default) * [`EntriesFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/entries.rb) — abstract filter for extracting the page's metadata diff --git a/docs/scraper-reference.md b/docs/scraper-reference.md index ec8e729f5e..c976642694 100644 --- a/docs/scraper-reference.md +++ b/docs/scraper-reference.md @@ -109,7 +109,7 @@ Default `html_filters`: Default `text_filters`: -* [`ImagesFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/images.rb) — downloads and inlines images as data URIs, optimizing them and re-encoding PNGs as lossless WebP +* [`ImagesFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/images.rb) — downloads and inlines images as data URIs, optimizing them and re-encoding PNGs and GIFs as lossless WebP * [`InnerHtmlFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/inner_html.rb) — converts the document to a string * [`CleanTextFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/clean_text.rb) — removes empty nodes * [`AttributionFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/attribution.rb) — appends the license info and link to the original document diff --git a/lib/docs/filters/core/images.rb b/lib/docs/filters/core/images.rb index e00a112be7..06afa31597 100644 --- a/lib/docs/filters/core/images.rb +++ b/lib/docs/filters/core/images.rb @@ -10,19 +10,23 @@ class ImagesFilter < Filter DEFAULT_MAX_SIZE = 120_000 # 120 kilobytes PNG_SIGNATURE = "\x89PNG\r\n\x1a\n".b + GIF_SIGNATURES = ['GIF87a'.b, 'GIF89a'.b].freeze CWEBP_COMMAND = %w(cwebp -quiet -lossless -z 9 -m 6 -metadata none -o - -- -).freeze + GIF2WEBP_COMMAND = %w(gif2webp -quiet -m 6 -metadata none -o - -- -).freeze def self.optimize_image_data(data) @image_optim ||= ImageOptim.new @image_optim.optimize_image_data(data) end - # Losslessly re-encodes a PNG as WebP, which is usually 10-50% smaller. - # Returns nil when the data isn't a PNG we can convert, when cwebp isn't - # available, or when the result would be bigger than the original. + # Losslessly re-encodes a PNG or GIF as WebP, which is usually smaller + # (10-50% for PNGs, considerably more for GIFs). Returns nil when the data + # isn't an image we can convert, when the encoder isn't available, or when + # the result would be bigger than the original. def self.convert_to_webp(data) - return unless png?(data) - webp = IO.popen(CWEBP_COMMAND, 'r+b', err: File::NULL) do |io| + command = webp_command(data) + return unless command + webp = IO.popen(command, 'r+b', err: File::NULL) do |io| io.write(data) io.close_write io.read @@ -32,13 +36,30 @@ def self.convert_to_webp(data) nil end + def self.webp_command(data) + if png?(data) + CWEBP_COMMAND + elsif gif?(data) + # unlike cwebp, gif2webp keeps every frame of an animation + GIF2WEBP_COMMAND + end + end + def self.png?(data) - return false unless data.byteslice(0, PNG_SIGNATURE.bytesize)&.b == PNG_SIGNATURE + return false unless starts_with?(data, PNG_SIGNATURE) # cwebp silently keeps the first frame of an animated PNG idat = data.index('IDAT'.b) idat.nil? || !data.byteslice(0, idat).include?('acTL'.b) end + def self.gif?(data) + GIF_SIGNATURES.any? { |signature| starts_with?(data, signature) } + end + + def self.starts_with?(data, signature) + data.byteslice(0, signature.bytesize)&.b == signature + end + def self.cache @cache ||= {} end diff --git a/test/lib/docs/filters/core/images_test.rb b/test/lib/docs/filters/core/images_test.rb index f7babdfe46..e1650923bc 100644 --- a/test/lib/docs/filters/core/images_test.rb +++ b/test/lib/docs/filters/core/images_test.rb @@ -26,6 +26,11 @@ def png_data end.to_blob end + # A 64x64 four-colour GIF89a; lossless WebP compresses it well below the GIF. + def gif_data + Base64.decode64('R0lGODlhQABAAPEAAB6Q///XANwUPC6LVywAAAAAQABAAAAC/4SPCcHtH4SctIqBs95D+QOFjUVO3Jl9nyiWJYqqHhu6JHzKCg3ZFs7RJXgPXwW4ESKIDiMFqVGCmAynCYqRGqhV6wXb0XIjXnBYOvZ+sdoFV21up8tg+ZvOFt+tcT0VXufHBJiHtufUZ/iHB2W3yBeoOMiI5DgJWag0h9koSETYKflJCWQ5ylnpyQOaKrpKimP6ilqqSsNa63oLCyO7Sxtry4IbrDvMGyPcgpyjXMMc5NwDnSRdRB1l3YSdoj3CneXdBdwrTkaebLyM3qz+zB7tPg1fLX9Nn22/jd+t/80fzt84RJE0HTKSyOAjgpmEbGIYSuEliK0knqKYy+IsjCvFNP7iWE7gOZDpPB4DeMbkOpLtVL5jGc/lPJj1ZN6jmc/mPpz9dP7jiaEAADs=') + end + # Splices an acTL chunk before IDAT to mimic an animated PNG. def apng_data data = png_data @@ -176,11 +181,29 @@ def stub_request(response) end end - context "with a non-PNG image" do + context "with a GIF image" do + it "converts it to WebP" do + @body = IMG_BODY + data = gif_data + stub_request make_response(body: data, mime_type: 'image/gif', content_length: data.bytesize) + src = filter_output.at_css('img')['src'] + assert src.start_with?('data:image/webp;base64,'), src[0, 40] + webp = image_from(src) + assert_equal 'RIFF', webp.byteslice(0, 4) + assert_equal 'WEBP', webp.byteslice(8, 4) + assert_operator webp.bytesize, :<, data.bytesize + end + + it "recognizes the GIF87a signature" do + assert Docs::ImagesFilter.convert_to_webp(gif_data.sub('GIF89a', 'GIF87a')) + end + end + + context "with an image we can't convert" do it "is left untouched" do @body = IMG_BODY - stub_request make_response(body: 'imgdata', mime_type: 'image/gif', content_length: 7) - expected = "data:image/gif;base64,#{Base64.strict_encode64('imgdata')}" + stub_request make_response(body: 'imgdata', mime_type: 'image/jpeg', content_length: 7) + expected = "data:image/jpeg;base64,#{Base64.strict_encode64('imgdata')}" assert_equal expected, filter_output.at_css('img')['src'] end end From f0e49f836cbba993999c9be0e3b84ea5e55eb206 Mon Sep 17 00:00:00 2001 From: Simon Legner Date: Sun, 13 Sep 2026 16:21:09 +0200 Subject: [PATCH 3/5] Convert scraped JPEG images to lossy WebP Re-encode JPEGs with `cwebp -q 80 -sharp_yuv`, which takes the JPEGs in the existing docs down to 73% of their size. Quality 80 is the knee of the size/SSIM curve measured over those images (mean SSIM 0.9935, worst 0.984); 85 gives up most of the savings for little quality. -sharp_yuv keeps text edges in screenshots and diagrams crisp. The pipeline was already lossy for PNGs via pngquant, and jpegoptim already strips EXIF, so dropping metadata here loses nothing new. Move the GIF test fixture to test/files/ next to the new JPEG one. --- docs/filter-reference.md | 2 +- docs/scraper-reference.md | 2 +- lib/docs/filters/core/images.rb | 21 ++++++++++---- test/files/image.gif | Bin 0 -> 338 bytes test/files/image.jpg | Bin 0 -> 1138 bytes test/lib/docs/filters/core/images_test.rb | 32 +++++++++++++++++----- 6 files changed, 42 insertions(+), 15 deletions(-) create mode 100644 test/files/image.gif create mode 100644 test/files/image.jpg diff --git a/docs/filter-reference.md b/docs/filter-reference.md index 4b3ed096fc..040814298a 100644 --- a/docs/filter-reference.md +++ b/docs/filter-reference.md @@ -82,7 +82,7 @@ The `call` method must return either `doc` or `html`, depending on the type of f * [`InnerHtmlFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/inner_html.rb) — converts the document to a string * [`CleanTextFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/clean_text.rb) — removes empty nodes * [`AttributionFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/attribution.rb) — appends the license info and link to the original document -* [`ImagesFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/images.rb) — downloads and inlines images as data URIs, optimizing them and re-encoding PNGs and GIFs as lossless WebP +* [`ImagesFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/images.rb) — downloads and inlines images as data URIs, optimizing them and re-encoding PNGs and GIFs as lossless WebP and JPEGs as lossy WebP * [`TitleFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/title.rb) — prepends the document with a title (disabled by default) * [`EntriesFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/entries.rb) — abstract filter for extracting the page's metadata diff --git a/docs/scraper-reference.md b/docs/scraper-reference.md index c976642694..7ddb9e093a 100644 --- a/docs/scraper-reference.md +++ b/docs/scraper-reference.md @@ -109,7 +109,7 @@ Default `html_filters`: Default `text_filters`: -* [`ImagesFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/images.rb) — downloads and inlines images as data URIs, optimizing them and re-encoding PNGs and GIFs as lossless WebP +* [`ImagesFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/images.rb) — downloads and inlines images as data URIs, optimizing them and re-encoding PNGs and GIFs as lossless WebP and JPEGs as lossy WebP * [`InnerHtmlFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/inner_html.rb) — converts the document to a string * [`CleanTextFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/clean_text.rb) — removes empty nodes * [`AttributionFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/attribution.rb) — appends the license info and link to the original document diff --git a/lib/docs/filters/core/images.rb b/lib/docs/filters/core/images.rb index 06afa31597..0b8ae38a47 100644 --- a/lib/docs/filters/core/images.rb +++ b/lib/docs/filters/core/images.rb @@ -11,7 +11,14 @@ class ImagesFilter < Filter PNG_SIGNATURE = "\x89PNG\r\n\x1a\n".b GIF_SIGNATURES = ['GIF87a'.b, 'GIF89a'.b].freeze - CWEBP_COMMAND = %w(cwebp -quiet -lossless -z 9 -m 6 -metadata none -o - -- -).freeze + JPEG_SIGNATURE = "\xff\xd8\xff".b + + # WebP q=80 is roughly equivalent to JPEG q=90, and -sharp_yuv keeps the + # edges of the screenshots and diagrams documentation is full of crisp. + JPEG_QUALITY = 80 + + CWEBP_LOSSLESS_COMMAND = %w(cwebp -quiet -lossless -z 9 -m 6 -metadata none -o - -- -).freeze + CWEBP_LOSSY_COMMAND = %W(cwebp -quiet -q #{JPEG_QUALITY} -m 6 -sharp_yuv -metadata none -o - -- -).freeze GIF2WEBP_COMMAND = %w(gif2webp -quiet -m 6 -metadata none -o - -- -).freeze def self.optimize_image_data(data) @@ -19,10 +26,10 @@ def self.optimize_image_data(data) @image_optim.optimize_image_data(data) end - # Losslessly re-encodes a PNG or GIF as WebP, which is usually smaller - # (10-50% for PNGs, considerably more for GIFs). Returns nil when the data - # isn't an image we can convert, when the encoder isn't available, or when - # the result would be bigger than the original. + # Re-encodes a PNG or GIF as lossless WebP and a JPEG as lossy WebP, all of + # which are usually smaller. Returns nil when the data isn't an image we can + # convert, when the encoder isn't available, or when the result would be + # bigger than the original. def self.convert_to_webp(data) command = webp_command(data) return unless command @@ -38,10 +45,12 @@ def self.convert_to_webp(data) def self.webp_command(data) if png?(data) - CWEBP_COMMAND + CWEBP_LOSSLESS_COMMAND elsif gif?(data) # unlike cwebp, gif2webp keeps every frame of an animation GIF2WEBP_COMMAND + elsif starts_with?(data, JPEG_SIGNATURE) + CWEBP_LOSSY_COMMAND end end diff --git a/test/files/image.gif b/test/files/image.gif new file mode 100644 index 0000000000000000000000000000000000000000..71146b2132fc736fd1e60890d2a0adcf1043e38a GIT binary patch literal 338 zcmV-Y0j>T=Nk%w1VL$*t0Pz3-9+3b4*8toUJT8k@EC2ui06+jh00092gpUcq?H`1k zw2FbV-b48Vg^fi{+?jo!D3&FPsvaA0 zijf+qRqF-Y(Q+eRa9BF_ezDpGS8Z;m%V7DN&KBRTaXl4Y=f?z?hqmX`XZVK)Ww*!| zs5r=oB$3{D!NLFWBQ6(s!GFJ>v{_dBfCo*Ys-Qg ztUJsbBiw?Vqg=xrFzGE5B0Edbqg~_>T+jP{@BG1=I_&W7d(V5$dCqg5#HYkp zVj%)EQ4mB)6h%pvV6s}KrcO~aOW~$rrlpp;X0(VvGbKrdU#do}S&RMipTsCJXh;KT zkc3H|+uo8}4PoQf{EUYuUMgByHJ4yO(3do$n%EA|BY-)|q( z_k`Y>+KWRyCb!>v)fPXWzOOTa_jbI^H0pbr_gk}z4q3DFe)_Unj@>_eq`Txw9o*Vo zQuA@hC*@<_*sVEuwfFa{4#bIO@zhE5 zHR>OQkR9rR;~ddVkGo#RELBRjLu&_IczQnrGsC`OL0mIbebY+UiyU-q*$MPmK#zM} z<7ZIEi8?-@@mazSN`^92SCgsTjd3xl*%l96vaQBPbw?l0^2S>ez}i&w9Hy;=IhBG2hFe+9eI%NY@>l?TZ`4m+%J?ye(7X+X(T85r& ze8F_at?JDEk<_x?^T7nw)!&p|2a+UgeNWZgOq)TojMf^i6DQw$=rQC5xl`3HDXOJ% bpyGc`)eG#(OyOl-hDyKS-5biyY~u4@riHj< literal 0 HcmV?d00001 diff --git a/test/lib/docs/filters/core/images_test.rb b/test/lib/docs/filters/core/images_test.rb index e1650923bc..acfda49e8c 100644 --- a/test/lib/docs/filters/core/images_test.rb +++ b/test/lib/docs/filters/core/images_test.rb @@ -26,9 +26,8 @@ def png_data end.to_blob end - # A 64x64 four-colour GIF89a; lossless WebP compresses it well below the GIF. - def gif_data - Base64.decode64('R0lGODlhQABAAPEAAB6Q///XANwUPC6LVywAAAAAQABAAAAC/4SPCcHtH4SctIqBs95D+QOFjUVO3Jl9nyiWJYqqHhu6JHzKCg3ZFs7RJXgPXwW4ESKIDiMFqVGCmAynCYqRGqhV6wXb0XIjXnBYOvZ+sdoFV21up8tg+ZvOFt+tcT0VXufHBJiHtufUZ/iHB2W3yBeoOMiI5DgJWag0h9koSETYKflJCWQ5ylnpyQOaKrpKimP6ilqqSsNa63oLCyO7Sxtry4IbrDvMGyPcgpyjXMMc5NwDnSRdRB1l3YSdoj3CneXdBdwrTkaebLyM3qz+zB7tPg1fLX9Nn22/jd+t/80fzt84RJE0HTKSyOAjgpmEbGIYSuEliK0knqKYy+IsjCvFNP7iWE7gOZDpPB4DeMbkOpLtVL5jGc/lPJj1ZN6jmc/mPpz9dP7jiaEAADs=') + def fixture(name) + File.binread(File.expand_path("../../../../files/#{name}", __dir__)) end # Splices an acTL chunk before IDAT to mimic an animated PNG. @@ -184,7 +183,7 @@ def stub_request(response) context "with a GIF image" do it "converts it to WebP" do @body = IMG_BODY - data = gif_data + data = fixture('image.gif') stub_request make_response(body: data, mime_type: 'image/gif', content_length: data.bytesize) src = filter_output.at_css('img')['src'] assert src.start_with?('data:image/webp;base64,'), src[0, 40] @@ -195,15 +194,34 @@ def stub_request(response) end it "recognizes the GIF87a signature" do - assert Docs::ImagesFilter.convert_to_webp(gif_data.sub('GIF89a', 'GIF87a')) + assert Docs::ImagesFilter.convert_to_webp(fixture('image.gif').sub('GIF89a', 'GIF87a')) + end + end + + context "with a JPEG image" do + it "converts it to WebP" do + @body = IMG_BODY + data = fixture('image.jpg') + stub_request make_response(body: data, mime_type: 'image/jpeg', content_length: data.bytesize) + src = filter_output.at_css('img')['src'] + assert src.start_with?('data:image/webp;base64,'), src[0, 40] + webp = image_from(src) + assert_equal 'RIFF', webp.byteslice(0, 4) + assert_equal 'WEBP', webp.byteslice(8, 4) + assert_operator webp.bytesize, :<, data.bytesize + end + + it "encodes it lossily" do + assert_equal Docs::ImagesFilter::CWEBP_LOSSY_COMMAND, + Docs::ImagesFilter.webp_command(fixture('image.jpg')) end end context "with an image we can't convert" do it "is left untouched" do @body = IMG_BODY - stub_request make_response(body: 'imgdata', mime_type: 'image/jpeg', content_length: 7) - expected = "data:image/jpeg;base64,#{Base64.strict_encode64('imgdata')}" + stub_request make_response(body: 'imgdata', mime_type: 'image/bmp', content_length: 7) + expected = "data:image/bmp;base64,#{Base64.strict_encode64('imgdata')}" assert_equal expected, filter_output.at_css('img')['src'] end end From 0bea3ae4e0fc48fc27e97c78a3307c7225c7a498 Mon Sep 17 00:00:00 2001 From: Simon Legner Date: Mon, 14 Sep 2026 13:25:58 +0200 Subject: [PATCH 4/5] Generate the icon spritesheets as WebP Pipe ChunkyPNG's output straight into `cwebp -lossless` instead of writing a PNG and running OptiPNG over it. No PNG is produced anymore and the two sheets go from 462 to 302 kilobytes. `--disable-optimization` now picks cwebp's fastest setting rather than skipping OptiPNG, keeping dev boots cheap. Sprockets 4 resolves `assets_compile` entries as paths, not globs, so the `*.png` entry never matched the sprites and they were left out of the manifest. List them explicitly. --- assets/javascripts/app/config.js.erb | 2 +- assets/stylesheets/global/_icons.scss.erb | 4 +- lib/app.rb | 6 +-- lib/tasks/sprites.thor | 50 +++++++++-------------- 4 files changed, 25 insertions(+), 37 deletions(-) diff --git a/assets/javascripts/app/config.js.erb b/assets/javascripts/app/config.js.erb index f8c8ee5598..a939a9a241 100644 --- a/assets/javascripts/app/config.js.erb +++ b/assets/javascripts/app/config.js.erb @@ -16,7 +16,7 @@ export const config = { version: <%= Time.now.to_i %>, release: <%= Time.now.utc.httpdate.to_json %>, mathml_stylesheet: '/mathml.css', - favicon_spritesheet: '<%= image_path('sprites/docs.png') %>', + favicon_spritesheet: '<%= image_path('sprites/docs.webp') %>', service_worker_path: '/service-worker.js', service_worker_enabled: <%= App.environment == :production || ENV['ENABLE_SERVICE_WORKER'] == 'true' %>, } diff --git a/assets/stylesheets/global/_icons.scss.erb b/assets/stylesheets/global/_icons.scss.erb index b2b22c22f1..d80f353df2 100644 --- a/assets/stylesheets/global/_icons.scss.erb +++ b/assets/stylesheets/global/_icons.scss.erb @@ -14,12 +14,12 @@ display: block; width: 1rem; height: 1rem; - background-image: image-url('sprites/docs.png'); + background-image: image-url('sprites/docs.webp'); background-size: <%= manifest['icons_per_row'] %>rem <%= manifest['icons_per_row'] %>rem; } @media (-webkit-min-device-pixel-ratio: 1.5), (min-resolution: 144dpi) { - %doc-icon { background-image: image-url('sprites/docs@2x.png'); } + %doc-icon { background-image: image-url('sprites/docs@2x.webp'); } } html._theme-dark { diff --git a/lib/app.rb b/lib/app.rb index 3ec6066496..3c74344136 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -56,7 +56,7 @@ class App < Sinatra::Application .sort .freeze - set :assets_compile, %w(*.png docs.json vendor.js unsupported.js application.css application-dark.css) + js_modules + set :assets_compile, %w(sprites/docs.webp sprites/docs@2x.webp docs.json vendor.js unsupported.js application.css application-dark.css) + js_modules require 'json' set :docs_prefix, 'docs' @@ -267,8 +267,8 @@ def service_worker_asset_urls javascript_path('vendor'), javascript_path('unsupported'), stylesheet_path('application'), - image_path('sprites/docs.png'), - image_path('sprites/docs@2x.png'), + image_path('sprites/docs.webp'), + image_path('sprites/docs@2x.webp'), ].compact end diff --git a/lib/tasks/sprites.thor b/lib/tasks/sprites.thor index d12d314585..0282f71a50 100644 --- a/lib/tasks/sprites.thor +++ b/lib/tasks/sprites.thor @@ -7,14 +7,18 @@ class SpritesCLI < Thor require 'docs' require 'chunky_png' require 'fileutils' - require 'image_optim' require 'terminal-table' super end + # cwebp re-encodes the pixels from scratch, so the PNG ChunkyPNG hands it is + # only a carrier: it goes in through a pipe at the cheapest compression and + # never reaches the disk. + CWEBP_COMMAND = %w(cwebp -quiet -lossless -metadata none).freeze + desc 'generate [--remove-public-icons] [--disable-optimization] [--verbose]', 'Generate the documentation icon spritesheets' option :remove_public_icons, type: :boolean, desc: 'Remove public/icons after generating the spritesheets' - option :disable_optimization, type: :boolean, desc: 'Disable optimizing the spritesheets with OptiPNG' + option :disable_optimization, type: :boolean, desc: "Encode the spritesheets at cwebp's fastest setting instead of its smallest" option :verbose, type: :boolean def generate items = get_items @@ -41,11 +45,6 @@ class SpritesCLI < Thor generate_spritesheet(16, items_with_icons) {|item| item[:icon_16]} generate_spritesheet(32, items_with_icons) {|item| item[:icon_32]} - unless options[:disable_optimization] - optimize_spritesheet(get_output_path(16)) - optimize_spritesheet(get_output_path(32)) - end - # Add Mongoose's icon details to docs without custom icons default_item = items_with_icons.find {|item| item[:type] == 'mongoose'} items_without_icons.each do |item| @@ -173,12 +172,20 @@ class SpritesCLI < Thor end FileUtils.mkdir_p(File.dirname(output_path)) - spritesheet.save(output_path) + save_spritesheet(spritesheet, output_path) end - def optimize_spritesheet(path) - logger.info("Optimizing spritesheet at #{path}") - image_optim.optimize_image!(path) + def save_spritesheet(spritesheet, path) + # -z 9 is the smallest (and slowest) lossless setting, -z 0 the fastest + command = [*CWEBP_COMMAND, '-z', options[:disable_optimization] ? '0' : '9', '-o', path, '--', '-'] + + begin + IO.popen(command, 'wb') {|io| io.write(spritesheet.to_blob(:fast_rgba))} + rescue SystemCallError + raise "Generating the spritesheets requires cwebp, install libwebp (Debian/Ubuntu: webp, Alpine: libwebp-tools)" + end + + raise "cwebp failed to write #{path}" unless $?.success? end def save_manifest(items, icons_per_row, path) @@ -213,7 +220,7 @@ class SpritesCLI < Thor end def get_output_path(size) - "assets/images/sprites/docs#{size == 32 ? '@2x' : ''}.png" + "assets/images/sprites/docs#{size == 32 ? '@2x' : ''}.webp" end def compile_scss_erb @@ -228,25 +235,6 @@ class SpritesCLI < Thor end end - def image_optim - @image_optim ||= ImageOptim.new( - :config_paths => [], - :advpng => false, - :gifsicle => false, - :jhead => false, - :jpegoptim => false, - :jpegrecompress => false, - :jpegtran => false, - :pngcrush => false, - :pngout => false, - :pngquant => false, - :svgo => false, - :optipng => { - :level => 7, - }, - ) - end - def logger @logger ||= Logger.new($stdout).tap do |logger| logger.level = options[:verbose] ? Logger::DEBUG : Logger::INFO From 12244b0e89368b814d3ac96a43af158d3360c80b Mon Sep 17 00:00:00 2001 From: Simon Legner Date: Mon, 14 Sep 2026 13:28:18 +0200 Subject: [PATCH 5/5] Let `thor sprites:generate` run on its own The task compiles every scss.erb template, and _environment.scss.erb read the `App` constant, which only exists once the app is loaded: invoking the task directly died with a NameError. Pass the templates a binding carrying `environment`, resolved from APP_ENV and RACK_ENV the way Sinatra resolves it. --- assets/stylesheets/components/_environment.scss.erb | 2 +- lib/tasks/sprites.thor | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/assets/stylesheets/components/_environment.scss.erb b/assets/stylesheets/components/_environment.scss.erb index 5e2201c30c..74dd486659 100644 --- a/assets/stylesheets/components/_environment.scss.erb +++ b/assets/stylesheets/components/_environment.scss.erb @@ -1,3 +1,3 @@ ._hide-in-development { - <%= App.environment != :production ? 'display: none;' : '' %> + <%= environment != :production ? 'display: none;' : '' %> } diff --git a/lib/tasks/sprites.thor b/lib/tasks/sprites.thor index 0282f71a50..df415282be 100644 --- a/lib/tasks/sprites.thor +++ b/lib/tasks/sprites.thor @@ -229,12 +229,19 @@ class SpritesCLI < Thor scss_erb_files.each do |erb_path| scss_path = erb_path.gsub('.erb', '') File.open(scss_path, 'w') do |f| - f.write(ERB.new(File.read(erb_path)).result) + f.write(ERB.new(File.read(erb_path)).result(erb_binding)) logger.info("Compiling #{erb_path} to #{scss_path}") end end end + # The templates are compiled without the app loaded, so whatever they need + # has to come from here rather than from `App`. + def erb_binding + environment = (ENV['APP_ENV'] || ENV['RACK_ENV'] || 'development').to_sym + binding + end + def logger @logger ||= Logger.new($stdout).tap do |logger| logger.level = options[:verbose] ? Logger::DEBUG : Logger::INFO