Fix Mastodon runtime drift and Ferrum fuzz disk usage

test/bench/docker_overlay_test.rb

@@ -2,12 +2,45 @@
 
 class DockerOverlayTest < Minitest::Test
   def test_mastodon_plus_copies_the_gemfile_ruby_version_before_bundle_install
-    dockerfile = File.read(File.join(Bench::ROOT, "tools/docker/mastodon/overlay/mastodon-ccache.Dockerfile"))
-    before_install = dockerfile.lines.take_while { !it.include?("bundle install") }
-    gemfile_copy = before_install.find { it.start_with?("COPY ") && it.include?("Gemfile") }
-    sources = Shellwords.split(gemfile_copy).drop(1)[...-1]
+    prepare_mastodon(mastodon_dockerfile) do |dockerfile|
+      before_install = dockerfile.lines.take_while { !it.include?("bundle install") }
+      gemfile_copy = before_install.find { it.start_with?("COPY ") && it.include?("Gemfile") }
+      sources = Shellwords.split(gemfile_copy).drop(1)[...-1]
 
-    assert sources.any? { File.fnmatch?(it, ".ruby-version", File::FNM_DOTMATCH) },
-      "the Gemfile reads .ruby-version before the full source is copied"
+      assert sources.any? { File.fnmatch?(it, ".ruby-version", File::FNM_DOTMATCH) },
+        "the Gemfile reads .ruby-version before the full source is copied"
+    end
   end
+
+  def test_mastodon_plus_preserves_upstream_runtime_and_dependency_updates
+    upstream = mastodon_dockerfile.sub('RUBY_VERSION="4.0.7"', 'RUBY_VERSION="4.0.8"')
+      .sub('NODE_MAJOR_VERSION="24"', 'NODE_MAJOR_VERSION="26"')
+      .sub('VIPS_VERSION=8.18.6', 'VIPS_VERSION=8.18.7')
+      .sub("COPY Gemfile*", "COPY .ruby-version Gemfile*")
+
+    prepare_mastodon(upstream) do |dockerfile|
+      %w[RUBY_VERSION NODE_MAJOR_VERSION VIPS_VERSION].each do |name|
+        assert_equal upstream.lines.grep(/^ARG #{name}=/), dockerfile.lines.grep(/^ARG #{name}=/)
+      end
+      assert_includes dockerfile, 'ENV CC="ccache cc" CXX="ccache c++"'
+      assert_includes dockerfile, "--mount=type=cache,id=mastodon-ccache"
+      assert_equal 1, dockerfile.scan("COPY .ruby-version Gemfile*").size
+    end
+  end
+
+  private
+    def prepare_mastodon(dockerfile)
+      Dir.mktmpdir("mastodon-overlay") do |dir|
+        FileUtils.mkdir_p(File.join(dir, "upstream"))
+        File.write(File.join(dir, "upstream/Dockerfile"), dockerfile)
+        FileUtils.cp_r(File.join(Bench::ROOT, "tools/docker/mastodon/overlay"), dir)
+        system("bash", File.join(Bench::ROOT, "tools/docker/mastodon/prepare.sh"), chdir: dir, exception: true)
+        yield File.read(File.join(dir, "overlay/mastodon-ccache.Dockerfile"))
+        assert_equal dockerfile, File.read(File.join(dir, "upstream/Dockerfile"))
+      end
+    end
+
+    def mastodon_dockerfile
+      File.read(File.join(Bench::ROOT, "test/bench/fixtures/mastodon.Dockerfile"))
+    end
 end

tools/docker/mastodon/overlay/mastodon-ccache.Dockerfile → test/bench/fixtures/mastodon.Dockerfile

@@ -158,9 +158,6 @@ RUN \
 FROM ${BASE_REGISTRY}/ruby:${RUBY_VERSION}-slim-${DEBIAN_VERSION} AS media-build
 
 ARG TARGETPLATFORM
-ARG CCACHE_VERSION=4.13.6
-ARG CCACHE_STORAGE_HTTP_VERSION=0.8
-ENV CC="ccache cc" CXX="ccache c++" CCACHE_COMPILERCHECK=content
 
 # Set default shell used for running commands
 SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-c"]
@@ -177,8 +174,6 @@ RUN \
   autoconf \
   automake \
   build-essential \
-  ca-certificates \
-  curl \
   libtool \
   meson \
   nasm \
@@ -210,29 +205,6 @@ RUN \
   libx265-dev \
   ;
 
-RUN set -eux; \
-  arch="$(dpkg --print-architecture)"; \
-  case "$arch" in \
-    amd64) ccache_arch=x86_64; helper_arch=amd64; ccache_sha=567b1b648411819590f918f045218c92da14418bdec3b30db94a3b4f5d77cf13; helper_sha=2c2cfafa39f5a4628201ccc11c81829197519159aa128fe00ea251f1f4f2461c ;; \
-    arm64) ccache_arch=aarch64; helper_arch=arm64; ccache_sha=fae67fb810e1f0d390409af6603355483572229e19183e68574cd0f851a6fb98; helper_sha=49587fb0534f5c6265fd1008267af795885f8297c6c51213708da74e4de9d475 ;; \
-    *) echo "unsupported ccache architecture: ${arch}" >&2; exit 1 ;; \
-  esac; \
-  ccache_archive="ccache-${CCACHE_VERSION}-linux-${ccache_arch}-glibc.tar.gz"; \
-  helper_archive="ccache-storage-http-go-${CCACHE_STORAGE_HTTP_VERSION}-linux-${helper_arch}.tar.gz"; \
-  curl -fsSL --retry 5 "https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/${ccache_archive}" -o "/tmp/${ccache_archive}"; \
-  echo "${ccache_sha}  /tmp/${ccache_archive}" | sha256sum --check; \
-  curl -fsSL --retry 5 "https://github.com/ccache/ccache-storage-http-go/releases/download/v${CCACHE_STORAGE_HTTP_VERSION}/${helper_archive}" -o "/tmp/${helper_archive}"; \
-  echo "${helper_sha}  /tmp/${helper_archive}" | sha256sum --check; \
-  tar xzf "/tmp/${ccache_archive}" -C /tmp; \
-  tar xzf "/tmp/${helper_archive}" -C /tmp; \
-  install -m 0755 "/tmp/ccache-${CCACHE_VERSION}-linux-${ccache_arch}-glibc/ccache" /usr/local/bin/ccache; \
-  install -m 0755 "/tmp/ccache-storage-http-go-${CCACHE_STORAGE_HTTP_VERSION}-linux-${helper_arch}/ccache-storage-http" /usr/local/bin/ccache-storage-http; \
-  ccache --version; \
-  helper_version="$(ccache-storage-http --version 2>&1 || true)"; \
-  printf "%s\n" "$helper_version"; \
-  printf "%s\n" "$helper_version" | grep -F "Version: ${CCACHE_STORAGE_HTTP_VERSION}"; \
-  rm -rf /tmp/ccache-*
-
 # Create temporary libvips specific build layer
 FROM media-build AS libvips
 
@@ -255,7 +227,7 @@ RUN meson setup build --prefix /usr/local/libvips --libdir=lib -Ddeprecated=fals
 WORKDIR /usr/local/libvips/src/vips-${VIPS_VERSION}/build
 
 # Compile and install libvips
-RUN --mount=type=cache,id=mastodon-ccache,target=/root/.cache/ccache ninja && ninja install && ccache --show-stats
+RUN ninja && ninja install
 
 # Create temporary ffmpeg specific build layer
 FROM media-build AS ffmpeg
@@ -274,16 +246,8 @@ RUN tar xf n${FFMPEG_VERSION}.tar.gz && mv FFmpeg-n${FFMPEG_VERSION} ffmpeg-${FF
 WORKDIR /usr/local/ffmpeg/src/ffmpeg-${FFMPEG_VERSION}
 
 # Configure and compile ffmpeg
-RUN --mount=type=cache,id=mastodon-ccache,target=/root/.cache/ccache \
-  configure_compiler_args=(); \
-  if [ -n "${CC:-}" ]; then \
-    configure_compiler_args+=(--cc="$CC"); \
-  fi; \
-  if [ -n "${CXX:-}" ]; then \
-    configure_compiler_args+=(--cxx="$CXX"); \
-  fi; \
+RUN \
   ./configure \
-  "${configure_compiler_args[@]}" \
   --prefix=/usr/local/ffmpeg \
   --toolchain=hardened \
   --disable-debug \
@@ -308,17 +272,11 @@ RUN --mount=type=cache,id=mastodon-ccache,target=/root/.cache/ccache \
   --enable-version3 \
   ; \
   make -j"$(nproc)"; \
-  make install; \
-  ccache --show-stats
+  make install;
 
 # Create temporary build layer from base image for Ruby dependencies
 FROM ruby AS ruby-build
 
-# Use ccache for native extensions while keeping compiler selection in this Dockerfile.
-COPY --from=media-build /usr/local/bin/ccache /usr/local/bin/ccache
-COPY --from=media-build /usr/local/bin/ccache-storage-http /usr/local/bin/ccache-storage-http
-ENV CC="ccache cc" CXX="ccache c++" CCACHE_COMPILERCHECK=content
-
 ARG TARGETPLATFORM
 
 # hadolint ignore=DL3008
@@ -327,7 +285,6 @@ RUN \
   --mount=type=cache,id=apt-cache-${TARGETPLATFORM},target=/var/cache/apt,sharing=locked \
   --mount=type=cache,id=apt-lib-${TARGETPLATFORM},target=/var/lib/apt,sharing=locked \
   # Install build tools and bundler dependencies from APT
-  apt-get update; \
   apt-get install -y --no-install-recommends \
   build-essential \
   git \
@@ -348,7 +305,7 @@ FROM ruby-build AS bundler
 ARG TARGETPLATFORM
 
 # Copy Gemfile config into working directory
-COPY .ruby-version Gemfile* /opt/mastodon/
+COPY Gemfile* /opt/mastodon/
 
 # Copy libvips for gems that need it during install
 COPY --from=libvips /usr/local/libvips/lib /usr/local/lib
@@ -368,8 +325,7 @@ RUN \
   # Configure bundle to not warn about root user
   bundle config set silence_root_warning "true"; \
   # Download and install required Gems
-  bundle install -j"$(nproc)"; \
-  ccache --show-stats
+  bundle install -j"$(nproc)";
 
 # Create temporary assets build layer from build layer
 FROM ruby-build AS precompiler

test/bench/fuzz_smoke_test.rb

@@ -0,0 +1,58 @@
+require_relative "test_helper"
+
+class FuzzSmokeTest < Minitest::Test
+  TARGETS = %w[traceparent config_decode proxy_protocol mesh_udp_frame k8s_crd plugin_config ocsp_response jwt_svid tls_material datagram_client_address].freeze
+
+  def test_verified_executables_are_removed_before_the_next_build
+    run_smoke do |dir, status|
+      assert status.success?
+      assert_equal TARGETS, File.readlines(File.join(dir, "fuzz-verified"), chomp: true)
+      assert_empty Dir.glob(File.join(dir, "target/x86_64-unknown-linux-gnu/release/deps/traceparent-*"))
+      assert File.exist?(File.join(dir, "target/x86_64-unknown-linux-gnu/release/deps/libshared.rlib"))
+    end
+  end
+
+  def test_failure_does_not_verify_or_delete_the_failed_target
+    run_smoke("FAIL_TARGET" => "config_decode") do |dir, status|
+      refute status.success?
+      assert_equal ["traceparent"], File.readlines(File.join(dir, "fuzz-verified"), chomp: true)
+      assert File.executable?(File.join(dir, "target/x86_64-unknown-linux-gnu/release/config_decode"))
+    end
+  end
+
+  def test_missing_executable_is_not_verified
+    run_smoke("MISSING_TARGET" => "traceparent") do |dir, status|
+      refute status.success?
+      refute File.exist?(File.join(dir, "fuzz-verified"))
+    end
+  end
+
+  private
+    def run_smoke(extra_env = {})
+      Dir.mktmpdir("fuzz-smoke") do |dir|
+        FileUtils.mkdir_p(File.join(dir, "bin"))
+        cargo = File.join(dir, "bin/cargo")
+        File.write(cargo, <<~BASH)
+          #!/usr/bin/env bash
+          set -euo pipefail
+          [ "$1" = test ] && exit 0
+          target=$5
+          release="$CARGO_TARGET_DIR/x86_64-unknown-linux-gnu/release"
+          test ! -d "$release/build/ferrum-edge-fuzz"
+          mkdir -p "$release/build/ferrum-edge-fuzz/unit/out" "$release/deps"
+          touch "$release/deps/libshared.rlib" "$release/deps/$target-old"
+          if [ "$target" != "${MISSING_TARGET:-}" ]; then
+            touch "$release/$target"
+            chmod +x "$release/$target"
+          fi
+          [ "$target" != "${FAIL_TARGET:-}" ]
+        BASH
+        File.chmod(0o755, cargo)
+        File.write(File.join(dir, "fuzz-verified"), "stale\n")
+        env = { "PATH" => "#{dir}/bin:#{ENV.fetch('PATH')}", "BENCH_DIR" => dir,
+          "CARGO_TARGET_DIR" => File.join(dir, "target"), "BENCH_BORINGCACHE" => nil, **extra_env }
+        _output, status = Open3.capture2e(env, "bash", File.join(Bench::ROOT, "tools/cargo/ferrum-fuzz/overlay/fuzz-smoke.sh"))
+        yield dir, status
+      end
+    end
+end

tools/README.md

@@ -116,6 +116,14 @@ A `mise.toml` beside `case.toml` pins the case toolchain; it is copied into the
 
 Each phase runs in `<work>/<tool>/<case>/` (`.work` locally, `$HOME/w` on Actions, or `BENCH_WORK`), rebuilt from scratch for every phase at the same short path, as an upstream checkout would be, so tools that key caches or names on absolute paths behave the same: the plans with run-scoped tags, `overlay/`, and the upstream checkout in `upstream/`. `prepare` and `check` run there with `bash -c`, outside the timer, so their paths start at the run directory (`upstream/...`), not at the plan directory.
 
+Mastodon plus applies its compiler-cache patch to the checked-out upstream
+Dockerfile during preparation, preserving that commit's runtime and library
+versions. Ferrum verifies each fuzz executable after its smoke run, records the
+completed target in `fuzz-verified`, then removes that executable and its package
+objects before the next target. These files were already excluded from every
+provider's cache; the shared dependency artifacts remain. Failed targets stop
+the script before verification or cleanup.
+
 ## Adding
 
 - A case: copy an existing case directory, change `case.toml` and `.boringcache.toml`, run `bin/bench check`.

tools/cargo/ferrum-fuzz/case.toml

@@ -36,7 +36,7 @@ prepare = [
   "CARGO_HOME=\"${RUNNER_TEMP:-/tmp}/cargo-fuzz-home\" CARGO_TARGET_DIR=\"${RUNNER_TEMP:-/tmp}/cargo-fuzz-target\" RUSTC_WRAPPER= cargo install cargo-fuzz@0.13.1 --locked --root \"$HOME/.cargo\"",
   "cp .boringcache.toml upstream/fuzz/.boringcache.toml && echo /fuzz/.boringcache.toml >> upstream/.git/info/exclude",
 ]
-check = "for target in traceparent config_decode proxy_protocol mesh_udp_frame k8s_crd plugin_config ocsp_response jwt_svid tls_material datagram_client_address; do test -x upstream/target/x86_64-unknown-linux-gnu/release/$target || exit 1; done && test -z \"$(git -C upstream status --porcelain=v1 --untracked-files=all)\""
+check = "for target in traceparent config_decode proxy_protocol mesh_udp_frame k8s_crd plugin_config ocsp_response jwt_svid tls_material datagram_client_address; do grep -Fxq \"$target\" fuzz-verified || exit 1; done && test -z \"$(git -C upstream status --porcelain=v1 --untracked-files=all)\""
 
 [env]
 CARGO_TERM_COLOR = "always"

tools/cargo/ferrum-fuzz/overlay/fuzz-smoke.sh

@@ -1,5 +1,9 @@
 set -euo pipefail
 
+verified="${BENCH_DIR:?}/fuzz-verified"
+release="${CARGO_TARGET_DIR:?}/x86_64-unknown-linux-gnu/release"
+rm -f "$verified"
+
 cargo_step() {
   local restore=$1 save=$2
   shift 2
@@ -13,11 +17,20 @@ cargo_step() {
   $BENCH_BORINGCACHE "${flags[@]}" "$@"
 }
 
+run_fuzz() {
+  local target=$1 max_len=$2 save=$3
+  cargo_step no-restore "$save" fuzz run --codegen-units 16 "$target" -- -runs=512 -max_total_time=8 -max_len="$max_len" -timeout=2 -rss_limit_mb=1024
+  test -x "$release/$target"
+  printf '%s\n' "$target" >> "$verified"
+  rm -f -- "$release/$target" "$release/deps/$target-"*
+  rm -rf -- "$release/build/ferrum-edge-fuzz"
+}
+
 CARGO_BUILD_JOBS=2 cargo_step restore no-save test --locked
 for target in traceparent config_decode proxy_protocol mesh_udp_frame k8s_crd plugin_config; do
-  cargo_step no-restore no-save fuzz run --codegen-units 16 "$target" -- -runs=512 -max_total_time=8 -max_len=4096 -timeout=2 -rss_limit_mb=1024
+  run_fuzz "$target" 4096 no-save
 done
 for target in ocsp_response jwt_svid tls_material; do
-  cargo_step no-restore no-save fuzz run --codegen-units 16 "$target" -- -runs=512 -max_total_time=8 -max_len=65536 -timeout=2 -rss_limit_mb=1024
+  run_fuzz "$target" 65536 no-save
 done
-cargo_step no-restore save fuzz run --codegen-units 16 datagram_client_address -- -runs=512 -max_total_time=8 -max_len=65536 -timeout=2 -rss_limit_mb=1024
+run_fuzz datagram_client_address 65536 save

tools/docker/mastodon/case.toml

@@ -1,6 +1,7 @@
 repo = "mastodon/mastodon"
 branch = "main"
 start_sha = "9b97c67be3c67e992c184167bd7425502aad12da"
+prepare = ["bash \"$BENCH_ROOT/tools/docker/mastodon/prepare.sh\""]
 check = "docker image inspect mastodon-server-benchmark:local"
 
 [runs]