Misc improvements for the bug fix Docker image

.taskcluster.yml

@@ -32,8 +32,24 @@ tasks:
       then: ${event.pull_request.body}
 
     taskboot_image: "mozilla/taskboot:0.5.2"
+
+    # The files a push adds, changes or removes.
+    changed_paths:
+      $if: 'tasks_for == "github-push"'
+      then:
+        $flattenDeep:
+          $map: { $eval: "event.commits" }
+          each(c):
+            - { $eval: "c.added" }
+            - { $eval: "c.modified" }
+            - { $eval: "c.removed" }
+      else: []
   in:
     $let:
+      # Whether a push changes the bug-fix agent image or what goes into it.
+      touches_bug_fix_image:
+        $eval: '" agents/bug-fix/" in (" " + join(changed_paths, " ")) || " libs/agent-tools/" in (" " + join(changed_paths, " ")) || " libs/hackbot-runtime/" in (" " + join(changed_paths, " ")) || " libs/phabricator-client/" in (" " + join(changed_paths, " ")) || " libs/phabricator-proxy/" in (" " + join(changed_paths, " ")) || "uv.lock" in changed_paths || "pyproject.toml" in changed_paths || ".taskcluster.yml" in changed_paths'
+
       default_task_definition:
         created: { $fromNow: "" }
         deadline: { $fromNow: "1 hour" }
@@ -287,6 +303,179 @@ tasks:
                 name: bugbug hackbot-ui tests
                 description: bugbug hackbot-ui tests
 
+        # Agent images use BuildKit features and a non-final target, which taskboot's
+        # build commands don't support, so call podman (what taskboot uses) directly.
+        # Its storage is a cache, so unchanged layers (system packages, mostly) are
+        # reused; layers left over from older builds are pruned after a week.
+        - $mergeDeep:
+            - { $eval: default_task_definition }
+            - taskId: { $eval: as_slugid("bug_fix_image_build") }
+              workerType: ci
+              payload:
+                capabilities:
+                  privileged: true
+                image: "${taskboot_image}"
+                cache:
+                  bugbug-bug-fix-image-podman: /var/lib/containers
+                command:
+                  - "/bin/sh"
+                  - "-cxe"
+                  - "git clone --quiet ${repository} /code &&
+                    cd /code &&
+                    git -c advice.detachedHead=false checkout ${head_rev} &&
+                    podman build --target agent -f agents/bug-fix/Dockerfile -t mozilla/bugbug-hackbot-agent-bug-fix:latest . &&
+                    mkdir /images &&
+                    podman save --format docker-archive mozilla/bugbug-hackbot-agent-bug-fix:latest | zstd -q -o /images/bugbug-hackbot-agent-bug-fix.tar.zst &&
+                    podman image prune --force --filter until=168h"
+                artifacts:
+                  public/bug-fix-agent:
+                    expires: { $fromNow: "2 weeks" }
+                    path: /images
+                    type: directory
+              scopes:
+                - docker-worker:capability:privileged
+                - docker-worker:cache:bugbug-bug-fix-image-podman
+                - generic-worker:cache:bugbug-bug-fix-image-podman
+              metadata:
+                name: bugbug bug-fix agent image build
+                description: bugbug bug-fix agent image build
+
+        # Runs inside the image, where the entrypoint has started the display and
+        # audio services. The tests download Firefox, minidump-stackwalk and Python
+        # packages, so failures are retried once to ride out network hiccups.
+        # TASKCLUSTER_ROOT_URL is unset as it is where the agent runs: mach would
+        # otherwise look Firefox's toolchains up on this cluster.
+        - $mergeDeep:
+            - { $eval: default_task_definition }
+            - taskId: { $eval: as_slugid("bug_fix_image_tests") }
+              dependencies:
+                - { $eval: as_slugid("bug_fix_image_build") }
+              payload:
+                image:
+                  type: task-image
+                  path: public/bug-fix-agent/bugbug-hackbot-agent-bug-fix.tar.zst
+                  taskId: { $eval: as_slugid("bug_fix_image_build") }
+                command:
+                  - "/bin/bash"
+                  - "-cx"
+                  - "git clone --quiet ${repository} /tmp/bugbug &&
+                    cd /tmp/bugbug &&
+                    git -c advice.detachedHead=false checkout ${head_rev} &&
+                    /usr/local/bin/python3 -m pip install --quiet --disable-pip-version-check --target /tmp/pydeps pytest==9.1.0 pytest-asyncio==1.4.0 &&
+                    cd agents/bug-fix &&
+                    unset TASKCLUSTER_ROOT_URL &&
+                    export PYTHONPATH=/tmp/pydeps BUG_FIX_IMAGE_INSIDE=1 &&
+                    (python -m pytest -v -rs tests/test_image.py -k 'not artifact_build and not android' ||
+                    python -m pytest -v -rs --last-failed tests/test_image.py -k 'not artifact_build and not android')"
+              metadata:
+                name: bugbug bug-fix agent image tests
+                description: bugbug bug-fix agent image tests
+
+        # Bootstraps a Firefox checkout, makes an artifact build and runs tests in it.
+        # Too slow for every pull request, it runs after merges that change the
+        # image or what goes into it.
+        - $if: 'tasks_for == "github-push" && head_branch == "refs/heads/master" && touches_bug_fix_image'
+          then:
+            $mergeDeep:
+              - { $eval: default_task_definition }
+              - taskId: { $eval: as_slugid("bug_fix_image_slow_tests") }
+                dependencies:
+                  - { $eval: as_slugid("bug_fix_image_build") }
+                workerType: compute-large
+                payload:
+                  image:
+                    type: task-image
+                    path: public/bug-fix-agent/bugbug-hackbot-agent-bug-fix.tar.zst
+                    taskId: { $eval: as_slugid("bug_fix_image_build") }
+                  command:
+                    - "/bin/bash"
+                    - "-cx"
+                    - "git clone --quiet ${repository} /tmp/bugbug &&
+                      cd /tmp/bugbug &&
+                      git -c advice.detachedHead=false checkout ${head_rev} &&
+                      /usr/local/bin/python3 -m pip install --quiet --disable-pip-version-check --target /tmp/pydeps pytest==9.1.0 pytest-asyncio==1.4.0 &&
+                      cd agents/bug-fix &&
+                      unset TASKCLUSTER_ROOT_URL &&
+                      export PYTHONPATH=/tmp/pydeps BUG_FIX_IMAGE_INSIDE=1 &&
+                      (python -m pytest -s -v -rs tests/test_image.py -k artifact_build ||
+                      python -m pytest -s -v -rs --last-failed tests/test_image.py -k artifact_build)"
+                metadata:
+                  name: bugbug bug-fix agent image slow tests
+                  description: bugbug bug-fix agent image slow tests
+
+        # Like the slow tests, but a full build with the agent's own mozconfig, as
+        # the agent makes. Too slow for every pull request, it runs after merges
+        # that change the image or what goes into it; a failure is not retried.
+        - $if: 'tasks_for == "github-push" && head_branch == "refs/heads/master" && touches_bug_fix_image'
+          then:
+            $mergeDeep:
+              - { $eval: default_task_definition }
+              - taskId: { $eval: as_slugid("bug_fix_image_full_build_tests") }
+                dependencies:
+                  - { $eval: as_slugid("bug_fix_image_build") }
+                deadline: { $fromNow: "8 hours" }
+                # Its objdir alone takes over 30 GiB, more than other pools' disks hold.
+                workerType: firefox-build
+                payload:
+                  maxRunTime: 21600
+                  env:
+                    BUG_FIX_IMAGE_FULL_BUILD: "1"
+                  image:
+                    type: task-image
+                    path: public/bug-fix-agent/bugbug-hackbot-agent-bug-fix.tar.zst
+                    taskId: { $eval: as_slugid("bug_fix_image_build") }
+                  command:
+                    - "/bin/bash"
+                    - "-cx"
+                    - "git clone --quiet ${repository} /tmp/bugbug &&
+                      cd /tmp/bugbug &&
+                      git -c advice.detachedHead=false checkout ${head_rev} &&
+                      /usr/local/bin/python3 -m pip install --quiet --disable-pip-version-check --target /tmp/pydeps pytest==9.1.0 pytest-asyncio==1.4.0 &&
+                      cd agents/bug-fix &&
+                      unset TASKCLUSTER_ROOT_URL &&
+                      export PYTHONPATH=/tmp/pydeps BUG_FIX_IMAGE_INSIDE=1 &&
+                      python -m pytest -s -v -rs tests/test_image.py -k full_build"
+                metadata:
+                  name: bugbug bug-fix agent image full build tests
+                  description: bugbug bug-fix agent image full build tests
+
+        # Bootstraps a Firefox checkout for Android, makes an artifact build and
+        # runs tests in it.
+        # Too slow for every pull request, it runs after merges that change the
+        # image or what goes into it.
+        - $if: 'tasks_for == "github-push" && head_branch == "refs/heads/master" && touches_bug_fix_image'
+          then:
+            $mergeDeep:
+              - { $eval: default_task_definition }
+              - taskId: { $eval: as_slugid("bug_fix_image_android_tests") }
+                dependencies:
+                  - { $eval: as_slugid("bug_fix_image_build") }
+                deadline: { $fromNow: "3 hours" }
+                # The Android SDK and NDK take ~14 GiB, which leaves other pools' disks
+                # nearly full.
+                workerType: firefox-build
+                payload:
+                  maxRunTime: 7200
+                  image:
+                    type: task-image
+                    path: public/bug-fix-agent/bugbug-hackbot-agent-bug-fix.tar.zst
+                    taskId: { $eval: as_slugid("bug_fix_image_build") }
+                  command:
+                    - "/bin/bash"
+                    - "-cx"
+                    - "git clone --quiet ${repository} /tmp/bugbug &&
+                      cd /tmp/bugbug &&
+                      git -c advice.detachedHead=false checkout ${head_rev} &&
+                      /usr/local/bin/python3 -m pip install --quiet --disable-pip-version-check --target /tmp/pydeps pytest==9.1.0 pytest-asyncio==1.4.0 &&
+                      cd agents/bug-fix &&
+                      unset TASKCLUSTER_ROOT_URL &&
+                      export PYTHONPATH=/tmp/pydeps BUG_FIX_IMAGE_INSIDE=1 &&
+                      (python -m pytest -s -v -rs tests/test_image.py -k android ||
+                      python -m pytest -s -v -rs --last-failed tests/test_image.py -k android)"
+                metadata:
+                  name: bugbug bug-fix agent image Android tests
+                  description: bugbug bug-fix agent image Android tests
+
         - $if: 'tasks_for == "github-push" && head_branch[:10] == "refs/tags/"'
           then:
             $mergeDeep:

agents/bug-fix/Dockerfile

@@ -17,9 +17,15 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     --mount=type=bind,target=/app,rw \
     uv sync --locked --no-dev --no-editable --package hackbot-agent-bug-fix
 
+# profiler-cli, the Firefox Profiler's command-line analyzer. It runs with the
+# node `mach bootstrap` installs, so this stage only installs it.
+FROM node:24-trixie-slim AS profiler-cli
+ARG PROFILER_CLI_VERSION=0.10.0
+RUN npm install --global --prefix /opt/profiler-cli --no-fund --no-audit \
+        @firefox-devtools/profiler-cli@${PROFILER_CLI_VERSION}
+
 FROM python:3.14 AS base
 
-COPY --from=builder /opt/venv /opt/venv
 WORKDIR /app
 
 ENV PYTHONUNBUFFERED=1
@@ -28,33 +34,103 @@ ENV PATH="/opt/venv/bin:$PATH"
 
 FROM base AS agent
 
+# The agent both builds Firefox and runs it, so it needs the build prerequisites
+# plus the browser's runtime libraries. `mach bootstrap` installs the *build*
+# toolchain into ~/.mozbuild, but runs unprivileged here, so it cannot install
+# these system packages itself.
 RUN apt-get update \
     && apt-get install -y --no-install-recommends \
+        # ./mach needs mercurial for its vcs checks even on a git checkout.
+        mercurial git curl zip unzip bzip2 ca-certificates \
+        # Debugging and inspecting: crashes, syscalls, JSON from mach and CI,
+        # profile databases.
+        gdb strace ripgrep jq sqlite3 \
         # Firefox runtime deps.
-        libgtk-3-0 libasound2t64 \
-        ffmpeg \
-        fluxbox \
-        xvfb \
+        libgtk-3-0t64 libdbus-glib-1-2 libx11-xcb1 libxcb1 libxcb-shm0 \
+        libxcb-render0 libxcb-glx0 libxcb-shape0 libxtst6 libxt6 libpci3 \
+        libasound2t64 libpulse0 libpango-1.0-0 libcairo2 libxcomposite1 \
+        libxdamage1 libxfixes3 libxrandr2 libxkbcommon0 libnss3 libnspr4 \
+        # Firefox renders through EGL on X11; Mesa's software one under Xvfb.
+        libegl1 libegl-mesa0 \
+        # Fonts. Beyond the basics, the scripts CI's test images cover (Arabic,
+        # math, Korean, Japanese), so text renders like it does there.
+        fontconfig fonts-liberation fonts-dejavu-core fonts-kacst-one \
+        fonts-stix fonts-unfonts-core fonts-vlgothic \
+        # Names libc frames in crash stacks.
+        libc6-dbg \
+        # Codecs, plus a PipeWire server used as a dummy audio sink/source by
+        # media tests, and pactl to inspect it.
+        ffmpeg pipewire wireplumber pipewire-pulse pulseaudio-utils \
+        # Headless display and session bus, started by docker-entrypoint.sh.
+        dbus-x11 x11-xserver-utils fluxbox xvfb \
     && rm -rf /var/lib/apt/lists/*
 
+# Gecko media tests use the pipewire audiotestsrc plugin as a dummy sound
+# source; it ships disabled. See firefox's
+# taskcluster/docker/recipes/ubuntu2404-test-system-setup-base.sh.
+RUN install -d -o root -g root -m 755 /etc/pipewire \
+    && sed -e '/^context.spa-libs = {/,/^}$/ s/#\(audiotestsrc\)/\1/' \
+        /usr/share/pipewire/pipewire.conf > /etc/pipewire/pipewire.conf
+
+# The Firefox tree's agent skills profile with samply and analyze profiles with
+# the Firefox Profiler's profiler-cli. samply and node come from `mach
+# bootstrap`, profiler-cli from the stage above.
+COPY --from=profiler-cli /opt/profiler-cli /opt/profiler-cli
+
+# The tree's skills run Python helper scripts with `uv run`.
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /usr/local/bin/
+
 # hackbot.toml lives at the agent root (not inside the package), so copy it into
 # the working dir; the runtime discovers it there (cwd) at startup.
 COPY agents/bug-fix/hackbot.toml /app/hackbot.toml
+COPY agents/bug-fix/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+RUN chmod +x /usr/local/bin/docker-entrypoint.sh
 
 RUN useradd --create-home --shell /bin/bash agent \
     && mkdir -p /workspace \
-    && chown agent:agent /workspace
+    && chown agent:agent /workspace \
+    # Xvfb runs unprivileged and cannot create its own socket directory.
+    && mkdir -p /tmp/.X11-unix \
+    && chmod 1777 /tmp/.X11-unix \
+    # Same identity hackbot_runtime commits with, so the agent's own `git commit`
+    # and `git stash` work too instead of failing for lack of one.
+    && git config --system user.name "Hackbot" \
+    && git config --system user.email "hackbot@mozilla.tld"
+
+ENV SCREEN_WIDTH=1920
+ENV SCREEN_HEIGHT=1080
+ENV SCREEN_DEPTH=24
+# For Firefox started from the agent's shell (`./mach run`, mochitests); grizzly
+# starts its own Xvfb for evaluate_testcase.
+ENV DISPLAY=:99
+# Home of the session bus and the PipeWire/PulseAudio sockets.
+ENV XDG_RUNTIME_DIR=/tmp/runtime-agent
+ENV DBUS_SESSION_BUS_ADDRESS=unix:path=/tmp/runtime-agent/bus
 
 # `mach bootstrap` installs the toolchain here at runtime; put it on PATH so the
-# agent's own `./mach build` (and the build_firefox tool) find rustc/clang.
-ENV PATH="/home/agent/.cargo/bin:/home/agent/.mozbuild/clang/bin:${PATH}"
+# agent's own `./mach build` (and the build_firefox tool) find rustc/clang,
+# evaluate_testcase finds minidump-stackwalk to symbolize crashes, and the
+# agent's shell finds node, samply and the developer tools (in ~/.cargo/bin),
+# and the profiler-cli that runs with that node.
+ENV PATH="/home/agent/.cargo/bin:/home/agent/.mozbuild/clang/bin:/home/agent/.mozbuild/minidump-stackwalk:/home/agent/.mozbuild/node/bin:/home/agent/.mozbuild/samply:/opt/profiler-cli/bin:${PATH}"
+# Likewise for Android, once bootstrapped for it: java, adb, aapt2, apksigner,
+# apkanalyzer, lint, sdkmanager. bootstrap_firefox links the versioned
+# directories they are in as `current`/`latest`.
+ENV PATH="${PATH}:/home/agent/.mozbuild/jdk/current/bin:/home/agent/.mozbuild/android-sdk-linux/platform-tools:/home/agent/.mozbuild/android-sdk-linux/build-tools/latest:/home/agent/.mozbuild/android-sdk-linux/cmdline-tools/latest/bin"
+
+# Last, as it changes with any Python change: the layers above (system
+# packages, mostly) stay cached across those.
+COPY --from=builder /opt/venv /opt/venv
 
 USER agent
 
+ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
 CMD ["python", "-m", "hackbot_agents.bug_fix"]
 
 FROM base AS broker
 
+COPY --from=builder /opt/venv /opt/venv
+
 RUN useradd --create-home --shell /bin/bash broker
 
 USER broker

agents/bug-fix/docker-entrypoint.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+# Give Firefox what it expects from a desktop session: an X display for anything
+# launched from the agent's shell (`./mach run`, mochitests), and a session bus
+# plus PipeWire so Web Audio and media tests have a working audio device.
+set -e
+
+mkdir -p -m 700 "$XDG_RUNTIME_DIR"
+
+Xvfb "$DISPLAY" -screen 0 "${SCREEN_WIDTH}x${SCREEN_HEIGHT}x${SCREEN_DEPTH}" -nolisten tcp &
+
+dbus-daemon --session --address="$DBUS_SESSION_BUS_ADDRESS" --fork --nopidfile
+
+# Their logs are noisy (no system bus, bluetooth or cameras here) and not useful
+# to the agent.
+pipewire >/dev/null 2>&1 &
+wireplumber >/dev/null 2>&1 &
+pipewire-pulse >/dev/null 2>&1 &
+
+# The agent may start Firefox seconds after boot, so wait for the display and the
+# audio server to accept connections rather than racing them.
+i=0
+while [ "$i" -lt 50 ]; do
+    xset -display "$DISPLAY" q >/dev/null 2>&1 && [ -S "$XDG_RUNTIME_DIR/pulse/native" ] && break
+    i=$((i + 1))
+    sleep 0.2
+done
+
+exec "$@"

agents/bug-fix/hackbot_agents/bug_fix/agent.py

@@ -133,6 +133,8 @@ def _write_mozconfig(fx_ctx: FirefoxContext) -> None:
         "ac_add_options --enable-debug\n"
         "ac_add_options --enable-optimize\n"
         "ac_add_options --enable-clang-plugin\n"
+        # For the evaluate_js_shell tool: desktop builds skip the JS shell.
+        "ac_add_options --enable-js-shell\n"
         f"mk_add_options MOZ_OBJDIR={quote(str(fx_ctx.objdir))}\n"
     )