fix: make the coreutils toolchain work in WORKSPACE mode and with aspect_bazel_lib 1.x

.github/workflows/ci.yaml

@@ -33,7 +33,7 @@
         with:
           egress-policy: audit
       - id: linux
         run: echo "os=ubuntu-latest" >> $GITHUB_OUTPUT
     outputs:
       # Will look like ["ubuntu-latest"]
       os: ${{ toJSON(steps.*.outputs.os) }}
@@ -48,13 +48,13 @@
           egress-policy: audit
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - id: bazel_91
         run: echo "bazelversion=9.1.0" >> $GITHUB_OUTPUT
       - id: bazel_86
         run: echo "bazelversion=8.6.0" >> $GITHUB_OUTPUT
       - id: bazel_83
         run: echo "bazelversion=8.3.0" >> $GITHUB_OUTPUT
       - id: bazel_76
         run: echo "bazelversion=7.6.1" >> $GITHUB_OUTPUT
     outputs:
       # Will look like ["<version from .bazelversion>", "x.y.z"]
       bazelversions: ${{ toJSON(steps.*.outputs.bazelversion) }}
@@ -86,7 +86,7 @@
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
-      - uses: bazel-contrib/setup-bazel@c5acdfb288317d0b5c0bbd7a396a3dc868bb0f86 # v0.19.0
+      - uses: bazel-contrib/setup-bazel@c5acdfb288317d0b5c0bbd7a396a3dc868bb0f86 # 0.19.0
         with:
           repository-cache: true
           bazelrc: |
@@ -113,6 +113,138 @@
         if: ${{ matrix.bazelversion <= '8.' && matrix.folder == '.' }}
         run: bazel query 'tests(//...) except docs/...' | xargs bazel test
 
+      # Bazel 9 dropped WORKSPACE; on 7 and 8 the published snippet still has to work.
+      - name: Test WORKSPACE mode
+        working-directory: ${{ matrix.folder }}
+        if: ${{ matrix.bazelversion < '9.' && matrix.folder == 'e2e/smoke' }}
+        run: bazel test //... --noenable_bzlmod --enable_workspace
+
+  # Runs the build through actiond, a local Remote Execution API worker that
+  # executes each action inside an empty chroot in a Linux VM on the runner.
+  # This is what keeps remote-execution support from regressing: an action that
+  # depends on the executor's userspace, or that opts out of remote execution,
+  # fails here and nowhere else. x86_64 only - GitHub's arm64 Linux runners
+  # have no /dev/kvm.
+  remote-execution:
+    name: Build examples/wolfi-base via actiond
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    permissions:
+      contents: read # Clone the repository
+    env:
+      ACTIOND_VERSION: v0.0.6
+      ACTIOND_SHA256: 006dc798d4363596fe8ab997606fc93766a0cc427c2d005cf4fc1765fa4c2052
+      ACTIOND_ENDPOINT: 127.0.0.1:8980
+
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
+        with:
+          egress-policy: audit
+
+      - name: Check out code
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Enable KVM and vhost-vsock
+        run: |
+          set -euxo pipefail
+          test -c /dev/kvm
+          sudo chmod a+rw /dev/kvm
+          if [[ ! -e /dev/vhost-vsock ]]; then
+            sudo modprobe vhost_vsock
+          fi
+          test -c /dev/vhost-vsock
+          sudo chmod a+rw /dev/vhost-vsock
+
+      - name: Download actiond
+        env:
+          GH_TOKEN: ${{ github.token }}
+        working-directory: ${{ runner.temp }}
+        run: |
+          set -euxo pipefail
+          gh release download "${ACTIOND_VERSION}" \
+            --repo hermeticbuild/actiond \
+            --pattern 'linux-actiond_linux_x86_64' \
+            --clobber
+          # actiond publishes no build provenance, so the digest is pinned
+          # here instead of read from a checksum file next to the asset.
+          echo "${ACTIOND_SHA256}  linux-actiond_linux_x86_64" | sha256sum -c -
+          chmod +x linux-actiond_linux_x86_64
+
+      - name: Start actiond worker
+        working-directory: ${{ runner.temp }}
+        run: |
+          set -euxo pipefail
+          setsid ./linux-actiond_linux_x86_64 serve-vm \
+            --listen="${ACTIOND_ENDPOINT}" \
+            --root="${RUNNER_TEMP}/actiond-root" \
+            --cas-image-size-mib=8192 \
+            --memory-mib=4096 \
+            --cpus=4 \
+            --start-timeout-ms=180000 \
+            >"${RUNNER_TEMP}/actiond.log" 2>&1 &
+          echo "$!" > "${RUNNER_TEMP}/actiond.pid"
+
+          # The worker is ready when it logs the gRPC bridge endpoint.
+          for _ in $(seq 1 240); do
+            if grep -Fq "actiond VM raw gRPC bridge listening on ${ACTIOND_ENDPOINT}" "${RUNNER_TEMP}/actiond.log"; then
+              exit 0
+            fi
+            if ! kill -0 "$(cat "${RUNNER_TEMP}/actiond.pid")" 2>/dev/null; then
+              echo "actiond exited before becoming ready" >&2
+              cat "${RUNNER_TEMP}/actiond.log" >&2
+              exit 1
+            fi
+            sleep 1
+          done
+          echo "timed out waiting for ${ACTIOND_ENDPOINT}" >&2
+          cat "${RUNNER_TEMP}/actiond.log" >&2
+          exit 1
+
+      - name: Set up Bazel
+        uses: bazel-contrib/setup-bazel@c5acdfb288317d0b5c0bbd7a396a3dc868bb0f86 # 0.19.0
+        with:
+          repository-cache: true
+          bazelrc: |
+            common --announce_rc --color=yes
+
+      # Regression gate for remote execution. --spawn_strategy=remote with no
+      # local fallback means an action that opts out of remote execution (a
+      # no-remote-exec tag) fails the build outright rather than quietly
+      # running on the host and passing. requires-bash is needed
+      # because rules_apko stages inputs with run_shell actions; everything
+      # they call is either a bash builtin or comes from the coreutils
+      # toolchain, so nothing else is inherited from the executor.
+      - name: Build examples/wolfi-base with remote execution
+        run: |
+          set -euxo pipefail
+          bazel build //examples/wolfi-base \
+            --remote_executor="grpc://${ACTIOND_ENDPOINT}" \
+            --remote_cache="grpc://${ACTIOND_ENDPOINT}" \
+            --remote_default_exec_properties=requires-bash= \
+            --remote_upload_local_results=false \
+            --noremote_cache_compression \
+            --spawn_strategy=remote \
+            --remote_local_fallback=false \
+            --verbose_failures
+
+      - name: Stop actiond
+        if: always()
+        run: |
+          if [[ -f "${RUNNER_TEMP}/actiond.pid" ]]; then
+            kill "$(cat "${RUNNER_TEMP}/actiond.pid")" 2>/dev/null || true
+          fi
+
+      - name: Upload actiond log
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: actiond-log
+          if-no-files-found: warn
+          path: ${{ runner.temp }}/actiond.log
+
   # Single summary gate that aggregates the dynamic test matrix. Branch
   # protection only needs to require this one context, so the Bazel version
   # matrix above can change without an infra change to the required checks.
@@ -121,6 +253,7 @@
     needs:
       - matrix-prep-bazelversion
       - matrix-prep-os
+      - remote-execution
       - test
     if: always()
     steps:

apko/private/apko_config.bzl

@@ -2,6 +2,26 @@
 
 load("@bazel_skylib//lib:paths.bzl", "paths")
 
+# Prebuilt uutils coreutils, registered by aspect_bazel_lib under bzlmod.
+# WORKSPACE users register it themselves (see e2e/smoke/WORKSPACE.bazel). The
+# Linux builds are statically linked against musl in recent aspect_bazel_lib
+# releases; older ones, including the 1.x rules_apko_dependencies() fetches,
+# link glibc. Declaring it as an action tool means mkdir/cp travel with the
+# action instead of being inherited from the executor image, which is what
+# lets these actions run on executors that ship no userspace of their own.
+COREUTILS_TOOLCHAIN_TYPE = "@aspect_bazel_lib//lib:coreutils_toolchain_type"
+
+def coreutils_bin(ctx):
+    """The coreutils binary from the resolved toolchain.
+
+    Args:
+        ctx: rule context
+
+    Returns:
+        The coreutils File.
+    """
+    return ctx.toolchains[COREUTILS_TOOLCHAIN_TYPE].coreutils_info.bin
+
 _PATH_CONVENTION_DOC = """
  When referencing other files in the config yaml file use paths relative to your Bazel workspace root. 
     For example, if you want to reference source file foo/bar/baz use foo/bar/baz. If you want to reference output file of foo/bar:rule and rule's 
@@ -69,7 +89,8 @@ def copy_to_workdir(ctx, src, dst):
 
     Uses cp instead of ctx.actions.symlink because Bazel 9 sandbox changes
     cause symlink outputs to dangle when used as inputs to subsequent
-    sandboxed actions.
+    sandboxed actions. mkdir and cp come from the coreutils toolchain rather
+    than the executor's PATH, so the action carries its own tools.
 
     When src is a list, dst must be a declared directory (tree artifact);
     each source is staged at its workspace-relative path inside dst. This
@@ -86,12 +107,17 @@ def copy_to_workdir(ctx, src, dst):
         dst: declared output File or directory
     """
 
-    # When f is a directory, append "/." so its contents land directly in
-    # target instead of nesting at target/<basename> (Bazel pre-creates
-    # declared output directories, so plain cp -r src tgt would nest).
-    copy_file_cmd = lambda f, target: 'mkdir -p "{dir}" && cp -rfL "{src}" "{tgt}"'.format(
+    coreutils = coreutils_bin(ctx)
+
+    # When f is a directory, remove the pre-created output directory first so
+    # cp creates it as a copy of f rather than nesting at target/<basename>.
+    # Copying "f/." into it does not work with uutils 0.0.16 (aspect_bazel_lib
+    # 1.x), which nests that as well.
+    copy_file_cmd = lambda f, target: '"{cu}" mkdir -p "{dir}" && {rm}"{cu}" cp -rfL "{src}" "{tgt}"'.format(
+        cu = coreutils.path,
         dir = paths.dirname(target),
-        src = (f.path + "/.") if f.is_directory else f.path,
+        rm = '"{}" rm -rf "{}" && '.format(coreutils.path, target) if f.is_directory else "",
+        src = f.path,
         tgt = target,
     )
 
@@ -110,9 +136,11 @@ def copy_to_workdir(ctx, src, dst):
     ctx.actions.run_shell(
         inputs = inputs,
         outputs = [dst],
+        tools = [coreutils],
         command = "\n".join(cmds),
         mnemonic = "CopyToWorkdir",
         progress_message = "Staging %s" % dst.short_path,
+        toolchain = COREUTILS_TOOLCHAIN_TYPE,
     )
 
 def prepare_apko_config_in_workdir(workdir, ctx):

apko/private/apko_image.bzl

@@ -2,7 +2,7 @@
 
 load("@bazel_skylib//lib:paths.bzl", "paths")
 load("@bazel_skylib//lib:versions.bzl", "versions")
-load("//apko/private:apko_config.bzl", "copy_to_workdir", "prepare_apko_config_in_workdir")
+load("//apko/private:apko_config.bzl", "COREUTILS_TOOLCHAIN_TYPE", "copy_to_workdir", "coreutils_bin", "prepare_apko_config_in_workdir")
 load("//apko/private:apko_run.bzl", "apko_run")
 
 _ATTRS = {
@@ -12,6 +12,10 @@ _ATTRS = {
     "architecture": attr.string(doc = "the CPU architecture which this image should be built to run on. See https://github.com/chainguard-dev/apko/blob/main/docs/apko_file.md#archs-top-level-element"),
     "tag": attr.string(doc = "tag to apply to the resulting docker tarball. only applicable when `output` is `docker`", mandatory = True),
     "args": attr.string_list(doc = "additional arguments to provide when running the `apko build` command."),
+    "use_default_shell_env": attr.bool(
+        doc = "whether the apko build action inherits the default shell environment (the client environment as filtered by `--action_env` / `--incompatible_strict_action_env`). Set to `False` to run apko with a hermetic action environment.",
+        default = True,
+    ),
 }
 
 def _impl(ctx):
@@ -81,7 +85,6 @@ def _impl(ctx):
         inputs.append(lockfile_copy)
         args.add("--lockfile={}".format(lockfile.short_path))
 
-    args.add("--cache-dir={}".format(cache_name))
     args.add("--offline")
 
     if ctx.attr.architecture:
@@ -102,15 +105,51 @@ def _impl(ctx):
     copy_to_workdir(ctx, apko_info.binary, apko_binary)
     inputs.append(apko_binary)
 
+    # apko writes into its --cache-dir even with --offline (it expands
+    # packages and creates directories on cache lookup). Bazel provides the
+    # prepopulated cache tree artifact as a read-only input, which remote
+    # executors enforce strictly, so we mirror it into a writable scratch
+    # directory as real directories containing per-file symlinks. apko only
+    # adds new cache entries and never rewrites existing files, so the
+    # symlinks stay read-only while new writes land in the scratch
+    # directories, avoiding a full copy of the cache. HOME falls back to a
+    # scratch directory when the action environment leaves it unset or empty,
+    # as with use_default_shell_env = False or an executor that sets none.
+    #
+    # Every command below is either a bash builtin or a coreutils subcommand,
+    # so the action depends on nothing from the executor image beyond a shell.
+    # COREUTILS is resolved to an absolute path up front because the EXIT trap
+    # fires after the cd into the workdir. The mirror is copied to a path that
+    # does not exist yet rather than as "src/." into an existing directory,
+    # because uutils 0.0.16 (registered by aspect_bazel_lib 1.x) nests the
+    # latter at dst/<basename>.
+    coreutils = coreutils_bin(ctx)
+    command = "\n".join([
+        "set -e",
+        'COREUTILS="$PWD/{coreutils}"',
+        'SCRATCH="$("$COREUTILS" mktemp -d)"',
+        "trap '\"$COREUTILS\" rm -rf \"$SCRATCH\"' EXIT",
+        '"$COREUTILS" mkdir "$SCRATCH/home"',
+        'CACHE_SRC="$(cd {cache_src} && pwd -P)"',
+        '"$COREUTILS" cp -Rs "$CACHE_SRC" "$SCRATCH/cache"',
+        'export HOME="${{HOME:-$SCRATCH/home}}"',
+        "cd {workdir}",
+        '{apko} "$@" --cache-dir="$SCRATCH/cache"',
+    ]).format(
+        coreutils = coreutils.path,
+        cache_src = cache_dir.path,
+        workdir = paths.join(ctx.bin_dir.path, ctx.label.workspace_root, ctx.label.package, workdir),
+        apko = apko_info.binary.short_path,
+    )
+
     ctx.actions.run_shell(
-        command = "cd {} && {} $@".format(paths.join(ctx.bin_dir.path, ctx.label.workspace_root, ctx.label.package, workdir), apko_info.binary.short_path),
+        command = command,
         arguments = [args],
         inputs = inputs,
-        tools = [apko_info.binary],
+        tools = [apko_info.binary, coreutils],
         outputs = [output],
-        use_default_shell_env = True,
-        execution_requirements = {"no-remote-exec": "1"},
-        toolchain = None,
+        use_default_shell_env = ctx.attr.use_default_shell_env,
+        toolchain = "@rules_apko//apko:toolchain_type",
     )
 
     return DefaultInfo(
@@ -120,7 +159,10 @@ def _impl(ctx):
 apko_image_lib = struct(
     attrs = _ATTRS,
     implementation = _impl,
-    toolchains = ["@rules_apko//apko:toolchain_type"],
+    toolchains = [
+        "@rules_apko//apko:toolchain_type",
+        COREUTILS_TOOLCHAIN_TYPE,
+    ],
 )
 
 _apko_image = rule(
@@ -137,6 +179,7 @@ def apko_image(
         output = "oci",
         architecture = None,
         args = [],
+        use_default_shell_env = True,
         **kwargs):
     """Build OCI images from APK packages directly without Dockerfile
 
@@ -181,6 +224,9 @@ def apko_image(
      architecture: the CPU architecture which this image should be built to run on. See https://github.com/chainguard-dev/apko/blob/main/docs/apko_file.md#archs-top-level-element"),
      tag:          tag to apply to the resulting docker tarball. only applicable when `output` is `docker`
      args:         additional arguments to provide when running the `apko build` command.
+     use_default_shell_env: whether the apko build action inherits the default shell environment
+        (the client environment as filtered by `--action_env` / `--incompatible_strict_action_env`).
+        Set to `False` to run apko with a hermetic action environment.
      **kwargs:       other common arguments like: tags, visibility.
     """
     _apko_image(
@@ -191,6 +237,7 @@ def apko_image(
         architecture = architecture,
         tag = tag,
         args = args,
+        use_default_shell_env = use_default_shell_env,
         **kwargs
     )
     config_label = native.package_relative_label(config)

apko/private/apko_show_config.bzl

@@ -1,7 +1,7 @@
 "A rule for expanding apko config"
 
 load("@bazel_skylib//lib:paths.bzl", "paths")
-load("//apko/private:apko_config.bzl", "copy_to_workdir", "prepare_apko_config_in_workdir")
+load("//apko/private:apko_config.bzl", "COREUTILS_TOOLCHAIN_TYPE", "copy_to_workdir", "prepare_apko_config_in_workdir")
 
 def _impl(ctx):
     output = ctx.actions.declare_file(ctx.attr.name)
@@ -45,6 +45,7 @@ def _impl(ctx):
         inputs = inputs,
         tools = [apko_info.binary],
         outputs = [output],
+        toolchain = "@rules_apko//apko:toolchain_type",
     )
 
     return DefaultInfo(
@@ -63,5 +64,8 @@ For more advanced use-cases (multi-file configuration), use target providing `Ap
         ),
     },
     doc = "Wrapper around `apko show-config` command to generate expanded config as bazel build action.",
-    toolchains = ["@rules_apko//apko:toolchain_type"],
+    toolchains = [
+        "@rules_apko//apko:toolchain_type",
+        COREUTILS_TOOLCHAIN_TYPE,
+    ],
 )