fix(ci): generate the SBOM with Syft instead of a BuildKit attestation

.github/workflows/build.yml

@@ -191,6 +191,36 @@ jobs:
           sarif_file: trivy-${{ matrix.arch }}.sarif
           category: trivy-${{ matrix.arch }}
 
+      # Scanned here rather than by BuildKit's own `sbom: true`. BuildKit bundles
+      # its SBOM as an in-toto attestation, and v0.32 began enforcing a 40 MiB
+      # cap on attestation payloads that had been unenforced since 2022
+      # (exporter/attestation/unbundle.go, moby/buildkit@9cdf6429). This image's
+      # SPDX document is roughly 52 MB per architecture - over the cap since
+      # long before it started biting - so the export failed outright the first
+      # weekly release after buildx-stable-1 rolled to v0.32.2, without a line
+      # of this repo changing. Scanning the loaded image here has no ceiling we
+      # do not control, and the result ships on the release instead.
+      #
+      # It runs on every build, publishing or not, so a pull request proves the
+      # scan still works; only the upload is gated.
+      - name: Generate SBOM
+        uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0.24.0
+        with:
+          image: devcontainer-devops:test-${{ matrix.arch }}
+          format: spdx-json
+          output-file: sbom-${{ matrix.arch }}.spdx.json
+          upload-artifact: false
+          upload-release-assets: false
+
+      - name: Upload SBOM
+        if: inputs.publish
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: sbom-${{ matrix.arch }}
+          path: sbom-${{ matrix.arch }}.spdx.json
+          if-no-files-found: error
+          retention-days: 1
+
       - name: Push by digest
         id: push
         if: inputs.publish
@@ -201,7 +231,8 @@ jobs:
           platforms: ${{ matrix.platform }}
           target: final
           labels: ${{ steps.meta.outputs.labels }}
-          sbom: true
+          # No `sbom: true` - see the Generate SBOM step above. Provenance is a
+          # few KB and stays where it is.
           provenance: mode=max
           cache-from: type=registry,ref=${{ env.IMAGE }}:buildcache-${{ matrix.arch }}
           cache-to: type=registry,ref=${{ env.IMAGE }}:buildcache-${{ matrix.arch }},mode=max

.github/workflows/release.yml

@@ -85,33 +85,36 @@ jobs:
     runs-on: ubuntu-24.04
     permissions:
       contents: write
-      packages: read
+      artifact-metadata: read
     steps:
       - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
 
-      - name: Log in to GHCR
-        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
+      # One SPDX document per architecture, scanned in the build job. Each is
+      # around 52 MB of JSON, which is a hostile download to hang off a release
+      # page, and it is highly repetitive - measured at 93% compression on
+      # v2026.08.04 - so both ship gzipped.
+      - name: Download SBOMs
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      # The SBOM is already attached to the image as a build attestation; this
-      # pulls it out into a file so the release has a downloadable copy.
-      # Two architectures' worth of package inventory is well over 100 MB of
-      # JSON, which is a hostile download to hang off a release page. It is
-      # highly repetitive, so it compresses hard - measured at 138.5 MB down to
-      # 10.0 MB on v2026.08.04, a saving of 93%.
-      - name: Extract SBOM
+          path: /tmp/sbom
+          pattern: sbom-*
+          merge-multiple: true
+
+      - name: Compress SBOMs
         run: |
           set -euo pipefail
-          docker buildx imagetools inspect \
-            "ghcr.io/${{ github.repository }}@${{ needs.build.outputs.digest }}" \
-            --format '{{ json .SBOM }}' > sbom.json
-          raw="$(wc -c < sbom.json)"
-          gzip -9 sbom.json
-          gz="$(wc -c < sbom.json.gz)"
-          echo "SBOM: ${raw} bytes raw, ${gz} bytes gzipped"
+          shopt -s nullglob
+          files=(/tmp/sbom/*.spdx.json)
+          if [ ${#files[@]} -eq 0 ]; then
+            echo "::error::no SBOMs were downloaded - nothing to attach"
+            exit 1
+          fi
+          for f in "${files[@]}"; do
+            raw="$(wc -c < "$f")"
+            gzip -9 "$f"
+            gz="$(wc -c < "${f}.gz")"
+            echo "$(basename "$f"): ${raw} bytes raw, ${gz} bytes gzipped"
+          done
 
       - name: Create release
         env:
@@ -139,17 +142,13 @@ jobs:
           gh attestation verify oci://ghcr.io/${{ github.repository }}:${VERSION} -R ${{ github.repository }}
           \`\`\`
 
-          \`sbom.json.gz\` below is the CycloneDX SBOM for both architectures,
-          gunzip to read. The same data is attached to the image itself as a
-          build attestation:
-
-          \`\`\`
-          docker buildx imagetools inspect ghcr.io/${{ github.repository }}:${VERSION} --format '{{ json .SBOM }}'
-          \`\`\`
+          \`sbom-amd64.spdx.json.gz\` and \`sbom-arm64.spdx.json.gz\` below are the
+          SPDX SBOMs for each architecture, generated with Syft. Gunzip to read;
+          each is an ordinary SPDX 2.3 document any SPDX tool will accept.
           EOF
 
           gh release create "v${VERSION}" \
             --target "${GITHUB_SHA}" \
             --title "v${VERSION}" \
             --notes-file notes.md \
-            sbom.json.gz
+            /tmp/sbom/*.spdx.json.gz

ARCHITECTURE.md

@@ -101,7 +101,7 @@ Developer      GitHub Actions          GHCR            DevContainer
    │                  │    manifest list │                   │
    │                  │                  │                   │
    │                  │─── Attest ──────▶│                   │
-   │                  │    SBOM + SLSA   │                   │
+   │                  │    SLSA build    │                   │
    │                  │                  │                   │
    │─────────────── Pull Image ─────────────────────────────▶│
    │                  │                  │                   │
@@ -188,11 +188,21 @@ honour. A digest is the only stable identifier; releases record theirs.
 
 ### Supply chain
 
-BuildKit generates an SBOM and full provenance for each pushed image, and
-`actions/attest` signs the merged manifest with a short-lived Sigstore
-certificate, pushing the attestation to the registry as an OCI referrer. That
-single mechanism covers signing - a separate cosign step would sign the same
-digest a second time with the same trust root, and was left out for that reason.
+BuildKit generates full provenance for each pushed image, and `actions/attest`
+signs the merged manifest with a short-lived Sigstore certificate, pushing the
+attestation to the registry as an OCI referrer. That single mechanism covers
+signing - a separate cosign step would sign the same digest a second time with
+the same trust root, and was left out for that reason.
+
+The SBOM is deliberately not a BuildKit attestation. BuildKit bundles one as an
+in-toto statement and enforces a 40 MiB ceiling on it; this image's SPDX document
+runs to about 52 MB per architecture, and the ceiling only started being enforced
+in BuildKit v0.32, which broke the weekly release with no change to this
+repository. Syft scans the tested image in the build job instead, and each
+release carries one SPDX document per architecture. The size that made the
+attestation impossible is inherent - a container this broad has some 6,800
+packages and 50,000 catalogued files - so the fix was to stop routing it through
+a mechanism with a cap we do not control.
 
 Trivy scans report to the Security tab and never gate the publish. An image
 bundling the Azure CLI, Ansible and a .NET SDK carries upstream HIGH findings at

CHANGELOG.md

@@ -103,6 +103,9 @@ contained on a given date; it cannot promise a compatibility contract.
   then verified by a commented-out line
 - The `shellcheck` and `hadolint` pre-commit hooks now read the same config
   as the CI lint job, so a clean run locally means a clean run in CI
+- The SBOM is generated by Syft in the build job and attached to each release as
+  one SPDX document per architecture, `sbom-<arch>.spdx.json.gz`, rather than
+  being embedded in the image by BuildKit and extracted from it afterwards
 
 ### Removed
 
@@ -126,6 +129,11 @@ contained on a given date; it cannot promise a compatibility contract.
   that branch could never have worked; it now uses the GitHub release asset
 - Line endings are consistent. The repo mixed CRLF docs with LF scripts and
   declared neither, so editing a file could silently leave it mixed
+- The weekly release builds again. BuildKit v0.32 began enforcing a 40 MiB cap
+  on in-toto attestation payloads that had gone unenforced since 2022, and this
+  image's SBOM has been roughly 52 MB per architecture for months, so the push
+  failed on both architectures the first Sunday after `buildx-stable-1` rolled
+  from v0.31.2 to v0.32.2
 
 ### Security
 

README.md

@@ -3,8 +3,8 @@
 A pre-built **VS Code dev container for DevOps and Infrastructure-as-Code work**:
 Terraform, Terragrunt, Azure CLI, Ansible, Kubernetes, Helm, PowerShell and .NET
 on Ubuntu 24.04. Published multi-architecture to the GitHub Container Registry
-with an SBOM and SLSA build provenance, so there is nothing to build before you
-start.
+with SLSA build provenance, and an SBOM on every release, so there is nothing to
+build before you start.
 
 It is also a **multi-root devcontainer**: one container, several Git
 repositories open in a single VS Code window. A DevOps change is rarely confined
@@ -318,22 +318,29 @@ image, pin the digest**, which every release records.
 
 ### Supply chain
 
-Every published image carries an SBOM and SLSA build provenance, generated by
-BuildKit and signed with a short-lived [Sigstore](https://www.sigstore.dev/)
-certificate. Verify that an image really came from this repository:
+Every published image carries SLSA build provenance, generated by BuildKit and
+signed with a short-lived [Sigstore](https://www.sigstore.dev/) certificate.
+Verify that an image really came from this repository:
 
 ```bash
 gh attestation verify oci://ghcr.io/dbhq-uk/devcontainer-devops:latest \
   -R dbhq-uk/devcontainer-devops
 ```
 
-Read the SBOM out of the image:
+Read the SBOM. Each release carries one SPDX document per architecture,
+generated with [Syft](https://github.com/anchore/syft) from the image that was
+tested and published:
 
 ```bash
-docker buildx imagetools inspect ghcr.io/dbhq-uk/devcontainer-devops:latest \
-  --format '{{ json .SBOM }}'
+gh release download --repo dbhq-uk/devcontainer-devops \
+  --pattern 'sbom-amd64.spdx.json.gz'
+gunzip sbom-amd64.spdx.json.gz
 ```
 
+The SBOM is not attached to the image as a build attestation. BuildKit caps an
+attestation payload at 40 MiB and this image's SPDX document is around 52 MB per
+architecture, so it ships with the release instead.
+
 Trivy scans each build for HIGH and CRITICAL vulnerabilities and reports them
 to the repository's Security tab. Scans report, they do not block: an image
 bundling the Azure CLI, Ansible and a .NET SDK always carries some upstream

SECURITY.md

@@ -55,8 +55,9 @@ Include:
    - Images are published to the GitHub Container Registry, public and
      anonymously pullable
    - Every build is scanned by Trivy, with findings reported to the Security tab
-   - Every published image carries an SBOM and Sigstore-signed SLSA build
-     provenance - verify before use (see below)
+   - Every published image carries Sigstore-signed SLSA build provenance, and
+     every release ships an SPDX SBOM per architecture - verify before use
+     (see below)
    - CI authenticates with the repository's own `GITHUB_TOKEN`; there are no
      registry credentials to store or rotate
 
@@ -162,11 +163,13 @@ gh attestation verify oci://ghcr.io/dbhq-uk/devcontainer-devops:latest \
   -R dbhq-uk/devcontainer-devops
 ```
 
-Inspect its SBOM:
+Inspect its SBOM. One SPDX document per architecture is attached to the release
+that published the image:
 
 ```bash
-docker buildx imagetools inspect ghcr.io/dbhq-uk/devcontainer-devops:latest \
-  --format '{{ json .SBOM }}'
+gh release download --repo dbhq-uk/devcontainer-devops \
+  --pattern 'sbom-amd64.spdx.json.gz'
+gunzip sbom-amd64.spdx.json.gz
 ```
 
 Scan it yourself: