From 6351391352e1e7a16c9c3717f4447cf0a8477472 Mon Sep 17 00:00:00 2001 From: Josh Sokol Date: Fri, 21 Aug 2026 08:28:51 -0500 Subject: [PATCH] ci(docker): verified bundle download with a pre-GA fallback on master The image build fetched the app bundle with a bare `curl -sL .../public/bundles/simplerisk-.tgz | tar xz`. Two problems. 1. public/bundles/ is the GA path, written only by propagate_release_bundle at the testing -> master cut. But bump_downstream_versions opens the docker update- PR at the TESTING cut, from base master -- so the version it pins has no prod bundle yet and container-validation fails on every release. Observed on update-20260709-001, update-20260811-001 and update-20260820-001. 2. Without --fail, curl streams the S3 error document into tar, so the failure surfaces as a tar exit code rather than an HTTP status. The real cause is invisible in the log, which is how this got misread as expected behaviour. The fix already exists on the testing branch and is ported here. PR #146 attempted this in July and was correctly closed: it added the ARG, the COPY and the generator changes but never added the script itself, so it would have failed at COPY. common/download_and_verify_bundle.sh (identical in both build contexts, taken verbatim from testing) downloads the prod bundle, resolves its published sha256 (md5 fallback) from the prod updates feed, and verifies before extracting. Fail-closed by default: a missing bundle, a missing feed hash or a mismatch aborts the build, so a swapped S3 object cannot be baked into a published image. PREGA_BUNDLE_FALLBACK=true -- set ONLY by container-validation -- allows a pre-GA build to fall back to bundles-test without verification, warning loudly, because a release has no published hash before GA. Ported surgically rather than copied. The testing branch's versions of these files also carry a PHP default bump (8.4 -> 8.5), a new source_mode generator parameter, an added php85 validation job and a trigger change -- all unrelated, none included here. Verified after patching: php_version is still 8.4, there are no source_mode references, and the container-validation diff is exactly four build_args lines. Patched the GENERATORS and re-ran them rather than editing the Dockerfiles: the Dockerfiles are generated, and make update_version regenerates them on every version bump, so a hand-edit would be silently overwritten. Also fixed the SQL fetch in the full-stack image, which had the same missing --fail: `curl -sL ... > /simplerisk.sql` writes a 404 body to disk, so the image would ship an HTML error page as its database schema. Verified by running the script in the real alpine/curl:8.12.1 downloader image across all three paths: fail-closed on a pre-GA version without the flag (exit 1), pre-GA fallback to bundles-test with the flag (exit 0, warned, extracted), and the verified-prod path against a GA version that exists. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/container-validation.yml | 8 +-- simplerisk-minimal/Dockerfile | 14 ++++- .../common/download_and_verify_bundle.sh | 61 +++++++++++++++++++ simplerisk-minimal/generate_dockerfile.sh | 14 ++++- simplerisk/Dockerfile | 13 +++- .../common/download_and_verify_bundle.sh | 61 +++++++++++++++++++ simplerisk/generate_dockerfile.sh | 13 +++- 7 files changed, 170 insertions(+), 14 deletions(-) create mode 100755 simplerisk-minimal/common/download_and_verify_bundle.sh create mode 100755 simplerisk/common/download_and_verify_bundle.sh diff --git a/.github/workflows/container-validation.yml b/.github/workflows/container-validation.yml index 6a015ac..730dba5 100644 --- a/.github/workflows/container-validation.yml +++ b/.github/workflows/container-validation.yml @@ -13,7 +13,7 @@ jobs: context_path: "simplerisk/" dockerfile_path: "simplerisk/Dockerfile" image_tag: "simplerisk/simplerisk:testing" - build_args: "ubuntu_version_code=jammy" + build_args: "ubuntu_version_code=jammy\nPREGA_BUNDLE_FALLBACK=true" simplerisk-noble: name: 'Verify simplerisk/simplerisk image based on Ubuntu 24.04 (Noble)' @@ -22,7 +22,7 @@ jobs: context_path: "simplerisk/" dockerfile_path: "simplerisk/Dockerfile" image_tag: "simplerisk/simplerisk:testing" - build_args: "ubuntu_version_code=noble" + build_args: "ubuntu_version_code=noble\nPREGA_BUNDLE_FALLBACK=true" simplerisk-minimal-php84: name: 'Verify simplerisk/simplerisk-minimal image based on PHP 8.3 with Apache' @@ -31,7 +31,7 @@ jobs: context_path: "simplerisk-minimal/" dockerfile_path: "simplerisk-minimal/Dockerfile" image_tag: "simplerisk/simplerisk-minimal:testing" - build_args: "php_version=8.3" + build_args: "php_version=8.3\nPREGA_BUNDLE_FALLBACK=true" simplerisk-minimal-php85: name: 'Verify simplerisk/simplerisk-minimal image based on PHP 8.4 with Apache' @@ -40,4 +40,4 @@ jobs: context_path: "simplerisk-minimal/" dockerfile_path: "simplerisk-minimal/Dockerfile" image_tag: "simplerisk/simplerisk-minimal:testing" - build_args: "php_version=8.4" + build_args: "php_version=8.4\nPREGA_BUNDLE_FALLBACK=true" diff --git a/simplerisk-minimal/Dockerfile b/simplerisk-minimal/Dockerfile index fb395b3..4dc4425 100644 --- a/simplerisk-minimal/Dockerfile +++ b/simplerisk-minimal/Dockerfile @@ -3,10 +3,20 @@ ARG php_version=8.4 FROM alpine/curl:8.12.1 AS downloader +# PREGA_BUNDLE_FALLBACK is a CI-ONLY switch, default false. Pre-GA the prod bundle +# for a new version does not exist yet (it lands in public/bundles/ only at GA). CI +# sets this true so a pre-GA build can fall back to the testing bundle WITHOUT hash +# verification (the release has no published hash yet). A released image is ALWAYS +# built from the VERIFIED prod bundle and NEVER from unverified testing bytes. +ARG PREGA_BUNDLE_FALLBACK=false + SHELL [ "/bin/ash", "-eo", "pipefail", "-c" ] -RUN mkdir -p /var/www && \ - curl -sL https://simplerisk-downloads.s3.amazonaws.com/public/bundles/simplerisk-20260519-001.tgz | tar xz -C /var/www +# Download the prod bundle, verify its published sha256 (md5 fallback) from the +# updates feed, then extract -- fail-closed unless PREGA_BUNDLE_FALLBACK allows the +# pre-GA path. See common/download_and_verify_bundle.sh. +COPY common/download_and_verify_bundle.sh /download_and_verify_bundle.sh +RUN PREGA_BUNDLE_FALLBACK="$PREGA_BUNDLE_FALLBACK" sh /download_and_verify_bundle.sh 20260519-001 FROM php:${php_version}-apache diff --git a/simplerisk-minimal/common/download_and_verify_bundle.sh b/simplerisk-minimal/common/download_and_verify_bundle.sh new file mode 100755 index 0000000..9a9691d --- /dev/null +++ b/simplerisk-minimal/common/download_and_verify_bundle.sh @@ -0,0 +1,61 @@ +#!/bin/sh +# Download the released SimpleRisk bundle for the given version, verify it against +# the sha256 (md5 fallback) published in the production updates feed, then extract +# it into /var/www. Run from the alpine/curl downloader stage of the generated +# Dockerfile. +# +# Fail-closed by default: a RELEASED image (PREGA_BUNDLE_FALLBACK unset/false) is +# built ONLY from the prod bundle and ONLY if it matches its published hash -- a +# missing prod bundle, a missing feed hash, or a mismatch aborts the build, so a +# swapped S3 object can never be baked into a published image. The bundle (S3 +# public/bundles) and its hash (served updates feed) are stored independently. +# +# PRE-GA CI ONLY: before GA the prod bundle/hash for a new version do not exist +# yet (they land at GA). When PREGA_BUNDLE_FALLBACK=true AND the prod bundle is +# absent, fall back to the testing bundle WITHOUT verification (there is no +# published hash to check yet) and warn loudly. This path is never taken for a +# released image. +set -eu + +VERSION="${1:?usage: download_and_verify_bundle.sh }" +case "$VERSION" in + [0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9]) : ;; + *) echo "ERROR: bad version format: $VERSION" >&2; exit 1 ;; +esac + +FEED="https://updates.simplerisk.com/releases.xml" +PROD_URL="https://simplerisk-downloads.s3.amazonaws.com/public/bundles/simplerisk-${VERSION}.tgz" +TEST_URL="https://bundles-test.simplerisk.com/simplerisk-${VERSION}.tgz" +TGZ="/tmp/simplerisk-${VERSION}.tgz" + +if curl -fsSL -o "$TGZ" "$PROD_URL"; then + echo "Downloaded prod bundle for ${VERSION}; resolving published hash from ${FEED} ..." + ENTRY="$(curl -fsSL "$FEED" | sed -n "//,/<\/release>/p")" + EXPECTED="$(printf '%s\n' "$ENTRY" | grep -oE '[0-9a-f]{64}' | grep -oE '[0-9a-f]{64}' | head -1 || true)" + ALGO=sha256 + if [ -z "$EXPECTED" ]; then + EXPECTED="$(printf '%s\n' "$ENTRY" | grep -oE '[0-9a-f]{32}' | grep -oE '[0-9a-f]{32}' | head -1 || true)" + ALGO=md5 + fi + if [ -z "$EXPECTED" ]; then + echo "ERROR: no bundle_sha256 or bundle_md5 for ${VERSION} in ${FEED} -- refusing to extract an unverifiable prod bundle" >&2 + exit 1 + fi + ACTUAL="$(${ALGO}sum "$TGZ" | cut -d' ' -f1)" + if [ "$ACTUAL" != "$EXPECTED" ]; then + echo "ERROR: bundle ${ALGO} mismatch for ${VERSION} -- expected ${EXPECTED}, got ${ACTUAL}" >&2 + exit 1 + fi + echo "Bundle ${ALGO} verified (${ACTUAL})." +elif [ "${PREGA_BUNDLE_FALLBACK:-false}" = "true" ]; then + echo "WARNING: prod bundle simplerisk-${VERSION}.tgz absent -- PRE-GA CI fallback to bundles-test (UNVERIFIED: the release has no published hash yet)." >&2 + curl -fsSL -o "$TGZ" "$TEST_URL" +else + echo "ERROR: prod bundle simplerisk-${VERSION}.tgz not found and PREGA_BUNDLE_FALLBACK != true -- refusing to build a release from unverified bytes" >&2 + exit 1 +fi + +mkdir -p /var/www +tar xzf "$TGZ" -C /var/www +rm -f "$TGZ" +echo "Extracted bundle to /var/www." diff --git a/simplerisk-minimal/generate_dockerfile.sh b/simplerisk-minimal/generate_dockerfile.sh index 357f11c..8075266 100755 --- a/simplerisk-minimal/generate_dockerfile.sh +++ b/simplerisk-minimal/generate_dockerfile.sh @@ -23,10 +23,20 @@ if [ "$release" != "testing" ]; then cat << EOF >> "${SCRIPT_LOCATION}/Dockerfile" FROM alpine/curl:8.12.1 AS downloader +# PREGA_BUNDLE_FALLBACK is a CI-ONLY switch, default false. Pre-GA the prod bundle +# for a new version does not exist yet (it lands in public/bundles/ only at GA). CI +# sets this true so a pre-GA build can fall back to the testing bundle WITHOUT hash +# verification (the release has no published hash yet). A released image is ALWAYS +# built from the VERIFIED prod bundle and NEVER from unverified testing bytes. +ARG PREGA_BUNDLE_FALLBACK=false + SHELL [ "/bin/ash", "-eo", "pipefail", "-c" ] -RUN mkdir -p /var/www && \\ - curl -sL https://simplerisk-downloads.s3.amazonaws.com/public/bundles/simplerisk-$release.tgz | tar xz -C /var/www +# Download the prod bundle, verify its published sha256 (md5 fallback) from the +# updates feed, then extract -- fail-closed unless PREGA_BUNDLE_FALLBACK allows the +# pre-GA path. See common/download_and_verify_bundle.sh. +COPY common/download_and_verify_bundle.sh /download_and_verify_bundle.sh +RUN PREGA_BUNDLE_FALLBACK="\$PREGA_BUNDLE_FALLBACK" sh /download_and_verify_bundle.sh $release EOF fi diff --git a/simplerisk/Dockerfile b/simplerisk/Dockerfile index 300327a..76aa9e2 100644 --- a/simplerisk/Dockerfile +++ b/simplerisk/Dockerfile @@ -5,11 +5,18 @@ FROM alpine/curl:8.12.1 AS downloader ARG DB_LANG=en +# CI-ONLY pre-GA switch (default false) -- see common/download_and_verify_bundle.sh. +ARG PREGA_BUNDLE_FALLBACK=false + SHELL [ "/bin/ash", "-eo", "pipefail", "-c" ] -RUN mkdir -p /var/www && \ - curl -sL https://simplerisk-downloads.s3.amazonaws.com/public/bundles/simplerisk-20260519-001.tgz | tar xz -C /var/www && \ - curl -sL "https://github.com/simplerisk/database/raw/master/simplerisk-$DB_LANG-20260519-001.sql" > /simplerisk.sql +# Download the prod bundle, verify its published sha256 (md5 fallback) from the +# updates feed, then extract (fail-closed) -- then fetch the release SQL schema. +# -fsSL on the SQL fetch too: without --fail, curl writes the 404 body into +# /simplerisk.sql and the image ships an HTML error page as its schema. +COPY common/download_and_verify_bundle.sh /download_and_verify_bundle.sh +RUN PREGA_BUNDLE_FALLBACK="$PREGA_BUNDLE_FALLBACK" sh /download_and_verify_bundle.sh 20260519-001 && \ + curl -fsSL "https://github.com/simplerisk/database/raw/master/simplerisk-$DB_LANG-20260519-001.sql" > /simplerisk.sql # Using Ubuntu image FROM ubuntu:${ubuntu_version_code} diff --git a/simplerisk/common/download_and_verify_bundle.sh b/simplerisk/common/download_and_verify_bundle.sh new file mode 100755 index 0000000..9a9691d --- /dev/null +++ b/simplerisk/common/download_and_verify_bundle.sh @@ -0,0 +1,61 @@ +#!/bin/sh +# Download the released SimpleRisk bundle for the given version, verify it against +# the sha256 (md5 fallback) published in the production updates feed, then extract +# it into /var/www. Run from the alpine/curl downloader stage of the generated +# Dockerfile. +# +# Fail-closed by default: a RELEASED image (PREGA_BUNDLE_FALLBACK unset/false) is +# built ONLY from the prod bundle and ONLY if it matches its published hash -- a +# missing prod bundle, a missing feed hash, or a mismatch aborts the build, so a +# swapped S3 object can never be baked into a published image. The bundle (S3 +# public/bundles) and its hash (served updates feed) are stored independently. +# +# PRE-GA CI ONLY: before GA the prod bundle/hash for a new version do not exist +# yet (they land at GA). When PREGA_BUNDLE_FALLBACK=true AND the prod bundle is +# absent, fall back to the testing bundle WITHOUT verification (there is no +# published hash to check yet) and warn loudly. This path is never taken for a +# released image. +set -eu + +VERSION="${1:?usage: download_and_verify_bundle.sh }" +case "$VERSION" in + [0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9]) : ;; + *) echo "ERROR: bad version format: $VERSION" >&2; exit 1 ;; +esac + +FEED="https://updates.simplerisk.com/releases.xml" +PROD_URL="https://simplerisk-downloads.s3.amazonaws.com/public/bundles/simplerisk-${VERSION}.tgz" +TEST_URL="https://bundles-test.simplerisk.com/simplerisk-${VERSION}.tgz" +TGZ="/tmp/simplerisk-${VERSION}.tgz" + +if curl -fsSL -o "$TGZ" "$PROD_URL"; then + echo "Downloaded prod bundle for ${VERSION}; resolving published hash from ${FEED} ..." + ENTRY="$(curl -fsSL "$FEED" | sed -n "//,/<\/release>/p")" + EXPECTED="$(printf '%s\n' "$ENTRY" | grep -oE '[0-9a-f]{64}' | grep -oE '[0-9a-f]{64}' | head -1 || true)" + ALGO=sha256 + if [ -z "$EXPECTED" ]; then + EXPECTED="$(printf '%s\n' "$ENTRY" | grep -oE '[0-9a-f]{32}' | grep -oE '[0-9a-f]{32}' | head -1 || true)" + ALGO=md5 + fi + if [ -z "$EXPECTED" ]; then + echo "ERROR: no bundle_sha256 or bundle_md5 for ${VERSION} in ${FEED} -- refusing to extract an unverifiable prod bundle" >&2 + exit 1 + fi + ACTUAL="$(${ALGO}sum "$TGZ" | cut -d' ' -f1)" + if [ "$ACTUAL" != "$EXPECTED" ]; then + echo "ERROR: bundle ${ALGO} mismatch for ${VERSION} -- expected ${EXPECTED}, got ${ACTUAL}" >&2 + exit 1 + fi + echo "Bundle ${ALGO} verified (${ACTUAL})." +elif [ "${PREGA_BUNDLE_FALLBACK:-false}" = "true" ]; then + echo "WARNING: prod bundle simplerisk-${VERSION}.tgz absent -- PRE-GA CI fallback to bundles-test (UNVERIFIED: the release has no published hash yet)." >&2 + curl -fsSL -o "$TGZ" "$TEST_URL" +else + echo "ERROR: prod bundle simplerisk-${VERSION}.tgz not found and PREGA_BUNDLE_FALLBACK != true -- refusing to build a release from unverified bytes" >&2 + exit 1 +fi + +mkdir -p /var/www +tar xzf "$TGZ" -C /var/www +rm -f "$TGZ" +echo "Extracted bundle to /var/www." diff --git a/simplerisk/generate_dockerfile.sh b/simplerisk/generate_dockerfile.sh index 7716e87..8dd41c8 100755 --- a/simplerisk/generate_dockerfile.sh +++ b/simplerisk/generate_dockerfile.sh @@ -23,11 +23,18 @@ FROM alpine/curl:8.12.1 AS downloader ARG DB_LANG=en +# CI-ONLY pre-GA switch (default false) -- see common/download_and_verify_bundle.sh. +ARG PREGA_BUNDLE_FALLBACK=false + SHELL [ "/bin/ash", "-eo", "pipefail", "-c" ] -RUN mkdir -p /var/www && \\ - curl -sL https://simplerisk-downloads.s3.amazonaws.com/public/bundles/simplerisk-$release.tgz | tar xz -C /var/www && \\ - curl -sL "https://github.com/simplerisk/database/raw/master/simplerisk-\$DB_LANG-$release.sql" > /simplerisk.sql +# Download the prod bundle, verify its published sha256 (md5 fallback) from the +# updates feed, then extract (fail-closed) -- then fetch the release SQL schema. +# -fsSL on the SQL fetch too: without --fail, curl writes the 404 body into +# /simplerisk.sql and the image ships an HTML error page as its schema. +COPY common/download_and_verify_bundle.sh /download_and_verify_bundle.sh +RUN PREGA_BUNDLE_FALLBACK="\$PREGA_BUNDLE_FALLBACK" sh /download_and_verify_bundle.sh $release && \\ + curl -fsSL "https://github.com/simplerisk/database/raw/master/simplerisk-\$DB_LANG-$release.sql" > /simplerisk.sql EOF fi