Merge pull request #122 from opensafely-core/ubuntu-pro

Add ESM support for 20.04 based python:v1

Author: bloodearnest

.github/workflows/build_and_publish.yaml

@@ -3,7 +3,8 @@ on:
   workflow_dispatch:
   push:
     branches: [main]
-
+env:
+  UBUNTU_PRO_TOKEN: ${{ secrets.UBUNTU_PRO_TOKEN }}
 jobs:
   publish:
     # note: this builds/tests all versions in serial for two reasons. Firstly we

.github/workflows/tests.yaml

@@ -1,6 +1,8 @@
 name: Run tests
 on:
   pull_request:
+env:
+  UBUNTU_PRO_TOKEN: ${{ secrets.UBUNTU_PRO_TOKEN }}
 jobs:
   version-tests:
     runs-on: ubuntu-22.04

.gitignore

@@ -1,3 +1,6 @@
 *~
 venv
+.venv
 __pycache__
+.env
+.secrets

DEVELOPERS.md

@@ -29,6 +29,14 @@ just build v2
 just test v2
 ```
 
+## ESM Packages on 20.04 images
+
+We still support the older python:v1, but it is based on Ubuntu 20.04, which
+has reached EOL for security upgrades. So we have enabled ESM via Ubuntu Pro for these images.
+
+This means that you do need a valid `UBUNTU_PRO_TOKEN` environment variable to
+build these images.
+
 
 ## Add a new package to existing version
 

Dockerfile

@@ -1,4 +1,4 @@
-# syntax=docker/dockerfile:1.2
+# syntax=docker/dockerfile:1.10
 #################################################
 #
 # We need base python dependencies on both the builder and python images, so
@@ -22,7 +22,12 @@ ENV ACTION_EXEC=python MAJOR_VERSION=${MAJOR_VERSION} BASE=${BASE}
 
 COPY ${MAJOR_VERSION}/dependencies.txt /opt/dependencies.txt
 # use space efficient utility from base image
-RUN /root/docker-apt-install.sh /opt/dependencies.txt
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    --mount=type=bind,source=${MAJOR_VERSION}/dependencies.txt,target=/tmp/dependencies.txt \
+    --mount=type=secret,id=ubuntu_pro_token,required=true \
+    mkdir /workspace; \
+    /root/docker-apt-install.sh /tmp/dependencies.txt
 
 # now we have python, set up a venv to install packages to, for isolation from
 # system python libraries 
@@ -42,8 +47,11 @@ FROM base-python as builder
 ARG MAJOR_VERSION
 
 # install build time dependencies 
-COPY ${MAJOR_VERSION}/build-dependencies.txt /opt/build-dependencies.txt
-RUN /root/docker-apt-install.sh /opt/build-dependencies.txt
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    --mount=type=bind,source=${MAJOR_VERSION}/build-dependencies.txt,target=/tmp/build-dependencies.txt \
+    --mount=type=secret,id=ubuntu_pro_token \
+    /root/docker-apt-install.sh /tmp/build-dependencies.txt
 
 COPY ${MAJOR_VERSION}/requirements.txt /opt/requirements.txt
 COPY ${MAJOR_VERSION}/packages.md /opt/packages.md

docker-compose.yml

@@ -5,6 +5,8 @@ services:
     build:
       context: .
       target: base-python
+      secrets:
+        - ubuntu_pro_token
       cache_from:  # should speed up the build in CI, where we have a cold cache
         - ghcr.io/opensafely-core/base-action:${BASE}
         - ghcr.io/opensafely-core/python:${MAJOR_VERSION}
@@ -24,3 +26,7 @@ services:
     image: python:${MAJOR_VERSION}
     build:
       target: python
+
+secrets:
+  ubuntu_pro_token:
+    file: ${UBUNTU_PRO_TOKEN_FILE:-.secrets/ubuntu_pro_token}

justfile

@@ -1,3 +1,6 @@
+set dotenv-load := true
+
+export UBUNTU_PRO_TOKEN_FILE := env_var_or_default('UBUNTU_PRO_TOKEN_FILE', justfile_directory() + "/.secrets/ubuntu_pro_token")
 export DOCKER_BUILDKIT := "1"
 # technically, these could differ by 1 seconds, but thats unlikely and doesn't matter
 # human readable, used as label in docker image
@@ -6,8 +9,20 @@ export BUILD_DATE := `date +'%y-%m-%dT%H:%M:%S.%3NZ'`
 export BUILD_NUMBER := `date +'%y%m%d%H%M%S'`
 export REVISION := `git rev-parse --short HEAD`
 
+ensure-pro-token:
+  #!/bin/bash
+  set -euo pipefail
+  token_file="{{ UBUNTU_PRO_TOKEN_FILE }}"
+  if test -z "${UBUNTU_PRO_TOKEN:-}"; then
+    echo "UBUNTU_PRO_TOKEN is required to create $token_file" >&2
+    exit 1
+  fi
+  mkdir -p "$(dirname "$token_file")"
+  umask 077
+  printf '%s' "$UBUNTU_PRO_TOKEN" > "$token_file"
+
 # build docker image for version
-build version target="python" *args="":
+build version target="python" *args="": ensure-pro-token
     docker compose --env-file {{ version }}/env build --pull {{ args }} {{ target }} 
 
 
@@ -29,8 +44,9 @@ render version *args:
 
 # run linters
 check:
-    @docker pull hadolint/hadolint:v2.12.0
-    @docker run --rm -i hadolint/hadolint:v2.12.0 < Dockerfile
+    @docker run --rm -i hadolint/hadolint:v2.14.0 < Dockerfile
+    @ls scripts/*.sh | xargs docker run --rm -v "$PWD:/mnt:ro" koalaman/shellcheck:v0.11.0
+    @docker run --rm -v "$PWD:/repo:ro" --workdir /repo rhysd/actionlint:1.7.10 -color
 
 
 # publish version (dry run by default - pass "true" to perform publish)

tests/test_packaging.py

@@ -0,0 +1,11 @@
+from pathlib import Path
+import subprocess
+
+import pytest
+
+os_release = Path("/etc/os-release").read_text()
+
+@pytest.mark.skipif('VERSION_ID="20.04"' not in os_release, reason="20.04 only")
+def test_esm():
+    output = subprocess.check_output(["dpkg-query", "-W", "-f='${Package}\t${Version}\n'", "libssl1.1"], text=True)
+    assert "esm" in output