diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index ebe783406..e7bf452ad 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -24,6 +24,7 @@ jobs: run_core_e2e: ${{ steps.labels.outputs.run_core_e2e }} run_gpu_e2e: ${{ steps.labels.outputs.run_gpu_e2e }} run_kubernetes_ha_e2e: ${{ steps.labels.outputs.run_kubernetes_ha_e2e }} + run_centos_selinux_e2e: ${{ steps.labels.outputs.run_centos_selinux_e2e }} run_any_e2e: ${{ steps.labels.outputs.run_any_e2e }} steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -41,12 +42,14 @@ jobs: run_core_e2e=true run_gpu_e2e=true run_kubernetes_ha_e2e=true + run_centos_selinux_e2e=true else run_core_e2e="$(jq -r 'index("test:e2e") != null' <<< "$LABELS_JSON")" run_gpu_e2e="$(jq -r 'index("test:e2e-gpu") != null' <<< "$LABELS_JSON")" run_kubernetes_ha_e2e="$(jq -r 'index("test:e2e-kubernetes") != null' <<< "$LABELS_JSON")" + run_centos_selinux_e2e="$(jq -r 'index("test:e2e-centos") != null' <<< "$LABELS_JSON")" fi - if [ "$run_core_e2e" = "true" ] || [ "$run_gpu_e2e" = "true" ] || [ "$run_kubernetes_ha_e2e" = "true" ]; then + if [ "$run_core_e2e" = "true" ] || [ "$run_gpu_e2e" = "true" ] || [ "$run_kubernetes_ha_e2e" = "true" ] || [ "$run_centos_selinux_e2e" = "true" ]; then run_any_e2e=true else run_any_e2e=false @@ -55,6 +58,7 @@ jobs: echo "run_core_e2e=$run_core_e2e" echo "run_gpu_e2e=$run_gpu_e2e" echo "run_kubernetes_ha_e2e=$run_kubernetes_ha_e2e" + echo "run_centos_selinux_e2e=$run_centos_selinux_e2e" echo "run_any_e2e=$run_any_e2e" } >> "$GITHUB_OUTPUT" @@ -145,6 +149,16 @@ jobs: extra-helm-values: deploy/helm/openshell/ci/values-high-availability.yaml external-postgres-secret: openshell-ha-pg + centos-selinux-e2e: + needs: [pr_metadata, build-supervisor] + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_centos_selinux_e2e == 'true' + permissions: + contents: read + packages: read + uses: ./.github/workflows/e2e-centos-selinux.yml + with: + image-tag: ${{ github.sha }} + core-e2e-result: name: Core E2E result needs: [pr_metadata, build-gateway, build-supervisor, e2e, kubernetes-e2e] @@ -225,3 +239,28 @@ jobs: fi done exit "$failed" + + centos-selinux-e2e-result: + name: CentOS Stream 10 SELinux E2E result + needs: [pr_metadata, build-supervisor, centos-selinux-e2e] + if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_centos_selinux_e2e == 'true' + runs-on: ubuntu-latest + steps: + - name: Verify CentOS Stream 10 SELinux E2E jobs + env: + BUILD_SUPERVISOR_RESULT: ${{ needs.build-supervisor.result }} + CENTOS_SELINUX_E2E_RESULT: ${{ needs.centos-selinux-e2e.result }} + run: | + set -euo pipefail + failed=0 + for item in \ + "build-supervisor:$BUILD_SUPERVISOR_RESULT" \ + "centos-selinux-e2e:$CENTOS_SELINUX_E2E_RESULT"; do + name="${item%%:*}" + result="${item#*:}" + if [ "$result" != "success" ]; then + echo "::error::$name concluded $result" + failed=1 + fi + done + exit "$failed" diff --git a/.github/workflows/e2e-centos-selinux.yml b/.github/workflows/e2e-centos-selinux.yml new file mode 100644 index 000000000..c2f870976 --- /dev/null +++ b/.github/workflows/e2e-centos-selinux.yml @@ -0,0 +1,342 @@ +name: CentOS Stream 10 Docker/Podman/SELinux Compatibility + +# Exercises the Docker and Podman compute drivers on CentOS Stream 10 with +# SELinux enforcing. This is the cross-compatibility check requested in +# https://github.com/NVIDIA/OpenShell/pull/2092: RHEL-family distros ship +# Podman but not Docker, so this job installs both engines, confirms they +# coexist on one host, and runs the standard Docker/Podman e2e suites +# against a real SELinux-enforcing kernel (unlike the Ubuntu-based +# `ghcr.io/nvidia/openshell/ci` image used by e2e-test.yml, which does not +# enforce SELinux). This gives bind-mount relabeling changes (e.g. the +# `selinux_label`/`:z`/`:Z` support added in #2092) a host that can +# actually deny mislabeled mounts. +# +# Unlike a bare RHEL 10 runner, this does NOT require an org admin to +# provision a new GitHub-hosted runner class. Instead, it boots a real +# CentOS Stream 10 guest under QEMU/KVM on the existing self-hosted +# `linux-amd64-cpu8` runner (the same runner class already used by +# driver-vm-linux.yml with `--privileged`), and drives the guest entirely +# over SSH. This does require that runner class to expose `/dev/kvm`; the +# `create` step below fails fast with a clear error if it doesn't. +# +# The guest is created and torn down with +# `tasks/scripts/vm/centos-selinux-vm.sh`, the same script contributors can +# run locally (`mise run vm:centos-selinux:create`) to reproduce this +# environment outside CI. +# +# The host job itself runs in the same Ubuntu-based CI container as the +# rest of the suite; only the guest VM is CentOS Stream 10 with SELinux +# enforcing. + +on: + workflow_call: + inputs: + image-tag: + description: "Image tag to test (typically the commit SHA)" + required: true + type: string + runner: + description: "GitHub Actions runner label for the host job (must expose /dev/kvm)" + required: false + type: string + default: "linux-amd64-cpu8" + checkout-ref: + description: "Git ref to check out for test inputs (defaults to the workflow SHA)" + required: false + type: string + default: "" + mise-version: + description: "mise version to install in the CentOS Stream guest" + required: false + type: string + default: "v2026.4.25" + +permissions: + contents: read + packages: read + +jobs: + centos-selinux-compat: + name: CentOS Stream 10 Docker + Podman + SELinux + runs-on: ${{ inputs.runner }} + timeout-minutes: 45 + permissions: + contents: read + packages: read + container: + image: ghcr.io/nvidia/openshell/ci:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: --privileged --device=/dev/kvm + env: + MISE_VERSION: ${{ inputs.mise-version }} + IMAGE_TAG: ${{ inputs.image-tag }} + OPENSHELL_SUPERVISOR_IMAGE: ${{ format('ghcr.io/nvidia/openshell/supervisor:{0}', inputs.image-tag) }} + OPENSHELL_REGISTRY: ghcr.io/nvidia/openshell + OPENSHELL_REGISTRY_HOST: ghcr.io + OPENSHELL_REGISTRY_NAMESPACE: nvidia/openshell + OPENSHELL_REGISTRY_USERNAME: ${{ github.actor }} + OPENSHELL_CENTOS_SELINUX_VM_STATE_DIR: /tmp/openshell-e2e-centos + VM_BOOT_TIMEOUT_MINUTES: "10" + SSH_PORT: "2222" + GUEST_USER: e2e + GUEST_WORKDIR: /home/e2e/openshell + SSH_OPTS: -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -o BatchMode=yes + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ inputs['checkout-ref'] || github.sha }} + + - name: Record host facts + run: | + set -euo pipefail + cat /etc/os-release + uname -a + ls -l /dev/kvm || echo "/dev/kvm not present" + + - name: Install host VM tooling + run: | + set -euo pipefail + apt-get update + apt-get install -y --no-install-recommends \ + qemu-system-x86 \ + qemu-utils \ + genisoimage \ + openssh-client \ + curl \ + ca-certificates + rm -rf /var/lib/apt/lists/* + + - name: Create the CentOS Stream 10 VM + # VM_ACCEL=kvm makes this fail fast (instead of silently falling back + # to slow software emulation) if the runner doesn't expose /dev/kvm. + env: + VM_ACCEL: kvm + run: tasks/scripts/vm/centos-selinux-vm.sh create + + - name: Load guest connection details + run: | + set -euo pipefail + cat "${OPENSHELL_CENTOS_SELINUX_VM_STATE_DIR}/connection.env" >> "$GITHUB_ENV" + + - name: Copy repository into the guest + run: | + set -euo pipefail + tar -C "${GITHUB_WORKSPACE}" -czf /tmp/openshell-e2e-centos/repo.tar.gz --exclude=.git . + # shellcheck disable=SC2086 + scp -P "${SSH_PORT}" -i "${GUEST_KEY}" ${SSH_OPTS} \ + /tmp/openshell-e2e-centos/repo.tar.gz "${GUEST_USER}@127.0.0.1:/tmp/repo.tar.gz" + # shellcheck disable=SC2086 + ssh -p "${SSH_PORT}" -i "${GUEST_KEY}" ${SSH_OPTS} "${GUEST_USER}@127.0.0.1" \ + "mkdir -p '${GUEST_WORKDIR}' && tar -xzf /tmp/repo.tar.gz -C '${GUEST_WORKDIR}'" + + - name: Write guest environment file + run: | + set -euo pipefail + # shellcheck disable=SC2086,SC2087 + ssh -p "${SSH_PORT}" -i "${GUEST_KEY}" ${SSH_OPTS} "${GUEST_USER}@127.0.0.1" \ + "cat > '${GUEST_WORKDIR}/.e2e.env' && chmod 600 '${GUEST_WORKDIR}/.e2e.env'" <> "$GITHUB_OUTPUT" + + - name: Install build and test dependencies (guest) + run: | + # shellcheck disable=SC2086 + ssh -p "${SSH_PORT}" -i "${GUEST_KEY}" ${SSH_OPTS} "${GUEST_USER}@127.0.0.1" bash -s <<'REMOTE' + set -euo pipefail + sudo dnf install -y \ + ca-certificates \ + curl \ + git \ + gcc \ + gcc-c++ \ + make \ + clang \ + clang-devel \ + z3-devel \ + pkgconf-pkg-config \ + openssl-devel \ + openssh-clients \ + python3 \ + cmake \ + socat \ + unzip \ + xz \ + jq \ + rsync \ + zstd \ + audit \ + policycoreutils + REMOTE + + - name: Install Docker Engine and Podman (guest) + # CentOS Stream's GenericCloud image ships neither by default, per + # the RHEL-family cross-compatibility check requested in + # https://github.com/NVIDIA/OpenShell/pull/2092#issuecomment-4856873804. + run: | + # shellcheck disable=SC2086 + ssh -p "${SSH_PORT}" -i "${GUEST_KEY}" ${SSH_OPTS} "${GUEST_USER}@127.0.0.1" bash -s <<'REMOTE' + set -euo pipefail + curl -fsSL https://get.docker.com | sudo bash + sudo sed -i "s#/usr/bin/dockerd#/usr/bin/dockerd --ip-forward-no-drop#g" /usr/lib/systemd/system/docker.service + sudo systemctl daemon-reload + sudo systemctl enable --now docker + sudo dnf install -y podman + REMOTE + + - name: Grant guest user access to the Docker socket + run: | + # shellcheck disable=SC2086 + ssh -p "${SSH_PORT}" -i "${GUEST_KEY}" ${SSH_OPTS} "${GUEST_USER}@127.0.0.1" bash -s <<'REMOTE' + set -euo pipefail + sudo groupadd -f docker + sudo usermod -aG docker "$(id -un)" + sudo chmod 666 /var/run/docker.sock + REMOTE + + - name: Verify Docker and Podman coexist (guest) + run: | + # shellcheck disable=SC2086 + ssh -p "${SSH_PORT}" -i "${GUEST_KEY}" ${SSH_OPTS} "${GUEST_USER}@127.0.0.1" bash -s <<'REMOTE' + set -euo pipefail + docker version + docker info + podman version + podman info + REMOTE + + - name: Install mise (guest) + run: | + # shellcheck disable=SC2086 + ssh -p "${SSH_PORT}" -i "${GUEST_KEY}" ${SSH_OPTS} "${GUEST_USER}@127.0.0.1" bash -s <<'REMOTE' + set -euo pipefail + source ~/openshell/.e2e.env + curl https://mise.run | MISE_VERSION="${MISE_VERSION}" sh + REMOTE + + - name: Install tools (guest) + run: | + # shellcheck disable=SC2086 + ssh -p "${SSH_PORT}" -i "${GUEST_KEY}" ${SSH_OPTS} "${GUEST_USER}@127.0.0.1" bash -s <<'REMOTE' + set -euo pipefail + source ~/openshell/.e2e.env + export PATH="$HOME/.local/bin:$HOME/.local/share/mise/shims:$PATH" + cd ~/openshell + mise install --locked + REMOTE + + - name: Log in to GHCR with Docker (guest) + run: | + # shellcheck disable=SC2086 + ssh -p "${SSH_PORT}" -i "${GUEST_KEY}" ${SSH_OPTS} "${GUEST_USER}@127.0.0.1" bash -s <<'REMOTE' + set -euo pipefail + source ~/openshell/.e2e.env + echo "${OPENSHELL_REGISTRY_PASSWORD}" | docker login ghcr.io -u "${OPENSHELL_REGISTRY_USERNAME}" --password-stdin + REMOTE + + - name: Log in to GHCR with Podman (guest) + run: | + # shellcheck disable=SC2086 + ssh -p "${SSH_PORT}" -i "${GUEST_KEY}" ${SSH_OPTS} "${GUEST_USER}@127.0.0.1" bash -s <<'REMOTE' + set -euo pipefail + source ~/openshell/.e2e.env + echo "${OPENSHELL_REGISTRY_PASSWORD}" | podman login ghcr.io -u "${OPENSHELL_REGISTRY_USERNAME}" --password-stdin + REMOTE + + - name: Run Docker-backed e2e suite (guest) + run: | + # shellcheck disable=SC2086 + ssh -p "${SSH_PORT}" -i "${GUEST_KEY}" ${SSH_OPTS} "${GUEST_USER}@127.0.0.1" bash -s <<'REMOTE' + set -euo pipefail + source ~/openshell/.e2e.env + export PATH="$HOME/.local/bin:$HOME/.local/share/mise/shims:$PATH" + cd ~/openshell + mise run --no-deps --skip-deps e2e:docker + REMOTE + + - name: Run Podman-backed e2e suite (guest) + run: | + # shellcheck disable=SC2086 + ssh -p "${SSH_PORT}" -i "${GUEST_KEY}" ${SSH_OPTS} "${GUEST_USER}@127.0.0.1" bash -s <<'REMOTE' + set -euo pipefail + source ~/openshell/.e2e.env + export PATH="$HOME/.local/bin:$HOME/.local/share/mise/shims:$PATH" + cd ~/openshell + mise run --no-deps --skip-deps e2e:podman + REMOTE + + - name: Check for unexpected SELinux denials (guest) + env: + AUDIT_WINDOW_START: ${{ steps.audit_window.outputs.start }} + run: | + set -euo pipefail + # shellcheck disable=SC2086 + ssh -p "${SSH_PORT}" -i "${GUEST_KEY}" ${SSH_OPTS} "${GUEST_USER}@127.0.0.1" \ + "AUDIT_WINDOW_START='${AUDIT_WINDOW_START}' bash -s" <<'REMOTE' + set -euo pipefail + echo "Searching audit log for AVC denials since ${AUDIT_WINDOW_START}" + if sudo ausearch -m avc,user_avc -ts "${AUDIT_WINDOW_START}" 2>/dev/null | tee /tmp/avc-denials.log | grep -q .; then + echo "::error::SELinux denied one or more operations during the e2e suites; see the log below" + cat /tmp/avc-denials.log + exit 1 + fi + echo "No SELinux denials recorded." + REMOTE + + - name: Collect SELinux/audit context on failure (guest) + if: failure() + env: + AUDIT_WINDOW_START: ${{ steps.audit_window.outputs.start }} + run: | + set +e + # shellcheck disable=SC2086 + ssh -p "${SSH_PORT}" -i "${GUEST_KEY}" ${SSH_OPTS} "${GUEST_USER}@127.0.0.1" \ + "AUDIT_WINDOW_START='${AUDIT_WINDOW_START}' bash -s" <<'REMOTE' + echo "=== getenforce ===" + getenforce + echo "=== recent AVC denials ===" + sudo ausearch -m avc,user_avc -ts "${AUDIT_WINDOW_START}" 2>/dev/null + echo "=== dockerd status ===" + sudo systemctl status docker --no-pager + echo "=== recent kernel log ===" + sudo journalctl -k --no-pager | tail -n 200 + REMOTE + echo "=== host QEMU console log ===" + cat /tmp/openshell-e2e-centos/console.log + exit 0 + + - name: Destroy the CentOS Stream VM + if: always() + run: tasks/scripts/vm/centos-selinux-vm.sh destroy || true diff --git a/.github/workflows/e2e-label-help.yml b/.github/workflows/e2e-label-help.yml index 1190bcd3d..d5821fa08 100644 --- a/.github/workflows/e2e-label-help.yml +++ b/.github/workflows/e2e-label-help.yml @@ -19,7 +19,7 @@ permissions: {} jobs: hint: name: Post next-step hint for E2E label - if: github.event.label.name == 'test:e2e' || github.event.label.name == 'test:e2e-gpu' || github.event.label.name == 'test:e2e-kubernetes' + if: github.event.label.name == 'test:e2e' || github.event.label.name == 'test:e2e-gpu' || github.event.label.name == 'test:e2e-kubernetes' || github.event.label.name == 'test:e2e-centos' runs-on: ubuntu-latest permissions: pull-requests: write @@ -55,6 +55,11 @@ jobs: build_summary="gateway and supervisor images" status_summary="This is an optional proof-of-life suite; failures are visible in the workflow run but do not publish a required CI gate status." ;; + test:e2e-centos) + suite_summary="the CentOS Stream 10 Docker/Podman/SELinux compatibility suite" + build_summary="gateway and supervisor images" + status_summary="This is an optional suite that boots a VM inside the job; failures are visible in the workflow run but do not publish a required CI gate status." + ;; *) echo "Unrecognized label $LABEL_NAME"; exit 1 ;; esac diff --git a/CI.md b/CI.md index d04668aaf..5ab312dd3 100644 --- a/CI.md +++ b/CI.md @@ -10,15 +10,21 @@ PR CI that runs on NVIDIA self-hosted runners uses NVIDIA's copy-pr-bot. The bot `Branch Checks` run automatically after copy-pr-bot mirrors the PR. `Required CI Gates` posts PR-head statuses that verify the mirror exists, is current, and ran the expected push-based workflows. E2E suites are opt-in because they are more expensive and publish temporary images. -Three opt-in labels enable the long-running E2E suites: +Four opt-in labels enable the long-running E2E suites: - `test:e2e` runs the standard E2E suite in `Branch E2E Checks` - `test:e2e-gpu` runs GPU E2E in `Branch E2E Checks` - `test:e2e-kubernetes` runs Kubernetes E2E with the HA Helm overlay (`replicaCount: 2` and bundled PostgreSQL) in `Branch E2E Checks` +- `test:e2e-centos` runs the CentOS Stream 10 Docker/Podman/SELinux + compatibility suite in `Branch E2E Checks`. It boots a CentOS Stream 10 + VM under QEMU/KVM on the runner, installs Docker alongside Podman, and + runs both `e2e:docker` and `e2e:podman` against a real SELinux-enforcing + kernel. Reproduce it locally with `mise run vm:centos-selinux:create` + (see `tasks/scripts/vm/centos-selinux-vm.sh`). When multiple labels are present, `Branch E2E Checks` builds the shared gateway and supervisor images once and fans out all enabled suites in parallel. -The `OpenShell / E2E` and `OpenShell / GPU E2E` required statuses are evaluated from separate suite result jobs inside that workflow. `test:e2e-kubernetes` is optional while HA behavior is under active iteration: failures are visible in the workflow run but do not publish a required CI gate status. +The `OpenShell / E2E` and `OpenShell / GPU E2E` required statuses are evaluated from separate suite result jobs inside that workflow. `test:e2e-kubernetes` and `test:e2e-centos` are optional: failures are visible in the workflow run but do not publish a required CI gate status. The GitHub ruleset should require the `OpenShell / ...` statuses published by `Required CI Gates`, not the push-triggered workflow jobs directly. @@ -71,7 +77,7 @@ Flow: 1. Open the PR. copy-pr-bot mirrors it to `pull-request/` automatically. 2. The mirror push runs `Branch Checks` automatically. `Required CI Gates` keeps the PR blocked until the mirror exists, matches the PR head SHA, and the required push-based workflow succeeds. The first `Branch E2E Checks` run only resolves metadata and skips expensive jobs unless an E2E label is already set. -3. A maintainer applies `test:e2e`, `test:e2e-gpu`, and/or `test:e2e-kubernetes`. `E2E Label Help` posts a comment with a link to the existing gated workflow run. +3. A maintainer applies `test:e2e`, `test:e2e-gpu`, `test:e2e-kubernetes`, and/or `test:e2e-centos`. `E2E Label Help` posts a comment with a link to the existing gated workflow run. 4. The maintainer opens that link and clicks **Re-run all jobs**. This time `pr_metadata` sees the label and the build/E2E jobs run. 5. When the run finishes, the matching `OpenShell / ...` gate status flips to green automatically. 6. New commits push to the mirror automatically and re-trigger `Branch Checks` plus any labeled E2E jobs in `Branch E2E Checks`. @@ -110,7 +116,7 @@ The bot's full administrator documentation is internal to NVIDIA. The only comma | File | Role | |---|---| | `.github/workflows/branch-checks.yml` | Required non-E2E PR checks. Triggers on `push: pull-request/[0-9]+`. | -| `.github/workflows/branch-e2e.yml` | Opt-in standard, GPU, and Kubernetes HA E2E. Triggers on `push: pull-request/[0-9]+` and runs jobs selected by `test:e2e`, `test:e2e-gpu`, or `test:e2e-kubernetes`. | +| `.github/workflows/branch-e2e.yml` | Opt-in standard, GPU, Kubernetes HA, and CentOS Stream 10 SELinux E2E. Triggers on `push: pull-request/[0-9]+` and runs jobs selected by `test:e2e`, `test:e2e-gpu`, `test:e2e-kubernetes`, or `test:e2e-centos`. | | `.github/workflows/helm-lint.yml` | Helm chart validation. Triggers on `push: pull-request/[0-9]+` and skips lint jobs unless Helm inputs changed. | | `.github/actions/pr-gate/action.yml` | Composite action that resolves PR metadata and verifies the required label is set. | | `.github/actions/pr-merge-base/action.yml` | Composite action that resolves and fetches the merge-base commit for `pull-request/` push workflows. | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b78f36522..b66f9c353 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -305,4 +305,4 @@ DCO sign-off is separate from cryptographic commit signing. CI requires signing ## CI -How PR CI runs, the `test:e2e`, `test:e2e-gpu`, and `test:e2e-kubernetes` labels, copy-pr-bot, and commit-signing setup are documented in [CI.md](CI.md). +How PR CI runs, the `test:e2e`, `test:e2e-gpu`, `test:e2e-kubernetes`, and `test:e2e-centos` labels, copy-pr-bot, and commit-signing setup are documented in [CI.md](CI.md). diff --git a/architecture/build.md b/architecture/build.md index a3cb2e25f..ef1a5040b 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -144,7 +144,7 @@ The high-level CI model: 1. PR-context gate jobs publish required statuses for the PR head commit. 2. Standard branch checks run from trusted mirror branches. -3. Label-gated E2E, GPU, and Kubernetes checks run from trusted mirror branches. +3. Label-gated E2E, GPU, Kubernetes, and CentOS Stream/SELinux compatibility checks run from trusted mirror branches. 4. Gate jobs verify that the mirror branch matches the PR head and that the expected non-gate workflow actually ran. 5. Release workflows rebuild and publish binaries, wheels, images, and docs. diff --git a/tasks/scripts/vm/centos-selinux-vm.sh b/tasks/scripts/vm/centos-selinux-vm.sh new file mode 100755 index 000000000..7117375c1 --- /dev/null +++ b/tasks/scripts/vm/centos-selinux-vm.sh @@ -0,0 +1,386 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Create, inspect, and tear down a CentOS Stream 10 VM under QEMU for +# Docker/Podman/SELinux compatibility testing. +# +# RHEL-family distros (RHEL, CentOS Stream) ship Podman but not Docker, and +# enforce SELinux by default — unlike the Ubuntu-based +# ghcr.io/nvidia/openshell/ci image our other e2e lanes run in. This script +# boots a real CentOS Stream 10 guest under hardware-accelerated QEMU so both +# engines, and any SELinux-sensitive behavior (e.g. bind-mount `:z`/`:Z` +# relabeling), can be exercised against a kernel that actually enforces +# SELinux. +# +# This is the same script `.github/workflows/e2e-centos-selinux.yml` uses to +# boot the guest in CI. Run it locally to reproduce that environment: +# +# mise run vm:centos-selinux:create +# mise run vm:centos-selinux:ssh +# # ... install/exercise docker, podman, mise run e2e:docker, etc ... +# mise run vm:centos-selinux:destroy +# +# Usage: +# centos-selinux-vm.sh create [--force] +# centos-selinux-vm.sh destroy +# centos-selinux-vm.sh status +# centos-selinux-vm.sh ssh [-- ...] +# +# Environment overrides (all optional): +# OPENSHELL_CENTOS_SELINUX_VM_STATE_DIR State dir (default: /.cache/vm/centos-selinux) +# CENTOS_IMAGE_URL CentOS Stream 10 GenericCloud qcow2 image URL +# VM_MEMORY_MB Guest memory in MB (default: 8192) +# VM_CPUS Guest vCPUs (default: 4) +# VM_DISK_GB Guest root disk size in GB after resize (default: 40) +# SSH_PORT Host TCP port forwarded to the guest SSH daemon (default: 2222) +# VM_BOOT_TIMEOUT_MINUTES Minutes to wait for guest SSH to come up (default: 10) +# VM_ACCEL Force QEMU accelerator: kvm, hvf, or tcg (default: auto-detect) +# GUEST_USER Guest username created via cloud-init (default: e2e) +# +# The base cloud image is downloaded once and cached; each `create` boots a +# fresh copy-on-write overlay disk, so repeated local runs don't re-download +# or mutate the cached base image. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tasks/scripts/vm/_lib.sh +source "${SCRIPT_DIR}/_lib.sh" +ROOT="$(vm_lib_root)" + +STATE_DIR="${OPENSHELL_CENTOS_SELINUX_VM_STATE_DIR:-${ROOT}/.cache/vm/centos-selinux}" +CENTOS_IMAGE_URL="${CENTOS_IMAGE_URL:-https://cloud.centos.org/centos/10-stream/x86_64/images/CentOS-Stream-GenericCloud-10-latest.x86_64.qcow2}" +VM_MEMORY_MB="${VM_MEMORY_MB:-8192}" +VM_CPUS="${VM_CPUS:-4}" +VM_DISK_GB="${VM_DISK_GB:-40}" +SSH_PORT="${SSH_PORT:-2222}" +VM_BOOT_TIMEOUT_MINUTES="${VM_BOOT_TIMEOUT_MINUTES:-10}" +VM_ACCEL="${VM_ACCEL:-auto}" +GUEST_USER="${GUEST_USER:-e2e}" +GUEST_NAME="openshell-e2e-centos" + +BASE_IMAGE="${STATE_DIR}/centos-base.qcow2" +DISK_IMAGE="${STATE_DIR}/disk.qcow2" +SEED_ISO="${STATE_DIR}/seed.iso" +GUEST_KEY="${STATE_DIR}/guest_ed25519" +PID_FILE="${STATE_DIR}/qemu.pid" +CONSOLE_LOG="${STATE_DIR}/console.log" +CONNECTION_ENV="${STATE_DIR}/connection.env" +SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -o BatchMode=yes) + +usage() { + sed -n '2,40p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' +} + +log() { echo "==> $*"; } +err() { echo "ERROR: $*" >&2; } + +require_tool() { + local tool="$1" hint="$2" + if ! command -v "${tool}" >/dev/null 2>&1; then + err "missing required tool '${tool}'." + echo " ${hint}" >&2 + exit 1 + fi +} + +check_dependencies() { + require_tool qemu-img "Install QEMU (e.g. 'sudo apt-get install qemu-utils' or 'brew install qemu')." + require_tool qemu-system-x86_64 "Install QEMU (e.g. 'sudo apt-get install qemu-system-x86' or 'brew install qemu')." + require_tool ssh "Install an OpenSSH client." + require_tool ssh-keygen "Install an OpenSSH client." + require_tool curl "Install curl." + if ! command -v genisoimage >/dev/null 2>&1 \ + && ! command -v mkisofs >/dev/null 2>&1 \ + && ! command -v xorriso >/dev/null 2>&1; then + err "missing an ISO-building tool (genisoimage, mkisofs, or xorriso)." + echo " Install one, e.g. 'sudo apt-get install genisoimage' or 'brew install cdrtools'." >&2 + exit 1 + fi +} + +sha256_file() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +build_seed_iso() { + local out="$1" volid="$2" dir="$3" + if command -v genisoimage >/dev/null 2>&1; then + genisoimage -output "${out}" -volid "${volid}" -joliet -rock "${dir}/user-data" "${dir}/meta-data" >/dev/null + elif command -v mkisofs >/dev/null 2>&1; then + mkisofs -output "${out}" -volid "${volid}" -joliet -rock "${dir}/user-data" "${dir}/meta-data" >/dev/null + else + xorriso -as genisoimage -output "${out}" -volid "${volid}" -joliet -rock "${dir}/user-data" "${dir}/meta-data" >/dev/null + fi +} + +detect_accel() { + if [ "${VM_ACCEL}" != "auto" ]; then + echo "${VM_ACCEL}" + return + fi + case "$(uname -s)" in + Linux) + if [ -c /dev/kvm ] && [ -r /dev/kvm ] && [ -w /dev/kvm ]; then + echo "kvm" + else + echo "tcg" + fi + ;; + Darwin) + if [ "$(uname -m)" = "x86_64" ]; then + echo "hvf" + else + echo "tcg" + fi + ;; + *) + echo "tcg" + ;; + esac +} + +vm_pid_alive() { + [ -f "${PID_FILE}" ] && kill -0 "$(cat "${PID_FILE}")" 2>/dev/null +} + +download_base_image() { + mkdir -p "${STATE_DIR}" + if [ -f "${BASE_IMAGE}" ]; then + log "Using cached base image: ${BASE_IMAGE}" + return + fi + log "Downloading CentOS Stream 10 GenericCloud image..." + local tmp="${BASE_IMAGE}.part" + curl -fsSL -o "${tmp}" "${CENTOS_IMAGE_URL}" + if curl -fsSL -o "${STATE_DIR}/centos-base.qcow2.SHA256SUM" "${CENTOS_IMAGE_URL}.SHA256SUM" 2>/dev/null; then + local expected actual + expected="$(grep -oE '[0-9a-f]{64}' "${STATE_DIR}/centos-base.qcow2.SHA256SUM" | head -n1)" + actual="$(sha256_file "${tmp}")" + if [ "${expected}" != "${actual}" ]; then + rm -f "${tmp}" + err "CentOS Stream image checksum mismatch (expected ${expected}, got ${actual})" + exit 1 + fi + log "Verified image checksum: ${actual}" + else + echo "WARNING: no SHA256SUM published alongside the image; skipping checksum verification" >&2 + fi + mv "${tmp}" "${BASE_IMAGE}" +} + +ensure_ssh_key() { + if [ -f "${GUEST_KEY}" ]; then + return + fi + log "Generating ephemeral guest SSH keypair..." + ssh-keygen -t ed25519 -N "" -C "openshell-e2e-centos" -f "${GUEST_KEY}" >/dev/null +} + +build_overlay_disk() { + log "Creating ${VM_DISK_GB}G writable overlay disk from cached base image..." + rm -f "${DISK_IMAGE}" + qemu-img create -f qcow2 -F qcow2 -b "${BASE_IMAGE}" "${DISK_IMAGE}" >/dev/null + qemu-img resize "${DISK_IMAGE}" "${VM_DISK_GB}G" >/dev/null +} + +# Written line-by-line (not an indented heredoc) so the generated files have +# no leading whitespace: cloud-init requires "#cloud-config" to be the exact +# first bytes of user-data to recognize the format. +build_cloud_init_seed() { + log "Building cloud-init seed image..." + local guest_pubkey + guest_pubkey="$(cat "${GUEST_KEY}.pub")" + { + echo "instance-id: ${GUEST_NAME}" + echo "local-hostname: ${GUEST_NAME}" + } >"${STATE_DIR}/meta-data" + { + echo "#cloud-config" + echo "hostname: openshell-e2e-guest" + echo "manage_etc_hosts: true" + echo "disable_root: true" + echo "ssh_pwauth: false" + echo "users:" + echo " - name: ${GUEST_USER}" + echo " gecos: OpenShell E2E" + echo " groups: [wheel]" + echo " sudo: [\"ALL=(ALL) NOPASSWD:ALL\"]" + echo " shell: /bin/bash" + echo " lock_passwd: true" + echo " ssh_authorized_keys:" + echo " - ${guest_pubkey}" + echo "package_update: false" + echo "package_upgrade: false" + } >"${STATE_DIR}/user-data" + head -n1 "${STATE_DIR}/user-data" | grep -qx '#cloud-config' + build_seed_iso "${SEED_ISO}" cidata "${STATE_DIR}" +} + +boot_vm() { + local accel + accel="$(detect_accel)" + if [ "${accel}" = "kvm" ] && { [ ! -c /dev/kvm ] || [ ! -r /dev/kvm ] || [ ! -w /dev/kvm ]; }; then + err "/dev/kvm is not available or not accessible." + echo " This host cannot provide hardware-accelerated KVM. Set VM_ACCEL=tcg to fall back" >&2 + echo " to (slow) software emulation, or run on a host/container with /dev/kvm exposed." >&2 + exit 1 + fi + if [ "${accel}" = "tcg" ]; then + echo "WARNING: no hardware acceleration available; booting with software emulation (tcg)." >&2 + echo " This will be significantly slower to boot and run." >&2 + fi + + local cpu_flag="host" + [ "${accel}" = "tcg" ] && cpu_flag="max" + + log "Booting CentOS Stream 10 VM (accel=${accel})..." + # -display none + -monitor none (rather than -nographic) because + # -nographic redirects the serial/monitor console to the controlling tty, + # which QEMU refuses to combine with -daemonize ("cannot be used with + # -daemonize"). -serial file:... captures the console independently. + qemu-system-x86_64 \ + -name "${GUEST_NAME}" \ + -machine "q35,accel=${accel}" -cpu "${cpu_flag}" \ + -smp "${VM_CPUS}" -m "${VM_MEMORY_MB}" \ + -drive file="${DISK_IMAGE}",if=virtio,format=qcow2 \ + -drive file="${SEED_ISO}",if=virtio,format=raw \ + -netdev "user,id=net0,hostfwd=tcp::${SSH_PORT}-:22" \ + -device virtio-net-pci,netdev=net0 \ + -display none \ + -monitor none \ + -serial file:"${CONSOLE_LOG}" \ + -pidfile "${PID_FILE}" \ + -daemonize +} + +wait_for_guest_ssh() { + log "Waiting for guest SSH on 127.0.0.1:${SSH_PORT} (timeout: ${VM_BOOT_TIMEOUT_MINUTES}m)..." + local deadline=$(($(date +%s) + VM_BOOT_TIMEOUT_MINUTES * 60)) + until ssh -p "${SSH_PORT}" -i "${GUEST_KEY}" "${SSH_OPTS[@]}" "${GUEST_USER}@127.0.0.1" true 2>/dev/null; do + if [ "$(date +%s)" -ge "${deadline}" ]; then + err "guest SSH did not become reachable within ${VM_BOOT_TIMEOUT_MINUTES} minutes." + echo "=== console log (${CONSOLE_LOG}) ===" >&2 + tail -n 200 "${CONSOLE_LOG}" >&2 2>/dev/null || true + exit 1 + fi + sleep 5 + done + log "Guest SSH is reachable." +} + +write_connection_env() { + cat >"${CONNECTION_ENV}" </dev/null || true + fi + if [ -f "${PID_FILE}" ]; then + local pid + pid="$(cat "${PID_FILE}")" + for _ in $(seq 1 20); do + kill -0 "${pid}" 2>/dev/null || break + sleep 1 + done + kill -9 "${pid}" 2>/dev/null || true + rm -f "${PID_FILE}" + fi + rm -f "${DISK_IMAGE}" "${SEED_ISO}" "${CONSOLE_LOG}" "${CONNECTION_ENV}" \ + "${STATE_DIR}/user-data" "${STATE_DIR}/meta-data" + log "VM stopped. Cached base image and guest SSH key preserved under ${STATE_DIR}." +} + +cmd_ssh() { + if ! vm_pid_alive; then + err "VM is not running. Run 'centos-selinux-vm.sh create' first." + exit 1 + fi + if [ "$#" -gt 0 ] && [ "$1" = "--" ]; then + shift + fi + exec ssh -p "${SSH_PORT}" -i "${GUEST_KEY}" "${SSH_OPTS[@]}" "${GUEST_USER}@127.0.0.1" "$@" +} + +main() { + local subcommand="${1:-}" + [ "$#" -gt 0 ] && shift || true + case "${subcommand}" in + create) cmd_create "$@" ;; + destroy) cmd_destroy "$@" ;; + status) cmd_status "$@" ;; + ssh) cmd_ssh "$@" ;; + -h|--help|help|"") usage ;; + *) + err "unknown subcommand '${subcommand}'" + usage + exit 2 + ;; + esac +} + +main "$@" diff --git a/tasks/vm.toml b/tasks/vm.toml index 538168c14..d6c2b85dc 100644 --- a/tasks/vm.toml +++ b/tasks/vm.toml @@ -11,6 +11,11 @@ # # See tasks/gateway.toml for `gateway:vm`, # crates/openshell-driver-vm/README.md for the VM driver workflow. +# +# `vm:centos-selinux:*` is unrelated to the driver above: it boots a CentOS +# Stream 10 guest under QEMU for the Docker/Podman/SELinux compatibility +# suite (`.github/workflows/e2e-centos-selinux.yml`), and can be reproduced +# locally with the same script. # ═══════════════════════════════════════════════════════════════════════════ # Main Commands @@ -27,3 +32,23 @@ run = "tasks/scripts/vm/build-supervisor-bundle.sh" ["vm:smoke:orphan-cleanup"] description = "Smoke test: start gateway+driver, create a sandbox, signal the gateway, assert no orphaned processes survive" run = "tasks/scripts/vm/smoke-orphan-cleanup.sh" + +# ═══════════════════════════════════════════════════════════════════════════ +# CentOS Stream 10 Docker/Podman/SELinux compatibility VM +# ═══════════════════════════════════════════════════════════════════════════ + +["vm:centos-selinux:create"] +description = "Boot a CentOS Stream 10 VM under QEMU for Docker/Podman/SELinux compatibility testing (see e2e-centos-selinux.yml)" +run = "tasks/scripts/vm/centos-selinux-vm.sh create" + +["vm:centos-selinux:status"] +description = "Show whether the CentOS Stream 10 SELinux compat VM is running" +run = "tasks/scripts/vm/centos-selinux-vm.sh status" + +["vm:centos-selinux:ssh"] +description = "SSH into the running CentOS Stream 10 SELinux compat VM" +run = "tasks/scripts/vm/centos-selinux-vm.sh ssh" + +["vm:centos-selinux:destroy"] +description = "Power off and remove the CentOS Stream 10 SELinux compat VM" +run = "tasks/scripts/vm/centos-selinux-vm.sh destroy"