From 0df9995c888fb41fc54edd0947da436ad5300668 Mon Sep 17 00:00:00 2001 From: mhsong2 Date: Wed, 9 Sep 2026 17:31:07 +0800 Subject: [PATCH] feat: support xpu devices in runc sandboxes Package optional native runc and accelerator support across node images, Helm, Terraform, standalone deployment, and the Python SDK. Support NVIDIA GPUs with runsc or runc and physical Ascend NPUs with runc. Install Ascend integration from a checksum-pinned standalone adapter release while keeping the trusted mount profile in sandboxd. Update sandboxd to the matching external-adapter protocol implementation. Signed-off-by: mhsong2 --- .gitmodules | 2 +- AGENTS.md | 3 +- Makefile | 4 ++ README.md | 24 ++++++-- builder/node.Dockerfile | 55 ++++++++++++++++- deploy/README.md | 17 ++++++ .../charts/core/templates/node/configmap.yaml | 14 +++++ .../charts/core/templates/node/daemonset.yaml | 36 +++++++++++ deploy/akernel/charts/core/values.yaml | 8 +++ deploy/scripts/build-image.sh | 22 +++++++ deploy/scripts/configure.sh | 13 ++++ deploy/standalone/README.md | 25 +++++++- deploy/standalone/config/sandboxd_config.toml | 2 + deploy/standalone/start.sh | 59 +++++++++++++++++++ deploy/terraform/aliyun/main.tf | 1 + .../terraform/aliyun/terraform.tfvars.example | 4 ++ .../terraform/aliyun/values-akernel.yaml.tmpl | 3 + deploy/terraform/aliyun/variables.tf | 6 ++ deploy/terraform/huaweicloud/main.tf | 1 + .../huaweicloud/terraform.tfvars.example | 4 ++ .../huaweicloud/values-akernel.yaml.tmpl | 3 + deploy/terraform/huaweicloud/variables.tf | 6 ++ sdk/python/README.md | 23 +++++--- sdk/python/akernel_sdk/_sandbox_resources.py | 4 +- sdk/python/akernel_sdk/sandbox.py | 6 +- sdk/python/examples/gpu_sandbox.py | 9 ++- sdk/python/tests/unit/test_sandbox.py | 2 +- .../tests/unit/test_sandbox_resources.py | 4 ++ src/sandboxd | 2 +- 29 files changed, 334 insertions(+), 28 deletions(-) diff --git a/.gitmodules b/.gitmodules index 97ae8b8..6a898d0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "src/sandboxd"] path = src/sandboxd - url = https://github.com/inclusionAI/sandboxd.git + url = https://github.com/mhsong1998-dot/sandboxd.git [submodule "src/distill-fs"] path = src/distill-fs url = https://github.com/inclusionAI/distill-fs.git diff --git a/AGENTS.md b/AGENTS.md index fca1d2a..63fa268 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,8 +18,7 @@ enabled by an operator. Creation-time network policies and atomic runtime replacement support unrestricted networking, blocking new flows except the YuanRong control and published sandbox-port routes, or denying exact and leading-wildcard DNS names. -Experimental whole-device NVIDIA GPU requests require runsc. Configurable -writable-storage requests are supported by runsc and Firecracker. +Experimental whole-device NVIDIA GPU requests support runsc or explicitly enabled runc. Physical Ascend 310P and 910 A2/A3 requests require explicitly enabled runc and Ascend payloads. Accelerator model tokens match normalized driver-reported inventory; Ascend has no per-SKU allowlist. The optional FlowGW `/var/queue_schedule` mount is detected in standalone mode and opt-in with Helm `node.config.sandboxd.mountAscendQueueSchedule`. Configurable writable-storage requests are supported by runsc and Firecracker. Use AKernel when a task needs an isolated remote environment with command execution, file operations, interactive PTYs, port forwarding, or reverse diff --git a/Makefile b/Makefile index 100218d..84e6ac8 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,7 @@ MONITOR_STORAGE_CLASS ?= INSTALL_MONITOR ?= INSTALL_DRAGONFLY ?= ENABLE_RUNC ?= +ENABLE_ASCEND ?= GRAFANA_PUBLIC_ACCESS ?= GRAFANA_ADMIN_PASSWORD ?= IAM_SEED_HEX ?= @@ -49,12 +50,14 @@ help: @echo " make config NON_INTERACTIVE=1 ... Generate config from Make variables" @echo " make config INSTALL_DRAGONFLY=true Enable optional P2P image distribution" @echo " make config ENABLE_RUNC=true Build and register the optional runc runtime" + @echo " make config ENABLE_RUNC=true ENABLE_ASCEND=true Enable Ascend NPU nodes" @echo " make build IMAGE_TAG= Build the all-in-one image" @echo " make build RUNTIME_PROFILE=python Include optional Python runtimes" @echo " make build AKERNEL_ENABLE_KATA=false Exclude the optional Kata payload" @echo " make build AKERNEL_ENABLE_FIRECRACKER=false Exclude Firecracker" @echo " make build AKERNEL_ENABLE_RUNC=true Include the optional runc payload" @echo " make build RRT_RUNTIME_URL=... RRT_RUNTIME_SHA256=... Override RRT artifact" + @echo " make build AKERNEL_ENABLE_RUNC=true AKERNEL_ENABLE_ASCEND=true Include Ascend NPU support" @echo " make versions Show locally selected component versions" @echo " make push Push the configured all-in-one image" @echo " make plan Terraform plan" @@ -92,6 +95,7 @@ config: if [[ -n "$(INSTALL_MONITOR)" ]]; then args+=(--install-monitor "$(INSTALL_MONITOR)"); fi; \ if [[ -n "$(INSTALL_DRAGONFLY)" ]]; then args+=(--install-dragonfly "$(INSTALL_DRAGONFLY)"); fi; \ if [[ -n "$(ENABLE_RUNC)" ]]; then args+=(--enable-runc "$(ENABLE_RUNC)"); fi; \ + if [[ -n "$(ENABLE_ASCEND)" ]]; then args+=(--enable-ascend "$(ENABLE_ASCEND)"); fi; \ if [[ -n "$(GRAFANA_PUBLIC_ACCESS)" ]]; then args+=(--grafana-public-access "$(GRAFANA_PUBLIC_ACCESS)"); fi; \ if [[ -n "$(GRAFANA_ADMIN_PASSWORD)" ]]; then args+=(--grafana-admin-password "$(GRAFANA_ADMIN_PASSWORD)"); fi; \ if [[ -n "$(IAM_SEED_HEX)" ]]; then args+=(--iam-seed-hex "$(IAM_SEED_HEX)"); fi; \ diff --git a/README.md b/README.md index 2f54892..e4417c1 100644 --- a/README.md +++ b/README.md @@ -164,16 +164,30 @@ with Sandbox(cpu=1000, memory=2048) as sandbox: print(sandbox.files.read("/tmp/hello.txt")) ``` -Experimental gVisor sandboxes can request an exact NVIDIA GPU model: +Experimental runsc or runc sandboxes can request an exact NVIDIA GPU model: ```python -with Sandbox(xpu="gpu:l20:1") as sandbox: +with Sandbox(runtime="runc", xpu="gpu:l20:1") as sandbox: print(sandbox.commands.run("nvidia-smi -L").stdout) ``` -GPU sandboxes require a compatible NVIDIA node and the gVisor `runsc` -runtime. `storage_mb` is measured in MiB and is supported by `runsc` and -Firecracker. +GPU sandboxes require a compatible NVIDIA node and either the gVisor `runsc` +runtime or native `runc`. Runsc remains subject to its nvproxy driver-version +compatibility check; that check does not disable native-runc GPU support. +`storage_mb` is measured in MiB and is supported by `runsc` and Firecracker. + +Physical Ascend 310P and 910 A2/A3 sandboxes use the native `runc` runtime and the normalized model reported by the driver: + +```python +with Sandbox(runtime="runc", xpu="npu:ascend910b4:1") as sandbox: + print(sandbox.commands.run("npu-smi info").stdout) +``` + +Atlas 300I Pro nodes reporting `ascend310p3` use `npu:ascend310p3:1`. If an A3 node reports `ascend910`, use `npu:ascend910:1`; no per-SKU model allowlist or inferred model alias is applied. + +The node image and deployment must enable both runc and Ascend support. The +node-local provider discovers physical devices, owns exclusive leases, and +injects only the leased device nodes and versioned read-only driver mounts. See the complete [basic usage example](./sdk/python/examples/basic_usage.py), the [sandbox runtime example](./sdk/python/examples/sandbox_runtime.py), and the other [SDK examples](./sdk/python/examples/) for more operations. diff --git a/builder/node.Dockerfile b/builder/node.Dockerfile index 19669be..bea7d0d 100644 --- a/builder/node.Dockerfile +++ b/builder/node.Dockerfile @@ -7,6 +7,10 @@ ARG AKERNEL_RUNTIME_IMAGE=akernel-runtime:local ARG AKERNEL_RUNTIME_PROFILE=rrt ARG AKERNEL_ENABLE_KATA=true ARG AKERNEL_ENABLE_RUNC=false +ARG AKERNEL_ENABLE_ASCEND=false +ARG ASCEND_ADAPTER_VERSION= +ARG ASCEND_ADAPTER_AMD64_URL= +ARG ASCEND_ADAPTER_AMD64_SHA256= ARG AKERNEL_ENABLE_FIRECRACKER=true ARG SANDBOXD_BUILD_IMAGE=golang:1.25.5-bookworm ARG DISTILL_FS_BUILD_IMAGE=rust:1.85.0-bookworm @@ -114,6 +118,43 @@ WORKDIR /src/sandboxd COPY ./src/sandboxd/ ./ RUN make release +FROM ubuntu:24.04 AS ascend-adapter-true +ARG TARGETARCH +ARG ASCEND_ADAPTER_VERSION +ARG ASCEND_ADAPTER_AMD64_URL +ARG ASCEND_ADAPTER_AMD64_SHA256 +RUN apt-get update && \ + apt-get install -y --no-install-recommends ca-certificates curl && \ + rm -rf /var/lib/apt/lists/* +COPY ./src/sandboxd/configs/ascend/mounts.json /tmp/ascend-mounts.json +RUN set -eux; \ + test "${TARGETARCH:-amd64}" = "amd64"; \ + test -n "${ASCEND_ADAPTER_VERSION}"; \ + test -n "${ASCEND_ADAPTER_AMD64_URL}"; \ + test -n "${ASCEND_ADAPTER_AMD64_SHA256}"; \ + archive=/tmp/ascend-oci-adapter.tar.gz; \ + curl -fSL --retry 10 --retry-delay 2 --retry-all-errors \ + "${ASCEND_ADAPTER_AMD64_URL}" -o "${archive}"; \ + echo "${ASCEND_ADAPTER_AMD64_SHA256} ${archive}" | sha256sum -c -; \ + bundle="/tmp/ascend-oci-adapter_${ASCEND_ADAPTER_VERSION}_linux_amd64"; \ + tar -xzf "${archive}" -C /tmp; \ + test -x "${bundle}/bin/ascend-oci-adapter"; \ + test -f "${bundle}/licenses/ascend-oci-adapter/LICENSE"; \ + test -f "${bundle}/licenses/mind-cluster/LICENSE"; \ + test -f "${bundle}/licenses/mind-cluster/Third_Party_Open_Source_Software_Notice.md"; \ + install -D -m 0755 "${bundle}/bin/ascend-oci-adapter" \ + /ascend/usr/local/libexec/akernel/ascend-oci-adapter; \ + install -D -m 0644 /tmp/ascend-mounts.json \ + /ascend/etc/akernel/ascend/mounts.json; \ + mkdir -p /ascend/opt/akernel/licenses; \ + cp -a "${bundle}/licenses/." /ascend/opt/akernel/licenses/; \ + rm -rf "${archive}" "${bundle}" + +FROM ${SANDBOXD_BUILD_IMAGE} AS ascend-adapter-false +RUN mkdir -p /ascend/usr/local/libexec/akernel + +FROM ascend-adapter-${AKERNEL_ENABLE_ASCEND} AS ascend-adapter + FROM ${FIRECRACKER_BUILD_IMAGE} AS firecracker-runtime-true ARG FIRECRACKER_RELEASE ARG FIRECRACKER_AMD64_SHA256 @@ -216,6 +257,7 @@ RUN cargo build --locked --release --bin distill_fs FROM ${AKERNEL_NODE_BASE_IMAGE} ARG AKERNEL_ENABLE_KATA ARG AKERNEL_ENABLE_RUNC +ARG AKERNEL_ENABLE_ASCEND ARG AKERNEL_ENABLE_FIRECRACKER ARG AKERNEL_RUNTIME_PROFILE ARG AKERNEL_VERSION @@ -272,7 +314,10 @@ RUN set -eux; \ apt-get update; \ apt-get install -y --no-install-recommends \ "libnvidia-container1=${LIBNVIDIA_CONTAINER_VERSION}" \ - "libnvidia-container-tools=${LIBNVIDIA_CONTAINER_VERSION}"; \ + "libnvidia-container-tools=${LIBNVIDIA_CONTAINER_VERSION}" \ + "nvidia-container-toolkit=${LIBNVIDIA_CONTAINER_VERSION}"; \ + command -v nvidia-container-cli; \ + test -x /usr/bin/nvidia-container-runtime-hook; \ rm -rf /var/lib/apt/lists/* RUN if command -v update-alternatives >/dev/null 2>&1; then \ @@ -351,6 +396,7 @@ COPY --from=distill-fs-builder /src/distill-fs/target/release/distill_fs /usr/lo COPY --from=kata-runtime /kata/opt/kata /opt/kata COPY --from=runc-runtime /runc/usr/local/bin/ /usr/local/bin/ COPY --from=firecracker-runtime /firecracker/ / +COPY --from=ascend-adapter /ascend/ / RUN if [ "${AKERNEL_ENABLE_KATA}" = "true" ]; then \ ln -sf /opt/kata/runtime-rs/bin/containerd-shim-kata-v2 /usr/local/bin/containerd-shim-kata-v2; \ fi @@ -374,6 +420,12 @@ RUN if [ "${AKERNEL_ENABLE_RUNC}" = "true" ]; then \ test ! -e /usr/local/bin/runc; \ test ! -e /usr/local/bin/runc-shim; \ fi +RUN if [ "${AKERNEL_ENABLE_ASCEND}" = "true" ]; then \ + test "${AKERNEL_ENABLE_RUNC}" = "true"; \ + chmod 0755 /usr/local/libexec/akernel/ascend-oci-adapter; \ + else \ + test ! -e /usr/local/libexec/akernel/ascend-oci-adapter; \ + fi COPY ./builder/config/yr_services.yaml /tmp/yr_services_rrt.yaml COPY ./builder/config/yr_services_python.yaml /tmp/yr_services_python.yaml @@ -416,6 +468,7 @@ LABEL org.opencontainers.image.version="${AKERNEL_VERSION}" \ org.akernel.gvisor.release="${GVISOR_RELEASE}" \ org.akernel.runc.version="${RUNC_VERSION}" \ org.akernel.runc.enabled="${AKERNEL_ENABLE_RUNC}" \ + org.akernel.ascend.enabled="${AKERNEL_ENABLE_ASCEND}" \ org.akernel.kata.enabled="${AKERNEL_ENABLE_KATA}" \ org.akernel.firecracker.release="${FIRECRACKER_RELEASE}" \ org.akernel.firecracker.enabled="${AKERNEL_ENABLE_FIRECRACKER}" diff --git a/deploy/README.md b/deploy/README.md index 9ba6a48..f09f105 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -44,6 +44,23 @@ Terraform registers the runtime with sandboxd. Direct Helm users must likewise build with `AKERNEL_ENABLE_RUNC=true` and set `node.config.sandboxd.enableRunc=true`. +NVIDIA GPU sandboxes can use runsc or runc. The node container must include +the NVIDIA Container Toolkit hook and must be launched with access to the host +GPU devices and userspace driver libraries. Runc GPU requests additionally +require the optional runc payload and runtime registration described above. + +Ascend 310P3 and 910 A2/A3 support additionally requires an image built with +`AKERNEL_ENABLE_ASCEND=true` and `node.config.sandboxd.enableAscend=true`. +The build also requires `ASCEND_ADAPTER_VERSION`, `ASCEND_ADAPTER_AMD64_URL`, +and `ASCEND_ADAPTER_AMD64_SHA256` for a fixed +[ascend-oci-adapter](https://github.com/openYuanrong-mirror/ascend-oci-adapter) +release bundle. The Ascend option requires runc. The Helm profile mounts the +standard host +driver paths into the privileged node pod; schedule that profile only on NPU +nodes where those paths exist. FlowGW's `/var/queue_schedule` path is optional: +set `node.config.sandboxd.mountAscendQueueSchedule=true` only when that +directory exists on every selected Ascend node. The mount is read-only. + The iptables sandbox NAT backend remains the default. Terraform deployments can set `sandboxd_nat_backend = "bpfnat"` to use sandboxd's experimental embedded TC eBPF backend on nodes without iptables NAT or conntrack modules. diff --git a/deploy/akernel/charts/core/templates/node/configmap.yaml b/deploy/akernel/charts/core/templates/node/configmap.yaml index bfc8989..037a1eb 100644 --- a/deploy/akernel/charts/core/templates/node/configmap.yaml +++ b/deploy/akernel/charts/core/templates/node/configmap.yaml @@ -21,6 +21,20 @@ data: {{- end }} {{- $sandboxdConfig = replace $runcMarker "runc=\"/usr/local/bin/runc\"" $sandboxdConfig }} {{- end }} +{{- $ascendMarker := "# AKERNEL_XPU_ASCEND" }} +{{- if .Values.node.config.sandboxd.enableAscend }} +{{- if not .Values.node.config.sandboxd.enableRunc }} +{{- fail "node.config.sandboxd.enableAscend requires node.config.sandboxd.enableRunc=true" }} +{{- end }} +{{- if not (contains $ascendMarker $sandboxdConfig) }} +{{- fail "node.config.sandboxd.enableAscend requires the # AKERNEL_XPU_ASCEND marker in node.config.sandboxd.config" }} +{{- end }} +{{- $ascendConfig := `[plugin.xpu.ascend] +enabled=true +adapter="/usr/local/libexec/akernel/ascend-oci-adapter" +mount_profile="/etc/akernel/ascend/mounts.json"` }} +{{- $sandboxdConfig = replace $ascendMarker $ascendConfig $sandboxdConfig }} +{{- end }} {{ $sandboxdConfig | indent 4 }} registry.json: | diff --git a/deploy/akernel/charts/core/templates/node/daemonset.yaml b/deploy/akernel/charts/core/templates/node/daemonset.yaml index 01eb11d..2e0c0c2 100644 --- a/deploy/akernel/charts/core/templates/node/daemonset.yaml +++ b/deploy/akernel/charts/core/templates/node/daemonset.yaml @@ -172,6 +172,22 @@ spec: - mountPath: /home/akernel/sandboxd/config.toml name: akernel-node-config subPath: sandboxd_config.toml + {{- if .Values.node.config.sandboxd.enableAscend }} + - mountPath: /usr/local/Ascend/driver + name: ascend-driver + readOnly: true + - mountPath: /usr/local/dcmi + name: ascend-dcmi + readOnly: true + - mountPath: /usr/local/bin/npu-smi + name: ascend-npu-smi + readOnly: true + {{- if .Values.node.config.sandboxd.mountAscendQueueSchedule }} + - mountPath: /var/queue_schedule + name: ascend-queue-schedule + readOnly: true + {{- end }} + {{- end }} - mountPath: /home/akernel name: home-disk {{- if not .Values.node.homeDisk.useEphemeralPVC }} @@ -228,6 +244,26 @@ spec: emptyDir: medium: Memory sizeLimit: 500Mi + {{- if .Values.node.config.sandboxd.enableAscend }} + - name: ascend-driver + hostPath: + path: /usr/local/Ascend/driver + type: Directory + - name: ascend-dcmi + hostPath: + path: /usr/local/dcmi + type: Directory + - name: ascend-npu-smi + hostPath: + path: /usr/local/bin/npu-smi + type: File + {{- if .Values.node.config.sandboxd.mountAscendQueueSchedule }} + - name: ascend-queue-schedule + hostPath: + path: /var/queue_schedule + type: Directory + {{- end }} + {{- end }} {{- if or .Values.monitoring.prometheusEndpoint .Values.monitoring.lokiEndpoint .Values.monitoring.tempoEndpoint }} - name: otel-collector-config configMap: diff --git a/deploy/akernel/charts/core/values.yaml b/deploy/akernel/charts/core/values.yaml index ab638fd..611bb44 100644 --- a/deploy/akernel/charts/core/values.yaml +++ b/deploy/akernel/charts/core/values.yaml @@ -448,6 +448,12 @@ node: # The node image must also be built with AKERNEL_ENABLE_RUNC=true. # Sandboxd probes runc and runc-shim before advertising the runtime. enableRunc: false + # Requires an image built with AKERNEL_ENABLE_ASCEND=true and runc enabled. + # Deploy this profile only to Ascend nodes whose host driver paths exist. + enableAscend: false + # Mount FlowGW's optional queue-scheduling directory when it exists on + # every selected Ascend node. + mountAscendQueueSchedule: false config: |- rootDir="/home/akernel/sandboxd/root" storeDir="/home/akernel/sandboxd/store" @@ -505,6 +511,8 @@ node: shim_binary="/usr/local/bin/runc-shim" kvm_device="/dev/kvm" + # AKERNEL_XPU_ASCEND + # image-manager config. root holds image-manager runtime state # (daemons/, mount_records.db) and is wiped on pod change; input # configs live under /home/akernel/sandboxd/config/ so the wipe never diff --git a/deploy/scripts/build-image.sh b/deploy/scripts/build-image.sh index 53acd8d..f5ef1d8 100755 --- a/deploy/scripts/build-image.sh +++ b/deploy/scripts/build-image.sh @@ -31,6 +31,9 @@ open_yr_core_wheel_url="${OPEN_YR_CORE_WHEEL_URL:-}" open_yr_core_wheel_sha256="${OPEN_YR_CORE_WHEEL_SHA256:-}" rrt_runtime_url="${RRT_RUNTIME_URL:-}" rrt_runtime_sha256="${RRT_RUNTIME_SHA256:-}" +ascend_adapter_version="${ASCEND_ADAPTER_VERSION:-}" +ascend_adapter_amd64_url="${ASCEND_ADAPTER_AMD64_URL:-}" +ascend_adapter_amd64_sha256="${ASCEND_ADAPTER_AMD64_SHA256:-}" print_component_versions=0 component_revision() { @@ -149,6 +152,21 @@ case "${AKERNEL_ENABLE_RUNC:-false}" in *) die "AKERNEL_ENABLE_RUNC must be true or false" ;; esac +case "${AKERNEL_ENABLE_ASCEND:-false}" in + true|false) ;; + *) die "AKERNEL_ENABLE_ASCEND must be true or false" ;; +esac +if [[ "${AKERNEL_ENABLE_ASCEND:-false}" == "true" && + "${AKERNEL_ENABLE_RUNC:-false}" != "true" ]]; then + die "AKERNEL_ENABLE_ASCEND=true requires AKERNEL_ENABLE_RUNC=true" +fi +if [[ "${AKERNEL_ENABLE_ASCEND:-false}" == "true" && + ( -z "${ascend_adapter_version}" || + -z "${ascend_adapter_amd64_url}" || + -z "${ascend_adapter_amd64_sha256}" ) ]]; then + die "AKERNEL_ENABLE_ASCEND=true requires ASCEND_ADAPTER_VERSION, ASCEND_ADAPTER_AMD64_URL, and ASCEND_ADAPTER_AMD64_SHA256" +fi + repository="${repository:-akernel-all-in-one}" tag="${tag:-$(git -C "${AKERNEL_REPO_ROOT}" rev-parse --short HEAD)-$(date +%Y%m%d%H%M%S)}" @@ -208,6 +226,10 @@ node_build_args=( --build-arg "AKERNEL_RUNTIME_PROFILE=${runtime_profile}" --build-arg "AKERNEL_ENABLE_KATA=${AKERNEL_ENABLE_KATA:-true}" --build-arg "AKERNEL_ENABLE_RUNC=${AKERNEL_ENABLE_RUNC:-false}" + --build-arg "AKERNEL_ENABLE_ASCEND=${AKERNEL_ENABLE_ASCEND:-false}" + --build-arg "ASCEND_ADAPTER_VERSION=${ascend_adapter_version}" + --build-arg "ASCEND_ADAPTER_AMD64_URL=${ascend_adapter_amd64_url}" + --build-arg "ASCEND_ADAPTER_AMD64_SHA256=${ascend_adapter_amd64_sha256}" --build-arg "AKERNEL_ENABLE_FIRECRACKER=${AKERNEL_ENABLE_FIRECRACKER:-true}" --build-arg "AKERNEL_VERSION=${akernel_version}" --build-arg "AKERNEL_REVISION=${akernel_revision}" diff --git a/deploy/scripts/configure.sh b/deploy/scripts/configure.sh index 69f1915..5e388d0 100755 --- a/deploy/scripts/configure.sh +++ b/deploy/scripts/configure.sh @@ -31,6 +31,7 @@ image_tag_override="" install_monitor_override="" install_dragonfly_override="" enable_runc_override="" +enable_ascend_override="" grafana_public_access_override="" grafana_admin_password_override="" iam_seed_hex_override="" @@ -121,6 +122,10 @@ while [[ $# -gt 0 ]]; do enable_runc_override="$2" shift 2 ;; + --enable-ascend) + enable_ascend_override="$2" + shift 2 + ;; --grafana-public-access) grafana_public_access_override="$2" shift 2 @@ -221,6 +226,7 @@ set_or_prompt image_tag "All-in-one image tag" "${default_tag}" "${image_tag_ove set_or_prompt install_monitor "Install monitor chart (true/false)" "true" "${install_monitor_override}" set_or_prompt install_dragonfly "Install Dragonfly and dedicated node pools (true/false)" "false" "${install_dragonfly_override}" set_or_prompt enable_runc "Enable the optional runc runtime (true/false)" "false" "${enable_runc_override}" +set_or_prompt enable_ascend "Enable Ascend 310P3 or 910 A2/A3 NPUs (true/false)" "false" "${enable_ascend_override}" set_or_prompt grafana_public_access "Expose Grafana LoadBalancer (true/false)" "true" "${grafana_public_access_override}" set_or_prompt grafana_admin_password \ "Grafana admin password (empty to generate)" "" \ @@ -231,6 +237,10 @@ fi install_monitor="$(normalize_bool "${install_monitor}")" install_dragonfly="$(normalize_bool "${install_dragonfly}")" enable_runc="$(normalize_bool "${enable_runc}")" +enable_ascend="$(normalize_bool "${enable_ascend}")" +if [[ "${enable_ascend}" == "true" && "${enable_runc}" != "true" ]]; then + die "Ascend NPU support requires enable_runc=true" +fi grafana_public_access="$(normalize_bool "${grafana_public_access}")" dir="$(state_dir "${env_name}")" @@ -352,6 +362,7 @@ grafana_admin_password = "${grafana_admin_password}" install_dragonfly = ${install_dragonfly} enable_runc = ${enable_runc} +enable_ascend = ${enable_ascend} EOF ;; huaweicloud) @@ -406,6 +417,7 @@ grafana_admin_password = "${grafana_admin_password}" install_dragonfly = ${install_dragonfly} enable_runc = ${enable_runc} +enable_ascend = ${enable_ascend} EOF ;; esac @@ -427,6 +439,7 @@ CORE_NAMESPACE=akernel MONITOR_NAMESPACE=akernel-monitor INSTALL_DRAGONFLY=${install_dragonfly} AKERNEL_ENABLE_RUNC=${enable_runc} +AKERNEL_ENABLE_ASCEND=${enable_ascend} EOF chmod 600 "${tfvars_file}" "${config_file}" diff --git a/deploy/standalone/README.md b/deploy/standalone/README.md index b24912b..4da4c8a 100644 --- a/deploy/standalone/README.md +++ b/deploy/standalone/README.md @@ -37,8 +37,8 @@ Clients then select it with `Sandbox(runtime="runc")`. A sandbox may request the configured KVM character device with `extra_config={"enableKVM": True}` when the host exposes `/dev/kvm`. -Experimental NVIDIA GPU sandboxes use gVisor nvproxy. The host must provide a -compatible NVIDIA driver and NVIDIA Container Toolkit. Enable GPU access to +Experimental NVIDIA GPU sandboxes support gVisor nvproxy and native runc. The +host must provide a compatible NVIDIA driver and NVIDIA Container Toolkit. Enable GPU access to the node container with: ```bash @@ -49,6 +49,27 @@ The all-in-one image contains `nvidia-container-cli`, but not the host driver. Use `AKERNEL_GPU_DEVICES` to override Docker's `--gpus` value when only a device subset should be assigned. +To use the same GPU with the native runc sandbox backend, build and launch the +node with both `AKERNEL_ENABLE_RUNC=true` and `AKERNEL_ENABLE_GPU=true`, then +select `Sandbox(runtime="runc", xpu="gpu::1")`. + +Ascend 310P3 or 910 A2/A3 requires an image built with both optional payloads and a +host with the standard driver paths: + +```bash +AKERNEL_ENABLE_RUNC=true AKERNEL_ENABLE_ASCEND=true make build \ + IMAGE_REPOSITORY=akernel-ascend IMAGE_TAG=local +IMAGE=akernel-ascend:local \ + AKERNEL_ENABLE_RUNC=true AKERNEL_ENABLE_ASCEND=true ./start.sh +``` + +`start.sh` verifies and mounts `/usr/local/Ascend/driver`, +`/usr/local/dcmi`, and `/usr/local/bin/npu-smi` into the node container. +When FlowGW has created `/var/queue_schedule`, `start.sh` also mounts that +directory read-only; its absence does not prevent an Ascend standalone node +from starting. Sandboxd then applies a narrower, versioned mount profile to +each leased runc sandbox. + Explicit sandbox storage quotas for runsc and Firecracker use the bounded ext4 filestore mounted at `/home/akernel/filestore`. The standalone data directory is bind-mounted from the host, and sandboxd creates a loop-backed filesystem diff --git a/deploy/standalone/config/sandboxd_config.toml b/deploy/standalone/config/sandboxd_config.toml index ea41dd1..17cce6c 100644 --- a/deploy/standalone/config/sandboxd_config.toml +++ b/deploy/standalone/config/sandboxd_config.toml @@ -61,6 +61,8 @@ state_root="/run/sandboxd/runc" shim_binary="/usr/local/bin/runc-shim" kvm_device="/dev/kvm" +# AKERNEL_XPU_ASCEND + # image-manager config. root holds image-manager runtime state and is wiped # on pod change; input configs live under /home/akernel/sandboxd/config/ so the # wipe never hits them. diff --git a/deploy/standalone/start.sh b/deploy/standalone/start.sh index 8faed46..eff2a1c 100755 --- a/deploy/standalone/start.sh +++ b/deploy/standalone/start.sh @@ -24,6 +24,7 @@ SANDBOXD_CONFIG_FILE="${DATA_DIR}/sandboxd/config.toml" AKERNEL_NAT_BACKEND="${AKERNEL_NAT_BACKEND:-iptables}" AKERNEL_ENABLE_RUNC="${AKERNEL_ENABLE_RUNC:-false}" YR_IMAGE_PROCESS_CONFIG="${YR_IMAGE_PROCESS_CONFIG:-/run/akernel/yr-image-process.json}" +AKERNEL_ENABLE_ASCEND="${AKERNEL_ENABLE_ASCEND:-false}" LITEBUS_DATA_KEY="" # Container runtime command (docker or pouch) @@ -31,6 +32,7 @@ DOCKER_CMD="" DOCKER_PREFIX=() PROXY_RUN_ARGS=() GPU_RUN_ARGS=() +ASCEND_RUN_ARGS=() # Colors for output RED='\033[0;31m' @@ -104,6 +106,18 @@ check_prerequisites() { exit 1 ;; esac + case "${AKERNEL_ENABLE_ASCEND}" in + true|false) + ;; + *) + log_error "AKERNEL_ENABLE_ASCEND must be true or false" + exit 1 + ;; + esac + if [[ "${AKERNEL_ENABLE_ASCEND}" == "true" && "${AKERNEL_ENABLE_RUNC}" != "true" ]]; then + log_error "AKERNEL_ENABLE_ASCEND=true requires AKERNEL_ENABLE_RUNC=true" + exit 1 + fi # Create data directory mkdir -p "${DATA_DIR}" @@ -237,6 +251,35 @@ configure_gpu() { log_info "Enabling NVIDIA GPU access for the AKernel node container" } +configure_ascend() { + if [[ "${AKERNEL_ENABLE_ASCEND}" != "true" ]]; then + return 0 + fi + local required_path + for required_path in \ + /usr/local/Ascend/driver \ + /usr/local/dcmi \ + /usr/local/bin/npu-smi; do + if [[ ! -e "${required_path}" ]]; then + log_error "Ascend host path is missing: ${required_path}" + exit 1 + fi + done + ASCEND_RUN_ARGS=( + -v /usr/local/Ascend/driver:/usr/local/Ascend/driver:ro + -v /usr/local/dcmi:/usr/local/dcmi:ro + -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi:ro + ) + # FlowGW creates this directory only on hosts that use queue scheduling. + # Do not create it here: its absence means the optional integration is off. + if [[ -d /var/queue_schedule ]]; then + ASCEND_RUN_ARGS+=( + -v /var/queue_schedule:/var/queue_schedule:ro + ) + fi + log_info "Enabling Ascend NPU access for runc sandboxes" +} + configure_network() { local config_tmp="${SANDBOXD_CONFIG_FILE}.tmp" local sed_args=( @@ -268,6 +311,20 @@ configure_network() { -e 's|^[[:space:]]*# AKERNEL_RUNTIME_RUNC[[:space:]]*$|runc="/usr/local/bin/runc"|' ) fi + if [[ "${AKERNEL_ENABLE_ASCEND}" == "true" ]]; then + if ! grep -q '^[[:space:]]*# AKERNEL_XPU_ASCEND[[:space:]]*$' \ + "${CONFIG_DIR}/sandboxd_config.toml"; then + log_error "AKERNEL_ENABLE_ASCEND requires the # AKERNEL_XPU_ASCEND marker in sandboxd_config.toml" + exit 1 + fi + sed_args+=( + -e '/^[[:space:]]*# AKERNEL_XPU_ASCEND[[:space:]]*$/c\ +[plugin.xpu.ascend]\ +enabled=true\ +adapter="/usr/local/libexec/akernel/ascend-oci-adapter"\ +mount_profile="/etc/akernel/ascend/mounts.json"' + ) + fi sed "${sed_args[@]}" "${CONFIG_DIR}/sandboxd_config.toml" > "${config_tmp}" mv "${config_tmp}" "${SANDBOXD_CONFIG_FILE}" @@ -377,6 +434,7 @@ start_node_container() { -e ENABLE_METRICS="${ENABLE_METRICS:-false}" \ "${PROXY_RUN_ARGS[@]}" \ "${GPU_RUN_ARGS[@]}" \ + "${ASCEND_RUN_ARGS[@]}" \ --entrypoint=/usr/local/bin/akernel-entrypoint \ -v "${DATA_DIR}:/home/akernel" \ -v "${CONFIG_DIR}/oss_auths.json:/home/akernel/sandboxd/config/oss_auths.json:ro" \ @@ -530,6 +588,7 @@ ensure_image "${IMAGE}" ensure_image "${TRAEFIK_IMAGE}" configure_container_proxy configure_gpu +configure_ascend configure_network prepare_host_network_modules start_node_container diff --git a/deploy/terraform/aliyun/main.tf b/deploy/terraform/aliyun/main.tf index c23bf15..a6bd3a6 100644 --- a/deploy/terraform/aliyun/main.tf +++ b/deploy/terraform/aliyun/main.tf @@ -154,6 +154,7 @@ locals { traefik_enabled = var.traefik_enabled sandboxd_nat_backend = var.sandboxd_nat_backend enable_runc = var.enable_runc + enable_ascend = var.enable_ascend node_secret_create = var.node_secret_create node_home_use_csi_ephemeral = var.node_home_use_csi_ephemeral node_home_csi_storage_class = local.effective_node_home_csi_sc diff --git a/deploy/terraform/aliyun/terraform.tfvars.example b/deploy/terraform/aliyun/terraform.tfvars.example index eb411f9..39c8b93 100644 --- a/deploy/terraform/aliyun/terraform.tfvars.example +++ b/deploy/terraform/aliyun/terraform.tfvars.example @@ -176,6 +176,10 @@ node_secret_create = true # matching AKERNEL_ENABLE_RUNC=true profile setting. enable_runc = false +# Optional Ascend 310P3 or 910 A2/A3 support. Requires enable_runc=true and an image +# built with AKERNEL_ENABLE_ASCEND=true; use only for NPU node pools. +enable_ascend = false + # --- Monitor --- install_monitor = true monitor_namespace = "akernel-monitor" diff --git a/deploy/terraform/aliyun/values-akernel.yaml.tmpl b/deploy/terraform/aliyun/values-akernel.yaml.tmpl index 0ce6d32..ce5a9f8 100644 --- a/deploy/terraform/aliyun/values-akernel.yaml.tmpl +++ b/deploy/terraform/aliyun/values-akernel.yaml.tmpl @@ -145,6 +145,7 @@ node: cpuQuota: "100%" sandboxd: enableRunc: ${enable_runc} + enableAscend: ${enable_ascend} config: |- rootDir="/home/akernel/sandboxd/root" storeDir="/home/akernel/sandboxd/store" @@ -201,6 +202,8 @@ node: shim_binary="/usr/local/bin/runc-shim" kvm_device="/dev/kvm" + # AKERNEL_XPU_ASCEND + # image-manager config. root holds image-manager runtime state # (daemons/, mount_records.db) and is wiped on pod change; input # configs live under /home/akernel/sandboxd/config/ so the wipe never diff --git a/deploy/terraform/aliyun/variables.tf b/deploy/terraform/aliyun/variables.tf index d390e09..b4a4786 100644 --- a/deploy/terraform/aliyun/variables.tf +++ b/deploy/terraform/aliyun/variables.tf @@ -343,6 +343,12 @@ variable "enable_runc" { default = false } +variable "enable_ascend" { + type = bool + description = "Enable Ascend 310P3 or 910 A2/A3 physical NPUs; requires enable_runc and a node image built with AKERNEL_ENABLE_ASCEND=true." + default = false +} + variable "node_home_use_csi_ephemeral" { type = bool description = "Mount /home/akernel from a per-node CSI ephemeral volume instead of hostPath /home/akernel." diff --git a/deploy/terraform/huaweicloud/main.tf b/deploy/terraform/huaweicloud/main.tf index da5e56b..fbbfddc 100644 --- a/deploy/terraform/huaweicloud/main.tf +++ b/deploy/terraform/huaweicloud/main.tf @@ -113,6 +113,7 @@ locals { master_service_loadbalancer_ip = var.master_public_access_8888 ? var.master_service_loadbalancer_ip : "" sandboxd_nat_backend = var.sandboxd_nat_backend enable_runc = var.enable_runc + enable_ascend = var.enable_ascend node_secret_create = var.node_secret_create node_home_use_csi_ephemeral = var.node_home_use_csi_ephemeral node_home_csi_storage_class = var.node_home_csi_storage_class diff --git a/deploy/terraform/huaweicloud/terraform.tfvars.example b/deploy/terraform/huaweicloud/terraform.tfvars.example index c0f9e1c..9aed40e 100644 --- a/deploy/terraform/huaweicloud/terraform.tfvars.example +++ b/deploy/terraform/huaweicloud/terraform.tfvars.example @@ -93,6 +93,10 @@ node_image_tag = "" # matching AKERNEL_ENABLE_RUNC=true profile setting. enable_runc = false +# Optional Ascend 310P3 or 910 A2/A3 support. Requires enable_runc=true and an image +# built with AKERNEL_ENABLE_ASCEND=true; use only for NPU node pools. +enable_ascend = false + # Enable the /internal-stats endpoint sidecar on Traefik. # traefik_internal_stats_enabled = true # traefik_internal_stats_image = "my-registry.example.com/busybox:1.37.0-musl" diff --git a/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl b/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl index 89c3602..d92079a 100644 --- a/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl +++ b/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl @@ -125,6 +125,7 @@ node: cpuQuota: "100%" sandboxd: enableRunc: ${enable_runc} + enableAscend: ${enable_ascend} config: |- rootDir="/home/akernel/sandboxd/root" storeDir="/home/akernel/sandboxd/store" @@ -179,6 +180,8 @@ node: shim_binary="/usr/local/bin/runc-shim" kvm_device="/dev/kvm" + # AKERNEL_XPU_ASCEND + # image-manager config. root holds image-manager runtime state # (daemons/, mount_records.db) and is wiped on pod change; input # configs live under /home/akernel/sandboxd/config/ so the wipe never diff --git a/deploy/terraform/huaweicloud/variables.tf b/deploy/terraform/huaweicloud/variables.tf index 5c668bb..f278e06 100644 --- a/deploy/terraform/huaweicloud/variables.tf +++ b/deploy/terraform/huaweicloud/variables.tf @@ -375,6 +375,12 @@ variable "enable_runc" { default = false } +variable "enable_ascend" { + type = bool + description = "Enable Ascend 310P3 or 910 A2/A3 physical NPUs; requires enable_runc and a node image built with AKERNEL_ENABLE_ASCEND=true." + default = false +} + variable "node_home_use_csi_ephemeral" { type = bool description = "Mount /home/akernel from a per-node CSI ephemeral volume instead of hostPath /home/akernel." diff --git a/sdk/python/README.md b/sdk/python/README.md index c18d5ae..db79366 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -16,7 +16,7 @@ It supports two backends: - [Navigation](#navigation) - [Install and configure](#install-and-configure) - [Create a sandbox](#create-a-sandbox) - - [Experimental GPU and writable storage](#experimental-gpu-and-writable-storage) + - [Experimental GPU, NPU, and writable storage](#experimental-gpu-npu-and-writable-storage) - [Network ACLs](#network-acls) - [Sandbox runtimes](#sandbox-runtimes) - [Commands](#commands) @@ -113,7 +113,7 @@ Sandbox( ) ``` -### Experimental GPU and writable storage +### Experimental GPU, NPU, and writable storage Request a whole NVIDIA GPU by type, exact product model, and count: @@ -124,9 +124,18 @@ with Sandbox(xpu="gpu:l20:1") as sandbox: The `type:model:count` value is case-insensitive and canonicalized to lower case. The model is required and matched exactly; wildcard models are not -supported. The bundled backend currently requires the gVisor `runsc` runtime -and a node configured for gVisor nvproxy. Runtime compatibility is validated -by the backend rather than the SDK. +supported. The bundled backend supports GPU sandboxes with gVisor `runsc` or +native `runc`. Runsc remains subject to its nvproxy driver compatibility gate; +that gate does not disable native-runc GPU support. + +Request one physical Ascend 310P or 910 A2/A3 NPU with runc: + +```python +with Sandbox(runtime="runc", xpu="npu:ascend910b4:1") as sandbox: + print(sandbox.commands.run("npu-smi info").stdout) +``` + +The model token is matched exactly against the normalized driver-reported resources of the selected node, without a per-SKU allowlist. For a node reporting `ascend910`, use `npu:ascend910:1`; do not infer a different SKU token. Runtime and provider-owned environment validation remain backend-owned. Ascend driver library directories are added ahead of the image/request library paths, preserving CANN and application directories. Set the writable root filesystem quota in MiB: @@ -314,8 +323,8 @@ with Sandbox( `enableKVM` is owned by the runc backend and requires a usable `/dev/kvm` on the selected node. Runc supports OCI/EROFS root filesystems, read-only mounts, networking, command execution, and the default writable overlay. Experimental -GPU requests remain runsc-only; explicit `storage_mb` quotas are supported by -runsc and Firecracker. See the +GPU requests support runsc and runc, while Ascend NPU requests are runc-only; +explicit `storage_mb` quotas are supported by runsc and Firecracker. See the [sandbox runtime comparison](../../src/sandboxd/doc/runtime.md) for the runtime capability boundaries. diff --git a/sdk/python/akernel_sdk/_sandbox_resources.py b/sdk/python/akernel_sdk/_sandbox_resources.py index 3682065..5b11a3e 100644 --- a/sdk/python/akernel_sdk/_sandbox_resources.py +++ b/sdk/python/akernel_sdk/_sandbox_resources.py @@ -37,8 +37,8 @@ def normalize_xpu(value: str | None) -> str | None: if len(fields) != 3: raise ValueError("xpu must contain exactly three fields: type:model:count") xpu_type, model, count_text = (field.strip().lower() for field in fields) - if xpu_type != "gpu": - raise ValueError("xpu type must be gpu") + if xpu_type not in {"gpu", "npu"}: + raise ValueError("xpu type must be gpu or npu") if not model: raise ValueError("xpu model must be non-empty") if not _XPU_MODEL_PATTERN.fullmatch(model): diff --git a/sdk/python/akernel_sdk/sandbox.py b/sdk/python/akernel_sdk/sandbox.py index d939611..5a2f671 100644 --- a/sdk/python/akernel_sdk/sandbox.py +++ b/sdk/python/akernel_sdk/sandbox.py @@ -230,9 +230,9 @@ def __init__( inherit_entrypoint: Start the OCI image's effective ENTRYPOINT and CMD as the sandbox workload. Valid only with ``image``. xpu: Experimental whole-device accelerator request in - ``type:model:count`` format. Currently only exact-model NVIDIA - GPU requests are supported. The backend validates runtime - compatibility. + ``type:model:count`` format. Exact-model NVIDIA GPU and Ascend + NPU requests are supported; the backend validates runtime and + node compatibility. storage_mb: Experimental writable root filesystem quota in MiB. When omitted, the configured default is used. Explicit quotas are validated against the selected runtime by the backend. diff --git a/sdk/python/examples/gpu_sandbox.py b/sdk/python/examples/gpu_sandbox.py index 861b448..2e2ba25 100644 --- a/sdk/python/examples/gpu_sandbox.py +++ b/sdk/python/examples/gpu_sandbox.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Run a CUDA sample in an experimental gVisor GPU sandbox.""" +"""Run a CUDA sample in an experimental runsc or runc GPU sandbox.""" import os @@ -26,11 +26,14 @@ def main() -> None: model = os.environ.get("AKERNEL_GPU_MODEL", "a10").strip().lower() + runtime = os.environ.get("AKERNEL_GPU_RUNTIME", "runsc").strip().lower() + if runtime not in {"runsc", "runc"}: + raise ValueError("AKERNEL_GPU_RUNTIME must be runsc or runc") with Sandbox( image=CUDA_SAMPLE_IMAGE, - runtime="runsc", + runtime=runtime, xpu=f"gpu:{model}:1", - storage_mb=512, + storage_mb=512 if runtime == "runsc" else None, cpu=1000, memory=2048, schedule_timeout=120, diff --git a/sdk/python/tests/unit/test_sandbox.py b/sdk/python/tests/unit/test_sandbox.py index 739c101..5810dc3 100644 --- a/sdk/python/tests/unit/test_sandbox.py +++ b/sdk/python/tests/unit/test_sandbox.py @@ -297,7 +297,7 @@ def test_xpu_request_validation(self): (1, TypeError), ("gpu", ValueError), ("gpu::1", ValueError), - ("npu:l20:1", ValueError), + ("tpu:l20:1", ValueError), ("gpu:l20:0", ValueError), ("gpu:l20:1.5", ValueError), ("gpu:l20/evil:1", ValueError), diff --git a/sdk/python/tests/unit/test_sandbox_resources.py b/sdk/python/tests/unit/test_sandbox_resources.py index 98642a4..2fe03fc 100644 --- a/sdk/python/tests/unit/test_sandbox_resources.py +++ b/sdk/python/tests/unit/test_sandbox_resources.py @@ -30,6 +30,10 @@ def test_xpu_is_canonical_and_exact_model_is_escaped(self): xpu_custom_resource("gpu:a10.2:2"), (r"GPU/a10\.2/count", 2.0), ) + self.assertEqual( + xpu_custom_resource("NPU:Ascend910_9391:1"), + ("NPU/ascend910_9391/count", 1.0), + ) def test_storage_wire_value_and_upper_bound(self): self.assertEqual(storage_bytes(256), float(256 * 1024 * 1024)) diff --git a/src/sandboxd b/src/sandboxd index b892414..52ea5cf 160000 --- a/src/sandboxd +++ b/src/sandboxd @@ -1 +1 @@ -Subproject commit b892414a3b21f3c2a4481322767362af14fc701f +Subproject commit 52ea5cf2c3c1e52d63e36329ca660c26eba97d6f