diff --git a/.gitmodules b/.gitmodules index 97ae8b8..6a898d0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "src/sandboxd"] path = src/sandboxd - url = https://github.com/inclusionAI/sandboxd.git + url = https://github.com/mhsong1998-dot/sandboxd.git [submodule "src/distill-fs"] path = src/distill-fs url = https://github.com/inclusionAI/distill-fs.git diff --git a/AGENTS.md b/AGENTS.md index fca1d2a..63fa268 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,8 +18,7 @@ enabled by an operator. Creation-time network policies and atomic runtime replacement support unrestricted networking, blocking new flows except the YuanRong control and published sandbox-port routes, or denying exact and leading-wildcard DNS names. -Experimental whole-device NVIDIA GPU requests require runsc. Configurable -writable-storage requests are supported by runsc and Firecracker. +Experimental whole-device NVIDIA GPU requests support runsc or explicitly enabled runc. Physical Ascend 310P and 910 A2/A3 requests require explicitly enabled runc and Ascend payloads. Accelerator model tokens match normalized driver-reported inventory; Ascend has no per-SKU allowlist. The optional FlowGW `/var/queue_schedule` mount is detected in standalone mode and opt-in with Helm `node.config.sandboxd.mountAscendQueueSchedule`. Configurable writable-storage requests are supported by runsc and Firecracker. Use AKernel when a task needs an isolated remote environment with command execution, file operations, interactive PTYs, port forwarding, or reverse diff --git a/Makefile b/Makefile index 100218d..84e6ac8 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,7 @@ MONITOR_STORAGE_CLASS ?= INSTALL_MONITOR ?= INSTALL_DRAGONFLY ?= ENABLE_RUNC ?= +ENABLE_ASCEND ?= GRAFANA_PUBLIC_ACCESS ?= GRAFANA_ADMIN_PASSWORD ?= IAM_SEED_HEX ?= @@ -49,12 +50,14 @@ help: @echo " make config NON_INTERACTIVE=1 ... Generate config from Make variables" @echo " make config INSTALL_DRAGONFLY=true Enable optional P2P image distribution" @echo " make config ENABLE_RUNC=true Build and register the optional runc runtime" + @echo " make config ENABLE_RUNC=true ENABLE_ASCEND=true Enable Ascend NPU nodes" @echo " make build IMAGE_TAG= Build the all-in-one image" @echo " make build RUNTIME_PROFILE=python Include optional Python runtimes" @echo " make build AKERNEL_ENABLE_KATA=false Exclude the optional Kata payload" @echo " make build AKERNEL_ENABLE_FIRECRACKER=false Exclude Firecracker" @echo " make build AKERNEL_ENABLE_RUNC=true Include the optional runc payload" @echo " make build RRT_RUNTIME_URL=... RRT_RUNTIME_SHA256=... Override RRT artifact" + @echo " make build AKERNEL_ENABLE_RUNC=true AKERNEL_ENABLE_ASCEND=true Include Ascend NPU support" @echo " make versions Show locally selected component versions" @echo " make push Push the configured all-in-one image" @echo " make plan Terraform plan" @@ -92,6 +95,7 @@ config: if [[ -n "$(INSTALL_MONITOR)" ]]; then args+=(--install-monitor "$(INSTALL_MONITOR)"); fi; \ if [[ -n "$(INSTALL_DRAGONFLY)" ]]; then args+=(--install-dragonfly "$(INSTALL_DRAGONFLY)"); fi; \ if [[ -n "$(ENABLE_RUNC)" ]]; then args+=(--enable-runc "$(ENABLE_RUNC)"); fi; \ + if [[ -n "$(ENABLE_ASCEND)" ]]; then args+=(--enable-ascend "$(ENABLE_ASCEND)"); fi; \ if [[ -n "$(GRAFANA_PUBLIC_ACCESS)" ]]; then args+=(--grafana-public-access "$(GRAFANA_PUBLIC_ACCESS)"); fi; \ if [[ -n "$(GRAFANA_ADMIN_PASSWORD)" ]]; then args+=(--grafana-admin-password "$(GRAFANA_ADMIN_PASSWORD)"); fi; \ if [[ -n "$(IAM_SEED_HEX)" ]]; then args+=(--iam-seed-hex "$(IAM_SEED_HEX)"); fi; \ diff --git a/README.md b/README.md index 2f54892..e4417c1 100644 --- a/README.md +++ b/README.md @@ -164,16 +164,30 @@ with Sandbox(cpu=1000, memory=2048) as sandbox: print(sandbox.files.read("/tmp/hello.txt")) ``` -Experimental gVisor sandboxes can request an exact NVIDIA GPU model: +Experimental runsc or runc sandboxes can request an exact NVIDIA GPU model: ```python -with Sandbox(xpu="gpu:l20:1") as sandbox: +with Sandbox(runtime="runc", xpu="gpu:l20:1") as sandbox: print(sandbox.commands.run("nvidia-smi -L").stdout) ``` -GPU sandboxes require a compatible NVIDIA node and the gVisor `runsc` -runtime. `storage_mb` is measured in MiB and is supported by `runsc` and -Firecracker. +GPU sandboxes require a compatible NVIDIA node and either the gVisor `runsc` +runtime or native `runc`. Runsc remains subject to its nvproxy driver-version +compatibility check; that check does not disable native-runc GPU support. +`storage_mb` is measured in MiB and is supported by `runsc` and Firecracker. + +Physical Ascend 310P and 910 A2/A3 sandboxes use the native `runc` runtime and the normalized model reported by the driver: + +```python +with Sandbox(runtime="runc", xpu="npu:ascend910b4:1") as sandbox: + print(sandbox.commands.run("npu-smi info").stdout) +``` + +Atlas 300I Pro nodes reporting `ascend310p3` use `npu:ascend310p3:1`. If an A3 node reports `ascend910`, use `npu:ascend910:1`; no per-SKU model allowlist or inferred model alias is applied. + +The node image and deployment must enable both runc and Ascend support. The +node-local provider discovers physical devices, owns exclusive leases, and +injects only the leased device nodes and versioned read-only driver mounts. See the complete [basic usage example](./sdk/python/examples/basic_usage.py), the [sandbox runtime example](./sdk/python/examples/sandbox_runtime.py), and the other [SDK examples](./sdk/python/examples/) for more operations. diff --git a/builder/node.Dockerfile b/builder/node.Dockerfile index 19669be..bea7d0d 100644 --- a/builder/node.Dockerfile +++ b/builder/node.Dockerfile @@ -7,6 +7,10 @@ ARG AKERNEL_RUNTIME_IMAGE=akernel-runtime:local ARG AKERNEL_RUNTIME_PROFILE=rrt ARG AKERNEL_ENABLE_KATA=true ARG AKERNEL_ENABLE_RUNC=false +ARG AKERNEL_ENABLE_ASCEND=false +ARG ASCEND_ADAPTER_VERSION= +ARG ASCEND_ADAPTER_AMD64_URL= +ARG ASCEND_ADAPTER_AMD64_SHA256= ARG AKERNEL_ENABLE_FIRECRACKER=true ARG SANDBOXD_BUILD_IMAGE=golang:1.25.5-bookworm ARG DISTILL_FS_BUILD_IMAGE=rust:1.85.0-bookworm @@ -114,6 +118,43 @@ WORKDIR /src/sandboxd COPY ./src/sandboxd/ ./ RUN make release +FROM ubuntu:24.04 AS ascend-adapter-true +ARG TARGETARCH +ARG ASCEND_ADAPTER_VERSION +ARG ASCEND_ADAPTER_AMD64_URL +ARG ASCEND_ADAPTER_AMD64_SHA256 +RUN apt-get update && \ + apt-get install -y --no-install-recommends ca-certificates curl && \ + rm -rf /var/lib/apt/lists/* +COPY ./src/sandboxd/configs/ascend/mounts.json /tmp/ascend-mounts.json +RUN set -eux; \ + test "${TARGETARCH:-amd64}" = "amd64"; \ + test -n "${ASCEND_ADAPTER_VERSION}"; \ + test -n "${ASCEND_ADAPTER_AMD64_URL}"; \ + test -n "${ASCEND_ADAPTER_AMD64_SHA256}"; \ + archive=/tmp/ascend-oci-adapter.tar.gz; \ + curl -fSL --retry 10 --retry-delay 2 --retry-all-errors \ + "${ASCEND_ADAPTER_AMD64_URL}" -o "${archive}"; \ + echo "${ASCEND_ADAPTER_AMD64_SHA256} ${archive}" | sha256sum -c -; \ + bundle="/tmp/ascend-oci-adapter_${ASCEND_ADAPTER_VERSION}_linux_amd64"; \ + tar -xzf "${archive}" -C /tmp; \ + test -x "${bundle}/bin/ascend-oci-adapter"; \ + test -f "${bundle}/licenses/ascend-oci-adapter/LICENSE"; \ + test -f "${bundle}/licenses/mind-cluster/LICENSE"; \ + test -f "${bundle}/licenses/mind-cluster/Third_Party_Open_Source_Software_Notice.md"; \ + install -D -m 0755 "${bundle}/bin/ascend-oci-adapter" \ + /ascend/usr/local/libexec/akernel/ascend-oci-adapter; \ + install -D -m 0644 /tmp/ascend-mounts.json \ + /ascend/etc/akernel/ascend/mounts.json; \ + mkdir -p /ascend/opt/akernel/licenses; \ + cp -a "${bundle}/licenses/." /ascend/opt/akernel/licenses/; \ + rm -rf "${archive}" "${bundle}" + +FROM ${SANDBOXD_BUILD_IMAGE} AS ascend-adapter-false +RUN mkdir -p /ascend/usr/local/libexec/akernel + +FROM ascend-adapter-${AKERNEL_ENABLE_ASCEND} AS ascend-adapter + FROM ${FIRECRACKER_BUILD_IMAGE} AS firecracker-runtime-true ARG FIRECRACKER_RELEASE ARG FIRECRACKER_AMD64_SHA256 @@ -216,6 +257,7 @@ RUN cargo build --locked --release --bin distill_fs FROM ${AKERNEL_NODE_BASE_IMAGE} ARG AKERNEL_ENABLE_KATA ARG AKERNEL_ENABLE_RUNC +ARG AKERNEL_ENABLE_ASCEND ARG AKERNEL_ENABLE_FIRECRACKER ARG AKERNEL_RUNTIME_PROFILE ARG AKERNEL_VERSION @@ -272,7 +314,10 @@ RUN set -eux; \ apt-get update; \ apt-get install -y --no-install-recommends \ "libnvidia-container1=${LIBNVIDIA_CONTAINER_VERSION}" \ - "libnvidia-container-tools=${LIBNVIDIA_CONTAINER_VERSION}"; \ + "libnvidia-container-tools=${LIBNVIDIA_CONTAINER_VERSION}" \ + "nvidia-container-toolkit=${LIBNVIDIA_CONTAINER_VERSION}"; \ + command -v nvidia-container-cli; \ + test -x /usr/bin/nvidia-container-runtime-hook; \ rm -rf /var/lib/apt/lists/* RUN if command -v update-alternatives >/dev/null 2>&1; then \ @@ -351,6 +396,7 @@ COPY --from=distill-fs-builder /src/distill-fs/target/release/distill_fs /usr/lo COPY --from=kata-runtime /kata/opt/kata /opt/kata COPY --from=runc-runtime /runc/usr/local/bin/ /usr/local/bin/ COPY --from=firecracker-runtime /firecracker/ / +COPY --from=ascend-adapter /ascend/ / RUN if [ "${AKERNEL_ENABLE_KATA}" = "true" ]; then \ ln -sf /opt/kata/runtime-rs/bin/containerd-shim-kata-v2 /usr/local/bin/containerd-shim-kata-v2; \ fi @@ -374,6 +420,12 @@ RUN if [ "${AKERNEL_ENABLE_RUNC}" = "true" ]; then \ test ! -e /usr/local/bin/runc; \ test ! -e /usr/local/bin/runc-shim; \ fi +RUN if [ "${AKERNEL_ENABLE_ASCEND}" = "true" ]; then \ + test "${AKERNEL_ENABLE_RUNC}" = "true"; \ + chmod 0755 /usr/local/libexec/akernel/ascend-oci-adapter; \ + else \ + test ! -e /usr/local/libexec/akernel/ascend-oci-adapter; \ + fi COPY ./builder/config/yr_services.yaml /tmp/yr_services_rrt.yaml COPY ./builder/config/yr_services_python.yaml /tmp/yr_services_python.yaml @@ -416,6 +468,7 @@ LABEL org.opencontainers.image.version="${AKERNEL_VERSION}" \ org.akernel.gvisor.release="${GVISOR_RELEASE}" \ org.akernel.runc.version="${RUNC_VERSION}" \ org.akernel.runc.enabled="${AKERNEL_ENABLE_RUNC}" \ + org.akernel.ascend.enabled="${AKERNEL_ENABLE_ASCEND}" \ org.akernel.kata.enabled="${AKERNEL_ENABLE_KATA}" \ org.akernel.firecracker.release="${FIRECRACKER_RELEASE}" \ org.akernel.firecracker.enabled="${AKERNEL_ENABLE_FIRECRACKER}" diff --git a/deploy/README.md b/deploy/README.md index 9ba6a48..f09f105 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -44,6 +44,23 @@ Terraform registers the runtime with sandboxd. Direct Helm users must likewise build with `AKERNEL_ENABLE_RUNC=true` and set `node.config.sandboxd.enableRunc=true`. +NVIDIA GPU sandboxes can use runsc or runc. The node container must include +the NVIDIA Container Toolkit hook and must be launched with access to the host +GPU devices and userspace driver libraries. Runc GPU requests additionally +require the optional runc payload and runtime registration described above. + +Ascend 310P3 and 910 A2/A3 support additionally requires an image built with +`AKERNEL_ENABLE_ASCEND=true` and `node.config.sandboxd.enableAscend=true`. +The build also requires `ASCEND_ADAPTER_VERSION`, `ASCEND_ADAPTER_AMD64_URL`, +and `ASCEND_ADAPTER_AMD64_SHA256` for a fixed +[ascend-oci-adapter](https://github.com/openYuanrong-mirror/ascend-oci-adapter) +release bundle. The Ascend option requires runc. The Helm profile mounts the +standard host +driver paths into the privileged node pod; schedule that profile only on NPU +nodes where those paths exist. FlowGW's `/var/queue_schedule` path is optional: +set `node.config.sandboxd.mountAscendQueueSchedule=true` only when that +directory exists on every selected Ascend node. The mount is read-only. + The iptables sandbox NAT backend remains the default. Terraform deployments can set `sandboxd_nat_backend = "bpfnat"` to use sandboxd's experimental embedded TC eBPF backend on nodes without iptables NAT or conntrack modules. diff --git a/deploy/akernel/charts/core/templates/node/configmap.yaml b/deploy/akernel/charts/core/templates/node/configmap.yaml index bfc8989..037a1eb 100644 --- a/deploy/akernel/charts/core/templates/node/configmap.yaml +++ b/deploy/akernel/charts/core/templates/node/configmap.yaml @@ -21,6 +21,20 @@ data: {{- end }} {{- $sandboxdConfig = replace $runcMarker "runc=\"/usr/local/bin/runc\"" $sandboxdConfig }} {{- end }} +{{- $ascendMarker := "# AKERNEL_XPU_ASCEND" }} +{{- if .Values.node.config.sandboxd.enableAscend }} +{{- if not .Values.node.config.sandboxd.enableRunc }} +{{- fail "node.config.sandboxd.enableAscend requires node.config.sandboxd.enableRunc=true" }} +{{- end }} +{{- if not (contains $ascendMarker $sandboxdConfig) }} +{{- fail "node.config.sandboxd.enableAscend requires the # AKERNEL_XPU_ASCEND marker in node.config.sandboxd.config" }} +{{- end }} +{{- $ascendConfig := `[plugin.xpu.ascend] +enabled=true +adapter="/usr/local/libexec/akernel/ascend-oci-adapter" +mount_profile="/etc/akernel/ascend/mounts.json"` }} +{{- $sandboxdConfig = replace $ascendMarker $ascendConfig $sandboxdConfig }} +{{- end }} {{ $sandboxdConfig | indent 4 }} registry.json: | diff --git a/deploy/akernel/charts/core/templates/node/daemonset.yaml b/deploy/akernel/charts/core/templates/node/daemonset.yaml index 01eb11d..2e0c0c2 100644 --- a/deploy/akernel/charts/core/templates/node/daemonset.yaml +++ b/deploy/akernel/charts/core/templates/node/daemonset.yaml @@ -172,6 +172,22 @@ spec: - mountPath: /home/akernel/sandboxd/config.toml name: akernel-node-config subPath: sandboxd_config.toml + {{- if .Values.node.config.sandboxd.enableAscend }} + - mountPath: /usr/local/Ascend/driver + name: ascend-driver + readOnly: true + - mountPath: /usr/local/dcmi + name: ascend-dcmi + readOnly: true + - mountPath: /usr/local/bin/npu-smi + name: ascend-npu-smi + readOnly: true + {{- if .Values.node.config.sandboxd.mountAscendQueueSchedule }} + - mountPath: /var/queue_schedule + name: ascend-queue-schedule + readOnly: true + {{- end }} + {{- end }} - mountPath: /home/akernel name: home-disk {{- if not .Values.node.homeDisk.useEphemeralPVC }} @@ -228,6 +244,26 @@ spec: emptyDir: medium: Memory sizeLimit: 500Mi + {{- if .Values.node.config.sandboxd.enableAscend }} + - name: ascend-driver + hostPath: + path: /usr/local/Ascend/driver + type: Directory + - name: ascend-dcmi + hostPath: + path: /usr/local/dcmi + type: Directory + - name: ascend-npu-smi + hostPath: + path: /usr/local/bin/npu-smi + type: File + {{- if .Values.node.config.sandboxd.mountAscendQueueSchedule }} + - name: ascend-queue-schedule + hostPath: + path: /var/queue_schedule + type: Directory + {{- end }} + {{- end }} {{- if or .Values.monitoring.prometheusEndpoint .Values.monitoring.lokiEndpoint .Values.monitoring.tempoEndpoint }} - name: otel-collector-config configMap: diff --git a/deploy/akernel/charts/core/values.yaml b/deploy/akernel/charts/core/values.yaml index ab638fd..611bb44 100644 --- a/deploy/akernel/charts/core/values.yaml +++ b/deploy/akernel/charts/core/values.yaml @@ -448,6 +448,12 @@ node: # The node image must also be built with AKERNEL_ENABLE_RUNC=true. # Sandboxd probes runc and runc-shim before advertising the runtime. enableRunc: false + # Requires an image built with AKERNEL_ENABLE_ASCEND=true and runc enabled. + # Deploy this profile only to Ascend nodes whose host driver paths exist. + enableAscend: false + # Mount FlowGW's optional queue-scheduling directory when it exists on + # every selected Ascend node. + mountAscendQueueSchedule: false config: |- rootDir="/home/akernel/sandboxd/root" storeDir="/home/akernel/sandboxd/store" @@ -505,6 +511,8 @@ node: shim_binary="/usr/local/bin/runc-shim" kvm_device="/dev/kvm" + # AKERNEL_XPU_ASCEND + # image-manager config. root holds image-manager runtime state # (daemons/, mount_records.db) and is wiped on pod change; input # configs live under /home/akernel/sandboxd/config/ so the wipe never diff --git a/deploy/scripts/build-image.sh b/deploy/scripts/build-image.sh index 53acd8d..f5ef1d8 100755 --- a/deploy/scripts/build-image.sh +++ b/deploy/scripts/build-image.sh @@ -31,6 +31,9 @@ open_yr_core_wheel_url="${OPEN_YR_CORE_WHEEL_URL:-}" open_yr_core_wheel_sha256="${OPEN_YR_CORE_WHEEL_SHA256:-}" rrt_runtime_url="${RRT_RUNTIME_URL:-}" rrt_runtime_sha256="${RRT_RUNTIME_SHA256:-}" +ascend_adapter_version="${ASCEND_ADAPTER_VERSION:-}" +ascend_adapter_amd64_url="${ASCEND_ADAPTER_AMD64_URL:-}" +ascend_adapter_amd64_sha256="${ASCEND_ADAPTER_AMD64_SHA256:-}" print_component_versions=0 component_revision() { @@ -149,6 +152,21 @@ case "${AKERNEL_ENABLE_RUNC:-false}" in *) die "AKERNEL_ENABLE_RUNC must be true or false" ;; esac +case "${AKERNEL_ENABLE_ASCEND:-false}" in + true|false) ;; + *) die "AKERNEL_ENABLE_ASCEND must be true or false" ;; +esac +if [[ "${AKERNEL_ENABLE_ASCEND:-false}" == "true" && + "${AKERNEL_ENABLE_RUNC:-false}" != "true" ]]; then + die "AKERNEL_ENABLE_ASCEND=true requires AKERNEL_ENABLE_RUNC=true" +fi +if [[ "${AKERNEL_ENABLE_ASCEND:-false}" == "true" && + ( -z "${ascend_adapter_version}" || + -z "${ascend_adapter_amd64_url}" || + -z "${ascend_adapter_amd64_sha256}" ) ]]; then + die "AKERNEL_ENABLE_ASCEND=true requires ASCEND_ADAPTER_VERSION, ASCEND_ADAPTER_AMD64_URL, and ASCEND_ADAPTER_AMD64_SHA256" +fi + repository="${repository:-akernel-all-in-one}" tag="${tag:-$(git -C "${AKERNEL_REPO_ROOT}" rev-parse --short HEAD)-$(date +%Y%m%d%H%M%S)}" @@ -208,6 +226,10 @@ node_build_args=( --build-arg "AKERNEL_RUNTIME_PROFILE=${runtime_profile}" --build-arg "AKERNEL_ENABLE_KATA=${AKERNEL_ENABLE_KATA:-true}" --build-arg "AKERNEL_ENABLE_RUNC=${AKERNEL_ENABLE_RUNC:-false}" + --build-arg "AKERNEL_ENABLE_ASCEND=${AKERNEL_ENABLE_ASCEND:-false}" + --build-arg "ASCEND_ADAPTER_VERSION=${ascend_adapter_version}" + --build-arg "ASCEND_ADAPTER_AMD64_URL=${ascend_adapter_amd64_url}" + --build-arg "ASCEND_ADAPTER_AMD64_SHA256=${ascend_adapter_amd64_sha256}" --build-arg "AKERNEL_ENABLE_FIRECRACKER=${AKERNEL_ENABLE_FIRECRACKER:-true}" --build-arg "AKERNEL_VERSION=${akernel_version}" --build-arg "AKERNEL_REVISION=${akernel_revision}" diff --git a/deploy/scripts/configure.sh b/deploy/scripts/configure.sh index 69f1915..5e388d0 100755 --- a/deploy/scripts/configure.sh +++ b/deploy/scripts/configure.sh @@ -31,6 +31,7 @@ image_tag_override="" install_monitor_override="" install_dragonfly_override="" enable_runc_override="" +enable_ascend_override="" grafana_public_access_override="" grafana_admin_password_override="" iam_seed_hex_override="" @@ -121,6 +122,10 @@ while [[ $# -gt 0 ]]; do enable_runc_override="$2" shift 2 ;; + --enable-ascend) + enable_ascend_override="$2" + shift 2 + ;; --grafana-public-access) grafana_public_access_override="$2" shift 2 @@ -221,6 +226,7 @@ set_or_prompt image_tag "All-in-one image tag" "${default_tag}" "${image_tag_ove set_or_prompt install_monitor "Install monitor chart (true/false)" "true" "${install_monitor_override}" set_or_prompt install_dragonfly "Install Dragonfly and dedicated node pools (true/false)" "false" "${install_dragonfly_override}" set_or_prompt enable_runc "Enable the optional runc runtime (true/false)" "false" "${enable_runc_override}" +set_or_prompt enable_ascend "Enable Ascend 310P3 or 910 A2/A3 NPUs (true/false)" "false" "${enable_ascend_override}" set_or_prompt grafana_public_access "Expose Grafana LoadBalancer (true/false)" "true" "${grafana_public_access_override}" set_or_prompt grafana_admin_password \ "Grafana admin password (empty to generate)" "" \ @@ -231,6 +237,10 @@ fi install_monitor="$(normalize_bool "${install_monitor}")" install_dragonfly="$(normalize_bool "${install_dragonfly}")" enable_runc="$(normalize_bool "${enable_runc}")" +enable_ascend="$(normalize_bool "${enable_ascend}")" +if [[ "${enable_ascend}" == "true" && "${enable_runc}" != "true" ]]; then + die "Ascend NPU support requires enable_runc=true" +fi grafana_public_access="$(normalize_bool "${grafana_public_access}")" dir="$(state_dir "${env_name}")" @@ -352,6 +362,7 @@ grafana_admin_password = "${grafana_admin_password}" install_dragonfly = ${install_dragonfly} enable_runc = ${enable_runc} +enable_ascend = ${enable_ascend} EOF ;; huaweicloud) @@ -406,6 +417,7 @@ grafana_admin_password = "${grafana_admin_password}" install_dragonfly = ${install_dragonfly} enable_runc = ${enable_runc} +enable_ascend = ${enable_ascend} EOF ;; esac @@ -427,6 +439,7 @@ CORE_NAMESPACE=akernel MONITOR_NAMESPACE=akernel-monitor INSTALL_DRAGONFLY=${install_dragonfly} AKERNEL_ENABLE_RUNC=${enable_runc} +AKERNEL_ENABLE_ASCEND=${enable_ascend} EOF chmod 600 "${tfvars_file}" "${config_file}" diff --git a/deploy/standalone/README.md b/deploy/standalone/README.md index b24912b..4da4c8a 100644 --- a/deploy/standalone/README.md +++ b/deploy/standalone/README.md @@ -37,8 +37,8 @@ Clients then select it with `Sandbox(runtime="runc")`. A sandbox may request the configured KVM character device with `extra_config={"enableKVM": True}` when the host exposes `/dev/kvm`. -Experimental NVIDIA GPU sandboxes use gVisor nvproxy. The host must provide a -compatible NVIDIA driver and NVIDIA Container Toolkit. Enable GPU access to +Experimental NVIDIA GPU sandboxes support gVisor nvproxy and native runc. The +host must provide a compatible NVIDIA driver and NVIDIA Container Toolkit. Enable GPU access to the node container with: ```bash @@ -49,6 +49,27 @@ The all-in-one image contains `nvidia-container-cli`, but not the host driver. Use `AKERNEL_GPU_DEVICES` to override Docker's `--gpus` value when only a device subset should be assigned. +To use the same GPU with the native runc sandbox backend, build and launch the +node with both `AKERNEL_ENABLE_RUNC=true` and `AKERNEL_ENABLE_GPU=true`, then +select `Sandbox(runtime="runc", xpu="gpu::1")`. + +Ascend 310P3 or 910 A2/A3 requires an image built with both optional payloads and a +host with the standard driver paths: + +```bash +AKERNEL_ENABLE_RUNC=true AKERNEL_ENABLE_ASCEND=true make build \ + IMAGE_REPOSITORY=akernel-ascend IMAGE_TAG=local +IMAGE=akernel-ascend:local \ + AKERNEL_ENABLE_RUNC=true AKERNEL_ENABLE_ASCEND=true ./start.sh +``` + +`start.sh` verifies and mounts `/usr/local/Ascend/driver`, +`/usr/local/dcmi`, and `/usr/local/bin/npu-smi` into the node container. +When FlowGW has created `/var/queue_schedule`, `start.sh` also mounts that +directory read-only; its absence does not prevent an Ascend standalone node +from starting. Sandboxd then applies a narrower, versioned mount profile to +each leased runc sandbox. + Explicit sandbox storage quotas for runsc and Firecracker use the bounded ext4 filestore mounted at `/home/akernel/filestore`. The standalone data directory is bind-mounted from the host, and sandboxd creates a loop-backed filesystem diff --git a/deploy/standalone/config/sandboxd_config.toml b/deploy/standalone/config/sandboxd_config.toml index ea41dd1..17cce6c 100644 --- a/deploy/standalone/config/sandboxd_config.toml +++ b/deploy/standalone/config/sandboxd_config.toml @@ -61,6 +61,8 @@ state_root="/run/sandboxd/runc" shim_binary="/usr/local/bin/runc-shim" kvm_device="/dev/kvm" +# AKERNEL_XPU_ASCEND + # image-manager config. root holds image-manager runtime state and is wiped # on pod change; input configs live under /home/akernel/sandboxd/config/ so the # wipe never hits them. diff --git a/deploy/standalone/start.sh b/deploy/standalone/start.sh index 8faed46..eff2a1c 100755 --- a/deploy/standalone/start.sh +++ b/deploy/standalone/start.sh @@ -24,6 +24,7 @@ SANDBOXD_CONFIG_FILE="${DATA_DIR}/sandboxd/config.toml" AKERNEL_NAT_BACKEND="${AKERNEL_NAT_BACKEND:-iptables}" AKERNEL_ENABLE_RUNC="${AKERNEL_ENABLE_RUNC:-false}" YR_IMAGE_PROCESS_CONFIG="${YR_IMAGE_PROCESS_CONFIG:-/run/akernel/yr-image-process.json}" +AKERNEL_ENABLE_ASCEND="${AKERNEL_ENABLE_ASCEND:-false}" LITEBUS_DATA_KEY="" # Container runtime command (docker or pouch) @@ -31,6 +32,7 @@ DOCKER_CMD="" DOCKER_PREFIX=() PROXY_RUN_ARGS=() GPU_RUN_ARGS=() +ASCEND_RUN_ARGS=() # Colors for output RED='\033[0;31m' @@ -104,6 +106,18 @@ check_prerequisites() { exit 1 ;; esac + case "${AKERNEL_ENABLE_ASCEND}" in + true|false) + ;; + *) + log_error "AKERNEL_ENABLE_ASCEND must be true or false" + exit 1 + ;; + esac + if [[ "${AKERNEL_ENABLE_ASCEND}" == "true" && "${AKERNEL_ENABLE_RUNC}" != "true" ]]; then + log_error "AKERNEL_ENABLE_ASCEND=true requires AKERNEL_ENABLE_RUNC=true" + exit 1 + fi # Create data directory mkdir -p "${DATA_DIR}" @@ -237,6 +251,35 @@ configure_gpu() { log_info "Enabling NVIDIA GPU access for the AKernel node container" } +configure_ascend() { + if [[ "${AKERNEL_ENABLE_ASCEND}" != "true" ]]; then + return 0 + fi + local required_path + for required_path in \ + /usr/local/Ascend/driver \ + /usr/local/dcmi \ + /usr/local/bin/npu-smi; do + if [[ ! -e "${required_path}" ]]; then + log_error "Ascend host path is missing: ${required_path}" + exit 1 + fi + done + ASCEND_RUN_ARGS=( + -v /usr/local/Ascend/driver:/usr/local/Ascend/driver:ro + -v /usr/local/dcmi:/usr/local/dcmi:ro + -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi:ro + ) + # FlowGW creates this directory only on hosts that use queue scheduling. + # Do not create it here: its absence means the optional integration is off. + if [[ -d /var/queue_schedule ]]; then + ASCEND_RUN_ARGS+=( + -v /var/queue_schedule:/var/queue_schedule:ro + ) + fi + log_info "Enabling Ascend NPU access for runc sandboxes" +} + configure_network() { local config_tmp="${SANDBOXD_CONFIG_FILE}.tmp" local sed_args=( @@ -268,6 +311,20 @@ configure_network() { -e 's|^[[:space:]]*# AKERNEL_RUNTIME_RUNC[[:space:]]*$|runc="/usr/local/bin/runc"|' ) fi + if [[ "${AKERNEL_ENABLE_ASCEND}" == "true" ]]; then + if ! grep -q '^[[:space:]]*# AKERNEL_XPU_ASCEND[[:space:]]*$' \ + "${CONFIG_DIR}/sandboxd_config.toml"; then + log_error "AKERNEL_ENABLE_ASCEND requires the # AKERNEL_XPU_ASCEND marker in sandboxd_config.toml" + exit 1 + fi + sed_args+=( + -e '/^[[:space:]]*# AKERNEL_XPU_ASCEND[[:space:]]*$/c\ +[plugin.xpu.ascend]\ +enabled=true\ +adapter="/usr/local/libexec/akernel/ascend-oci-adapter"\ +mount_profile="/etc/akernel/ascend/mounts.json"' + ) + fi sed "${sed_args[@]}" "${CONFIG_DIR}/sandboxd_config.toml" > "${config_tmp}" mv "${config_tmp}" "${SANDBOXD_CONFIG_FILE}" @@ -377,6 +434,7 @@ start_node_container() { -e ENABLE_METRICS="${ENABLE_METRICS:-false}" \ "${PROXY_RUN_ARGS[@]}" \ "${GPU_RUN_ARGS[@]}" \ + "${ASCEND_RUN_ARGS[@]}" \ --entrypoint=/usr/local/bin/akernel-entrypoint \ -v "${DATA_DIR}:/home/akernel" \ -v "${CONFIG_DIR}/oss_auths.json:/home/akernel/sandboxd/config/oss_auths.json:ro" \ @@ -530,6 +588,7 @@ ensure_image "${IMAGE}" ensure_image "${TRAEFIK_IMAGE}" configure_container_proxy configure_gpu +configure_ascend configure_network prepare_host_network_modules start_node_container diff --git a/deploy/terraform/aliyun/main.tf b/deploy/terraform/aliyun/main.tf index c23bf15..a6bd3a6 100644 --- a/deploy/terraform/aliyun/main.tf +++ b/deploy/terraform/aliyun/main.tf @@ -154,6 +154,7 @@ locals { traefik_enabled = var.traefik_enabled sandboxd_nat_backend = var.sandboxd_nat_backend enable_runc = var.enable_runc + enable_ascend = var.enable_ascend node_secret_create = var.node_secret_create node_home_use_csi_ephemeral = var.node_home_use_csi_ephemeral node_home_csi_storage_class = local.effective_node_home_csi_sc diff --git a/deploy/terraform/aliyun/terraform.tfvars.example b/deploy/terraform/aliyun/terraform.tfvars.example index eb411f9..39c8b93 100644 --- a/deploy/terraform/aliyun/terraform.tfvars.example +++ b/deploy/terraform/aliyun/terraform.tfvars.example @@ -176,6 +176,10 @@ node_secret_create = true # matching AKERNEL_ENABLE_RUNC=true profile setting. enable_runc = false +# Optional Ascend 310P3 or 910 A2/A3 support. Requires enable_runc=true and an image +# built with AKERNEL_ENABLE_ASCEND=true; use only for NPU node pools. +enable_ascend = false + # --- Monitor --- install_monitor = true monitor_namespace = "akernel-monitor" diff --git a/deploy/terraform/aliyun/values-akernel.yaml.tmpl b/deploy/terraform/aliyun/values-akernel.yaml.tmpl index 0ce6d32..ce5a9f8 100644 --- a/deploy/terraform/aliyun/values-akernel.yaml.tmpl +++ b/deploy/terraform/aliyun/values-akernel.yaml.tmpl @@ -145,6 +145,7 @@ node: cpuQuota: "100%" sandboxd: enableRunc: ${enable_runc} + enableAscend: ${enable_ascend} config: |- rootDir="/home/akernel/sandboxd/root" storeDir="/home/akernel/sandboxd/store" @@ -201,6 +202,8 @@ node: shim_binary="/usr/local/bin/runc-shim" kvm_device="/dev/kvm" + # AKERNEL_XPU_ASCEND + # image-manager config. root holds image-manager runtime state # (daemons/, mount_records.db) and is wiped on pod change; input # configs live under /home/akernel/sandboxd/config/ so the wipe never diff --git a/deploy/terraform/aliyun/variables.tf b/deploy/terraform/aliyun/variables.tf index d390e09..b4a4786 100644 --- a/deploy/terraform/aliyun/variables.tf +++ b/deploy/terraform/aliyun/variables.tf @@ -343,6 +343,12 @@ variable "enable_runc" { default = false } +variable "enable_ascend" { + type = bool + description = "Enable Ascend 310P3 or 910 A2/A3 physical NPUs; requires enable_runc and a node image built with AKERNEL_ENABLE_ASCEND=true." + default = false +} + variable "node_home_use_csi_ephemeral" { type = bool description = "Mount /home/akernel from a per-node CSI ephemeral volume instead of hostPath /home/akernel." diff --git a/deploy/terraform/huaweicloud/main.tf b/deploy/terraform/huaweicloud/main.tf index da5e56b..fbbfddc 100644 --- a/deploy/terraform/huaweicloud/main.tf +++ b/deploy/terraform/huaweicloud/main.tf @@ -113,6 +113,7 @@ locals { master_service_loadbalancer_ip = var.master_public_access_8888 ? var.master_service_loadbalancer_ip : "" sandboxd_nat_backend = var.sandboxd_nat_backend enable_runc = var.enable_runc + enable_ascend = var.enable_ascend node_secret_create = var.node_secret_create node_home_use_csi_ephemeral = var.node_home_use_csi_ephemeral node_home_csi_storage_class = var.node_home_csi_storage_class diff --git a/deploy/terraform/huaweicloud/terraform.tfvars.example b/deploy/terraform/huaweicloud/terraform.tfvars.example index c0f9e1c..9aed40e 100644 --- a/deploy/terraform/huaweicloud/terraform.tfvars.example +++ b/deploy/terraform/huaweicloud/terraform.tfvars.example @@ -93,6 +93,10 @@ node_image_tag = "" # matching AKERNEL_ENABLE_RUNC=true profile setting. enable_runc = false +# Optional Ascend 310P3 or 910 A2/A3 support. Requires enable_runc=true and an image +# built with AKERNEL_ENABLE_ASCEND=true; use only for NPU node pools. +enable_ascend = false + # Enable the /internal-stats endpoint sidecar on Traefik. # traefik_internal_stats_enabled = true # traefik_internal_stats_image = "my-registry.example.com/busybox:1.37.0-musl" diff --git a/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl b/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl index 89c3602..d92079a 100644 --- a/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl +++ b/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl @@ -125,6 +125,7 @@ node: cpuQuota: "100%" sandboxd: enableRunc: ${enable_runc} + enableAscend: ${enable_ascend} config: |- rootDir="/home/akernel/sandboxd/root" storeDir="/home/akernel/sandboxd/store" @@ -179,6 +180,8 @@ node: shim_binary="/usr/local/bin/runc-shim" kvm_device="/dev/kvm" + # AKERNEL_XPU_ASCEND + # image-manager config. root holds image-manager runtime state # (daemons/, mount_records.db) and is wiped on pod change; input # configs live under /home/akernel/sandboxd/config/ so the wipe never diff --git a/deploy/terraform/huaweicloud/variables.tf b/deploy/terraform/huaweicloud/variables.tf index 5c668bb..f278e06 100644 --- a/deploy/terraform/huaweicloud/variables.tf +++ b/deploy/terraform/huaweicloud/variables.tf @@ -375,6 +375,12 @@ variable "enable_runc" { default = false } +variable "enable_ascend" { + type = bool + description = "Enable Ascend 310P3 or 910 A2/A3 physical NPUs; requires enable_runc and a node image built with AKERNEL_ENABLE_ASCEND=true." + default = false +} + variable "node_home_use_csi_ephemeral" { type = bool description = "Mount /home/akernel from a per-node CSI ephemeral volume instead of hostPath /home/akernel." diff --git a/sdk/python/README.md b/sdk/python/README.md index c18d5ae..db79366 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -16,7 +16,7 @@ It supports two backends: - [Navigation](#navigation) - [Install and configure](#install-and-configure) - [Create a sandbox](#create-a-sandbox) - - [Experimental GPU and writable storage](#experimental-gpu-and-writable-storage) + - [Experimental GPU, NPU, and writable storage](#experimental-gpu-npu-and-writable-storage) - [Network ACLs](#network-acls) - [Sandbox runtimes](#sandbox-runtimes) - [Commands](#commands) @@ -113,7 +113,7 @@ Sandbox( ) ``` -### Experimental GPU and writable storage +### Experimental GPU, NPU, and writable storage Request a whole NVIDIA GPU by type, exact product model, and count: @@ -124,9 +124,18 @@ with Sandbox(xpu="gpu:l20:1") as sandbox: The `type:model:count` value is case-insensitive and canonicalized to lower case. The model is required and matched exactly; wildcard models are not -supported. The bundled backend currently requires the gVisor `runsc` runtime -and a node configured for gVisor nvproxy. Runtime compatibility is validated -by the backend rather than the SDK. +supported. The bundled backend supports GPU sandboxes with gVisor `runsc` or +native `runc`. Runsc remains subject to its nvproxy driver compatibility gate; +that gate does not disable native-runc GPU support. + +Request one physical Ascend 310P or 910 A2/A3 NPU with runc: + +```python +with Sandbox(runtime="runc", xpu="npu:ascend910b4:1") as sandbox: + print(sandbox.commands.run("npu-smi info").stdout) +``` + +The model token is matched exactly against the normalized driver-reported resources of the selected node, without a per-SKU allowlist. For a node reporting `ascend910`, use `npu:ascend910:1`; do not infer a different SKU token. Runtime and provider-owned environment validation remain backend-owned. Ascend driver library directories are added ahead of the image/request library paths, preserving CANN and application directories. Set the writable root filesystem quota in MiB: @@ -314,8 +323,8 @@ with Sandbox( `enableKVM` is owned by the runc backend and requires a usable `/dev/kvm` on the selected node. Runc supports OCI/EROFS root filesystems, read-only mounts, networking, command execution, and the default writable overlay. Experimental -GPU requests remain runsc-only; explicit `storage_mb` quotas are supported by -runsc and Firecracker. See the +GPU requests support runsc and runc, while Ascend NPU requests are runc-only; +explicit `storage_mb` quotas are supported by runsc and Firecracker. See the [sandbox runtime comparison](../../src/sandboxd/doc/runtime.md) for the runtime capability boundaries. diff --git a/sdk/python/akernel_sdk/_sandbox_resources.py b/sdk/python/akernel_sdk/_sandbox_resources.py index 3682065..5b11a3e 100644 --- a/sdk/python/akernel_sdk/_sandbox_resources.py +++ b/sdk/python/akernel_sdk/_sandbox_resources.py @@ -37,8 +37,8 @@ def normalize_xpu(value: str | None) -> str | None: if len(fields) != 3: raise ValueError("xpu must contain exactly three fields: type:model:count") xpu_type, model, count_text = (field.strip().lower() for field in fields) - if xpu_type != "gpu": - raise ValueError("xpu type must be gpu") + if xpu_type not in {"gpu", "npu"}: + raise ValueError("xpu type must be gpu or npu") if not model: raise ValueError("xpu model must be non-empty") if not _XPU_MODEL_PATTERN.fullmatch(model): diff --git a/sdk/python/akernel_sdk/sandbox.py b/sdk/python/akernel_sdk/sandbox.py index d939611..5a2f671 100644 --- a/sdk/python/akernel_sdk/sandbox.py +++ b/sdk/python/akernel_sdk/sandbox.py @@ -230,9 +230,9 @@ def __init__( inherit_entrypoint: Start the OCI image's effective ENTRYPOINT and CMD as the sandbox workload. Valid only with ``image``. xpu: Experimental whole-device accelerator request in - ``type:model:count`` format. Currently only exact-model NVIDIA - GPU requests are supported. The backend validates runtime - compatibility. + ``type:model:count`` format. Exact-model NVIDIA GPU and Ascend + NPU requests are supported; the backend validates runtime and + node compatibility. storage_mb: Experimental writable root filesystem quota in MiB. When omitted, the configured default is used. Explicit quotas are validated against the selected runtime by the backend. diff --git a/sdk/python/examples/gpu_sandbox.py b/sdk/python/examples/gpu_sandbox.py index 861b448..2e2ba25 100644 --- a/sdk/python/examples/gpu_sandbox.py +++ b/sdk/python/examples/gpu_sandbox.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Run a CUDA sample in an experimental gVisor GPU sandbox.""" +"""Run a CUDA sample in an experimental runsc or runc GPU sandbox.""" import os @@ -26,11 +26,14 @@ def main() -> None: model = os.environ.get("AKERNEL_GPU_MODEL", "a10").strip().lower() + runtime = os.environ.get("AKERNEL_GPU_RUNTIME", "runsc").strip().lower() + if runtime not in {"runsc", "runc"}: + raise ValueError("AKERNEL_GPU_RUNTIME must be runsc or runc") with Sandbox( image=CUDA_SAMPLE_IMAGE, - runtime="runsc", + runtime=runtime, xpu=f"gpu:{model}:1", - storage_mb=512, + storage_mb=512 if runtime == "runsc" else None, cpu=1000, memory=2048, schedule_timeout=120, diff --git a/sdk/python/tests/unit/test_sandbox.py b/sdk/python/tests/unit/test_sandbox.py index 739c101..5810dc3 100644 --- a/sdk/python/tests/unit/test_sandbox.py +++ b/sdk/python/tests/unit/test_sandbox.py @@ -297,7 +297,7 @@ def test_xpu_request_validation(self): (1, TypeError), ("gpu", ValueError), ("gpu::1", ValueError), - ("npu:l20:1", ValueError), + ("tpu:l20:1", ValueError), ("gpu:l20:0", ValueError), ("gpu:l20:1.5", ValueError), ("gpu:l20/evil:1", ValueError), diff --git a/sdk/python/tests/unit/test_sandbox_resources.py b/sdk/python/tests/unit/test_sandbox_resources.py index 98642a4..2fe03fc 100644 --- a/sdk/python/tests/unit/test_sandbox_resources.py +++ b/sdk/python/tests/unit/test_sandbox_resources.py @@ -30,6 +30,10 @@ def test_xpu_is_canonical_and_exact_model_is_escaped(self): xpu_custom_resource("gpu:a10.2:2"), (r"GPU/a10\.2/count", 2.0), ) + self.assertEqual( + xpu_custom_resource("NPU:Ascend910_9391:1"), + ("NPU/ascend910_9391/count", 1.0), + ) def test_storage_wire_value_and_upper_bound(self): self.assertEqual(storage_bytes(256), float(256 * 1024 * 1024)) diff --git a/src/sandboxd b/src/sandboxd index b892414..52ea5cf 160000 --- a/src/sandboxd +++ b/src/sandboxd @@ -1 +1 @@ -Subproject commit b892414a3b21f3c2a4481322767362af14fc701f +Subproject commit 52ea5cf2c3c1e52d63e36329ca660c26eba97d6f