diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e6c682..b24c993 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,7 +121,11 @@ jobs: sudo modprobe loop sudo modprobe erofs sudo modprobe br_netfilter + sudo modprobe kvm + sudo modprobe kvm_intel || sudo modprobe kvm_amd || true test -c /dev/net/tun + test -c /dev/kvm + sudo chmod 0666 /dev/kvm sudo sysctl -w net.bridge.bridge-nf-call-iptables=1 - name: Build all-in-one image @@ -166,6 +170,25 @@ jobs: timeout 120s python "sdk/python/examples/${example}" done + - name: Run runsc and Firecracker checkpoint restore E2E + run: | + gateway_ip="$(docker inspect \ + --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' \ + akernel-traefik)" + test -n "${gateway_ip}" + token="$(cat deploy/standalone/data/token)" + export AKERNEL_TOKEN="${token}" + export AKERNEL_SERVER_ADDRESS="${gateway_ip}" + export AKERNEL_RUN_INTEGRATION=1 + export PYTHONPATH="${GITHUB_WORKSPACE}/sdk/python" + + for runtime in runsc firecracker; do + echo "=== Checkpoint/restore runtime=${runtime} ===" + AKERNEL_TEST_RUNTIME="${runtime}" timeout 300s python \ + sdk/python/tests/integration/test_sandbox.py \ + SandboxCheckpointIntegrationTest -v + done + - name: Show standalone diagnostics if: failure() run: | diff --git a/AGENTS.md b/AGENTS.md index 7fdbfed..f1db140 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -368,6 +368,25 @@ quotas for runsc and Firecracker use this local-disk filestore. Without an explicit quota, runsc retains its configured memory-backed overlay while Firecracker creates its configured sparse ext4 default. +The bundled node always enables YuanRong's sandbox snapshot data plane; there +is no separate global pause/resume/snapshot switch. It uses the DataSystem +backend by default and `/home/akernel/checkpoints` on the +existing node home disk as same-node persistent staging. Remote storage uses +only `snapshot_storage_backend=s3` with the shared SigV4 client and provider +profiles `generic`, `obs`, or `oss`; the old snapshot OBS backend and +`snapshot_obs_*` settings are unsupported. Provider profiles must allow legal +private endpoints and CNAMEs, while OSS requires virtual-hosted addressing. +Remote snapshots over 5 GiB are rejected before upload until multipart copy is +implemented. Do not remove the independent DataSystem backend or the unrelated +object-storage code-package downloader. +Kubernetes node roles use the downward-API `NODE_NAME` as the stable YuanRong +node identity, so a DaemonSet Pod replacement on the same physical node can +rebuild its local runtime and snapshot views. Standalone falls back to the +container hostname. +The public SDK does not expose snapshot TTLs: reusable checkpoints remain +until explicitly deleted. Keep the selected remote backend and checkpoint +staging configuration together when changing node startup arguments. + Keep detailed SDK reference material with the SDK. The root README should contain only the project-level entry points and representative examples: diff --git a/README.md b/README.md index aa78185..9ed91ce 100644 --- a/README.md +++ b/README.md @@ -207,7 +207,7 @@ See the complete [basic usage example](./sdk/python/examples/basic_usage.py), th - [x] Optional native Linux runc runtime - [x] Sandbox network ACL - [ ] Fork-based sandbox launch based on gVisor -- [ ] Sandbox checkpoint and restore +- [x] Sandbox checkpoint and restore for runsc and Firecracker - [ ] Support for GKE and AWS - [x] Cgroup v2 node support diff --git a/builder/node.Dockerfile b/builder/node.Dockerfile index 4b8404a..a1d6d61 100644 --- a/builder/node.Dockerfile +++ b/builder/node.Dockerfile @@ -394,6 +394,10 @@ COPY ./builder/scripts/yr_node_bootstrap.sh ${YR_INSTALLATION_DIR}/yr_node_boots COPY ./builder/scripts/master_entrypoint.sh ${YR_INSTALLATION_DIR}/entrypoint.sh COPY ./builder/scripts/*.sh /root/ COPY ./builder/systemd_services/*.service /etc/systemd/system/ +RUN chmod 0755 \ + /root/yr_pause_resume_args.sh \ + /root/detect-openyuanrong-s3-snapshot-capability.sh && \ + /root/detect-openyuanrong-s3-snapshot-capability.sh ${YR_INSTALLATION_DIR} RUN curl -fSL --retry 10 --retry-delay 2 --retry-all-errors \ "${OTELCOL_CONTRIB_URL}" \ diff --git a/builder/scripts/detect-openyuanrong-s3-snapshot-capability.sh b/builder/scripts/detect-openyuanrong-s3-snapshot-capability.sh new file mode 100755 index 0000000..1b64ee8 --- /dev/null +++ b/builder/scripts/detect-openyuanrong-s3-snapshot-capability.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# Copyright (c) 2026 Ant Group Corporation. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +yr_root="${1:?usage: $0 YR_ROOT}" +marker="${yr_root}/.akernel-s3-snapshot-capable" +config="${yr_root}/deploy/process/config.sh" +install="${yr_root}/functionsystem/deploy/install.sh" +agent="${yr_root}/functionsystem/bin/function_agent" + +rm -f "${marker}" +if [[ -f "${config}" && -f "${install}" && -x "${agent}" ]] \ + && grep -Fq 'snapshot_s3_provider:' "${config}" \ + && grep -Fq -- '--snapshot_s3_provider="${SNAPSHOT_S3_PROVIDER:-}"' "${install}" \ + && grep -aFq 'snapshot_s3_provider' "${agent}" \ + && grep -aFq 'remote S3 snapshot exceeds the 5 GiB capability limit' "${agent}"; then + touch "${marker}" +fi diff --git a/builder/scripts/tests/test-openyuanrong-s3-capability.sh b/builder/scripts/tests/test-openyuanrong-s3-capability.sh new file mode 100755 index 0000000..6801df9 --- /dev/null +++ b/builder/scripts/tests/test-openyuanrong-s3-capability.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# Copyright (c) 2026 Ant Group Corporation. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +detector="${root}/builder/scripts/detect-openyuanrong-s3-snapshot-capability.sh" +tmp=$(mktemp -d) +trap 'rm -rf "${tmp}"' EXIT +mkdir -p "${tmp}/deploy/process" "${tmp}/functionsystem/deploy" "${tmp}/functionsystem/bin" +printf '%s\n' 'snapshot_s3_provider:' >"${tmp}/deploy/process/config.sh" +printf '%s\n' '--snapshot_s3_provider="${SNAPSHOT_S3_PROVIDER:-}"' \ + >"${tmp}/functionsystem/deploy/install.sh" +printf '%s\n' 'snapshot_s3_provider' 'remote S3 snapshot exceeds the 5 GiB capability limit' \ + >"${tmp}/functionsystem/bin/function_agent" +chmod +x "${tmp}/functionsystem/bin/function_agent" + +"${detector}" "${tmp}" +[[ -f "${tmp}/.akernel-s3-snapshot-capable" ]] + +printf '%s\n' 'legacy function agent' >"${tmp}/functionsystem/bin/function_agent" +"${detector}" "${tmp}" +[[ ! -e "${tmp}/.akernel-s3-snapshot-capable" ]] + +echo "openYuanRong S3 capability detection checks passed" diff --git a/builder/scripts/yr_node_bootstrap.sh b/builder/scripts/yr_node_bootstrap.sh index 52e5e21..6f25e59 100755 --- a/builder/scripts/yr_node_bootstrap.sh +++ b/builder/scripts/yr_node_bootstrap.sh @@ -5,6 +5,7 @@ # SPDX-License-Identifier: Apache-2.0 ulimit -n 32768 export YR_RUNTIME_BACKEND=sandboxd +source /root/yr_pause_resume_args.sh resolve_node_ip() { local default_device @@ -41,6 +42,13 @@ resolve_node_ip() { YR_NODE_IP="$(resolve_node_ip)" echo "Using ${YR_NODE_IP} as the YuanRong node address" +CHECKPOINT_DIR="/home/akernel/checkpoints" +mkdir -p "${CHECKPOINT_DIR}" +configure_snapshot_args \ + /home/yuanrong/.akernel-rrt-capable \ + "${CHECKPOINT_DIR}" \ + "${AKS_LOCAL_MODE:-false}" \ + /home/yuanrong/.akernel-s3-snapshot-capable || exit 1 # Select the legacy etcd registry or the FunctionMaster HTTP provider. if [ "${TRAEFIK_MODE:-etcd}" = "etcd" ]; then @@ -108,7 +116,8 @@ if [ "x${AKS_LOCAL_MODE}" == "xtrue" ]; then --frontend_lease_bypass true \ --force_low_reliability_instance true \ --enable_sandbox_router true \ - --enable_direct_routing false + --enable_direct_routing false \ + "${standalone_snapshot_args[@]}" else /usr/bin/yr start \ --ip_address "${YR_NODE_IP}" \ @@ -130,7 +139,7 @@ else --metrics_config_file "/home/yuanrong/metrics/metrics_config.json" \ --enable_trace ${ENABLE_TRACE} \ --trace_config "$(cat /home/yuanrong/trace/trace_config.json)" \ - -n ${HOSTNAME} \ + -n "${NODE_NAME:-${HOSTNAME}}" \ --enable_traefik_registry=${ENABLE_TRAEFIK_REGISTRY} \ --traefik_enable_tls=${TRAEFIK_ENABLE_TLS:-false} \ --traefik_etcd_prefix=traefik \ @@ -145,5 +154,6 @@ else --function_proxy_merge_process_enable true \ --enable_direct_routing false \ --force_low_reliability_instance true \ + "${snapshot_args[@]}" \ --block true fi diff --git a/builder/scripts/yr_pause_resume_args.sh b/builder/scripts/yr_pause_resume_args.sh new file mode 100755 index 0000000..67c78c0 --- /dev/null +++ b/builder/scripts/yr_pause_resume_args.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +# Copyright (c) 2026 Ant Group Corporation. +# SPDX-License-Identifier: Apache-2.0 + +configure_snapshot_args() { + local rrt_capability_file="${1:?RRT capability file is required}" + local checkpoint_dir="${2:?checkpoint directory is required}" + local standalone="${3:?standalone mode value is required}" + local s3_capability_file="${4:-/home/yuanrong/.akernel-s3-snapshot-capable}" + local backend="${AKERNEL_SNAPSHOT_STORAGE_BACKEND:-datasystem}" + + snapshot_args=() + standalone_snapshot_args=() + unset SNAPSHOT_S3_ACCESS_KEY SNAPSHOT_S3_SECRET_KEY SNAPSHOT_S3_SECURITY_TOKEN + if [ ! -f "${rrt_capability_file}" ] \ + && [ ! -f /home/yuanrong/yr-runtime-rootfs.img ]; then + echo "snapshot requires an image built with the RRT runtime" >&2 + return 1 + fi + mkdir -p "${checkpoint_dir}" + if [ ! -w "${checkpoint_dir}" ]; then + echo "checkpoint directory is not writable: ${checkpoint_dir}" >&2 + return 1 + fi + snapshot_args=( + --snapshot_storage_backend "${backend}" + --checkpoint_dir "${checkpoint_dir}" + ) + case "${backend}" in + datasystem) ;; + s3) + if [ ! -f "${s3_capability_file}" ]; then + echo "S3 snapshot storage requires an S3-capable openYuanRong core" >&2 + return 1 + fi + local provider="${AKERNEL_SNAPSHOT_S3_PROVIDER:-}" + local endpoint="${AKERNEL_SNAPSHOT_S3_ENDPOINT:-}" + local region="${AKERNEL_SNAPSHOT_S3_REGION:-}" + local bucket="${AKERNEL_SNAPSHOT_S3_BUCKET:-}" + local access_key="${AKERNEL_SNAPSHOT_S3_ACCESS_KEY:-}" + local secret_key="${AKERNEL_SNAPSHOT_S3_SECRET_KEY:-}" + local security_token="${AKERNEL_SNAPSHOT_S3_SECURITY_TOKEN:-}" + local use_https="${AKERNEL_SNAPSHOT_S3_USE_HTTPS:-true}" + local path_style="${AKERNEL_SNAPSHOT_S3_PATH_STYLE:-true}" + case "${provider}" in generic|obs|oss) ;; *) + echo "AKERNEL_SNAPSHOT_S3_PROVIDER must be generic, obs, or oss" >&2 + return 1 + esac + if [ -z "${endpoint}" ] || [ -z "${region}" ] || [ -z "${bucket}" ] || \ + [ -z "${access_key}" ] || [ -z "${secret_key}" ]; then + echo "S3 snapshot storage requires endpoint, region, bucket, access key, and secret key" >&2 + return 1 + fi + case "${use_https}" in true|false) ;; *) + echo "AKERNEL_SNAPSHOT_S3_USE_HTTPS must be true or false" >&2 + return 1 + esac + case "${path_style}" in true|false) ;; *) + echo "AKERNEL_SNAPSHOT_S3_PATH_STYLE must be true or false" >&2 + return 1 + esac + if [ "${provider}" = "oss" ] && [ "${path_style}" = "true" ]; then + echo "OSS S3-compatible snapshot storage requires virtual-hosted addressing" >&2 + return 1 + fi + snapshot_args+=( + --snapshot_s3_provider "${provider}" + --snapshot_s3_endpoint "${endpoint}" + --snapshot_s3_region "${region}" + --snapshot_s3_bucket "${bucket}" + --snapshot_s3_use_https "${use_https}" + --snapshot_s3_path_style "${path_style}" + ) + export SNAPSHOT_S3_ACCESS_KEY="${access_key}" + export SNAPSHOT_S3_SECRET_KEY="${secret_key}" + export SNAPSHOT_S3_SECURITY_TOKEN="${security_token}" + ;; + *) + echo "AKERNEL_SNAPSHOT_STORAGE_BACKEND must be datasystem or s3" >&2 + return 1 + ;; + esac + standalone_snapshot_args=("${snapshot_args[@]}") + case "${standalone}" in + true) standalone_snapshot_args+=(--data_system_enable true) ;; + false) ;; + *) echo "AKS_LOCAL_MODE must be true or false" >&2; return 1 ;; + esac +} diff --git a/builder/systemd_services/yuanrong.service b/builder/systemd_services/yuanrong.service index 3960526..38070cd 100644 --- a/builder/systemd_services/yuanrong.service +++ b/builder/systemd_services/yuanrong.service @@ -4,7 +4,9 @@ Description=yuanrong.service [Service] #Type=simple PIDFile=/run/yuanrong.pid -PassEnvironment=ETCD_PORT ETCD_PEER_PORT ETCD_ADDRESS HOSTNAME AKS_LOCAL_MODE AKERNEL_NODE_IP INSTANCE_IP LITEBUS_DATA_KEY YR_LOG_PATH YR_INSTALLATION_DIR ENABLE_METRICS ENABLE_TRACE TRAEFIK_MODE TRAEFIK_ENABLE_TLS TRAEFIK_HTTP_ENTRYPOINT +PassEnvironment=ETCD_PORT ETCD_PEER_PORT ETCD_ADDRESS HOSTNAME NODE_NAME AKS_LOCAL_MODE AKERNEL_NODE_IP INSTANCE_IP LITEBUS_DATA_KEY YR_LOG_PATH YR_INSTALLATION_DIR ENABLE_METRICS ENABLE_TRACE TRAEFIK_MODE TRAEFIK_ENABLE_TLS TRAEFIK_HTTP_ENTRYPOINT YR_RRT_CONTROL_SOCKET_PATH +PassEnvironment=AKERNEL_SNAPSHOT_STORAGE_BACKEND +PassEnvironment=AKERNEL_SNAPSHOT_S3_PROVIDER AKERNEL_SNAPSHOT_S3_ENDPOINT AKERNEL_SNAPSHOT_S3_REGION AKERNEL_SNAPSHOT_S3_BUCKET AKERNEL_SNAPSHOT_S3_ACCESS_KEY AKERNEL_SNAPSHOT_S3_SECRET_KEY AKERNEL_SNAPSHOT_S3_SECURITY_TOKEN AKERNEL_SNAPSHOT_S3_USE_HTTPS AKERNEL_SNAPSHOT_S3_PATH_STYLE Environment="CONTAINER_EP=unix:///run/sandboxd/sandboxd.sock" Environment="RUNTIME_HOME_DIR=/home/yuanrong/runtime" Environment="YR_NOSET_CUDA_VISIBLE_DEVICES=1" diff --git a/deploy/README.md b/deploy/README.md index afaedf4..c14ef6a 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -207,6 +207,34 @@ image: Each component can still override `master.image`, `frontend.image`, or `node.image` when a split-image deployment is required. +### Snapshot object storage + +The core chart keeps the existing DataSystem snapshot backend by default. To +use S3-compatible object storage, create a Secret containing encrypted +credentials and configure the single S3 backend: + +```yaml +core: + snapshot: + storage: + backend: s3 + s3: + provider: generic # generic, obs, or oss + endpoint: minio.storage.svc:9000 + region: us-east-1 + bucket: akernel-snapshots + existingSecret: akernel-snapshot-s3 + useHttps: false + pathStyle: true +``` + +The Secret keys default to `access-key`, `secret-key`, and the optional +`security-token`. All providers share the same AWS Signature V4 client; +profiles do not select provider SDKs or restrict valid private endpoints and +CNAMEs. OSS requires virtual-hosted addressing. The removed `backend=obs` and +`snapshot_obs_*` configuration are not accepted. Remote snapshots larger than +5 GiB fail before upload because multipart CopyObject is not supported yet. + ### Public Traefik entrypoints For cloud deployments, use Traefik with two public entrypoints: diff --git a/deploy/akernel/charts/core/templates/node/daemonset.yaml b/deploy/akernel/charts/core/templates/node/daemonset.yaml index 36b8443..40f3bba 100644 --- a/deploy/akernel/charts/core/templates/node/daemonset.yaml +++ b/deploy/akernel/charts/core/templates/node/daemonset.yaml @@ -1,3 +1,20 @@ +{{- $snapshotBackend := .Values.snapshot.storage.backend -}} +{{- if not (has $snapshotBackend (list "datasystem" "s3")) -}} +{{- fail "snapshot.storage.backend must be datasystem or s3" -}} +{{- end -}} +{{- if eq $snapshotBackend "s3" -}} +{{- $snapshotProvider := .Values.snapshot.storage.s3.provider -}} +{{- if not (has $snapshotProvider (list "generic" "obs" "oss")) -}} +{{- fail "snapshot.storage.s3.provider must be generic, obs, or oss" -}} +{{- end -}} +{{- if and (eq $snapshotProvider "oss") .Values.snapshot.storage.s3.pathStyle -}} +{{- fail "OSS S3-compatible storage requires virtual-hosted addressing" -}} +{{- end -}} +{{- $snapshotEndpoint := required "snapshot.storage.s3.endpoint is required" .Values.snapshot.storage.s3.endpoint -}} +{{- $snapshotRegion := required "snapshot.storage.s3.region is required" .Values.snapshot.storage.s3.region -}} +{{- $snapshotBucket := required "snapshot.storage.s3.bucket is required" .Values.snapshot.storage.s3.bucket -}} +{{- $snapshotSecret := required "snapshot.storage.s3.existingSecret is required" .Values.snapshot.storage.s3.existingSecret -}} +{{- end -}} {{- if .Values.kruise.enabled }} apiVersion: apps.kruise.io/v1alpha1 {{- else }} @@ -80,6 +97,42 @@ spec: value: "node" - name: RUNSC_AKERNEL value: "1" + {{- if .Values.rrtControlSocketPath }} + - name: YR_RRT_CONTROL_SOCKET_PATH + value: {{ .Values.rrtControlSocketPath | quote }} + {{- end }} + - name: AKERNEL_SNAPSHOT_STORAGE_BACKEND + value: {{ .Values.snapshot.storage.backend | quote }} + {{- if eq .Values.snapshot.storage.backend "s3" }} + - name: AKERNEL_SNAPSHOT_S3_PROVIDER + value: {{ .Values.snapshot.storage.s3.provider | quote }} + - name: AKERNEL_SNAPSHOT_S3_ENDPOINT + value: {{ .Values.snapshot.storage.s3.endpoint | quote }} + - name: AKERNEL_SNAPSHOT_S3_REGION + value: {{ .Values.snapshot.storage.s3.region | quote }} + - name: AKERNEL_SNAPSHOT_S3_BUCKET + value: {{ .Values.snapshot.storage.s3.bucket | quote }} + - name: AKERNEL_SNAPSHOT_S3_USE_HTTPS + value: {{ .Values.snapshot.storage.s3.useHttps | quote }} + - name: AKERNEL_SNAPSHOT_S3_PATH_STYLE + value: {{ .Values.snapshot.storage.s3.pathStyle | quote }} + - name: AKERNEL_SNAPSHOT_S3_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.snapshot.storage.s3.existingSecret | quote }} + key: {{ .Values.snapshot.storage.s3.accessKeyKey | quote }} + - name: AKERNEL_SNAPSHOT_S3_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.snapshot.storage.s3.existingSecret | quote }} + key: {{ .Values.snapshot.storage.s3.secretKeyKey | quote }} + - name: AKERNEL_SNAPSHOT_S3_SECURITY_TOKEN + valueFrom: + secretKeyRef: + name: {{ .Values.snapshot.storage.s3.existingSecret | quote }} + key: {{ .Values.snapshot.storage.s3.securityTokenKey | quote }} + optional: true + {{- end }} - name: ETCD_ADDRESS value: {{ get $nodeEtcd "host" | default (printf "akernel-etcd.%s.svc.cluster.local" .Release.Namespace) | quote }} - name: ETCD_PORT diff --git a/deploy/akernel/charts/core/values.yaml b/deploy/akernel/charts/core/values.yaml index 6dfd158..6d81bab 100644 --- a/deploy/akernel/charts/core/values.yaml +++ b/deploy/akernel/charts/core/values.yaml @@ -23,6 +23,27 @@ image: tag: "latest" pullPolicy: IfNotPresent +# Directory exposed inside RRT sandboxes for the optional control socket. +# Empty disables the socket and the in-sandbox POST /checkpoint endpoint. +rrtControlSocketPath: "/run/openyuanrong" + +# Snapshot-backed sandbox lifecycle data plane. +snapshot: + storage: + backend: datasystem + s3: + # Supported provider profiles: generic, obs, oss. + provider: generic + endpoint: "" + region: us-east-1 + bucket: "" + existingSecret: "" + accessKeyKey: access-key + secretKeyKey: secret-key + securityTokenKey: security-token + useHttps: true + pathStyle: true + auth: # Existing Secret that contains the JWT signing seed. For # `helm template | kubectl apply`, prefer pre-creating this Secret with @@ -379,15 +400,6 @@ node: "noexec", "nodev" ] - }, - { - "destination": "/etc/resolv.conf", - "type": "bind", - "source": "/etc/resolv_akernel.conf", - "options": [ - "bind", - "ro" - ] } ], "linux": { diff --git a/deploy/akernel/tests/test-pause-resume-wiring.sh b/deploy/akernel/tests/test-pause-resume-wiring.sh new file mode 100755 index 0000000..2f3fee8 --- /dev/null +++ b/deploy/akernel/tests/test-pause-resume-wiring.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +# Copyright (c) 2026 Ant Group Corporation. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +chart="${repo_root}/deploy/akernel" +tmp_dir=$(mktemp -d) +trap 'rm -rf "${tmp_dir}"' EXIT + +default_render="${tmp_dir}/default.yaml" +s3_render="${tmp_dir}/s3.yaml" +helm template akernel-snapshot "${chart}" --set monitor.enabled=false >"${default_render}" +if grep -q 'name: AKERNEL_ENABLE_SNAPSHOT' "${default_render}"; then + echo "Helm still emits the removed snapshot enable switch" >&2 + exit 1 +fi +grep -A1 'name: AKERNEL_SNAPSHOT_STORAGE_BACKEND' "${default_render}" | grep -q 'value: "datasystem"' + +helm template akernel-snapshot "${chart}" \ + --set monitor.enabled=false \ + --set core.snapshot.storage.backend=s3 \ + --set core.snapshot.storage.s3.provider=generic \ + --set core.snapshot.storage.s3.endpoint=s3.private.example \ + --set core.snapshot.storage.s3.region=us-east-1 \ + --set core.snapshot.storage.s3.bucket=akernel-test \ + --set core.snapshot.storage.s3.existingSecret=akernel-snapshot-s3 \ + --set core.snapshot.storage.s3.pathStyle=true >"${s3_render}" +grep -A1 'name: AKERNEL_SNAPSHOT_S3_PROVIDER' "${s3_render}" | grep -q 'value: "generic"' +grep -A1 'name: AKERNEL_SNAPSHOT_S3_ENDPOINT' "${s3_render}" | grep -q 'value: "s3.private.example"' +grep -A5 'name: AKERNEL_SNAPSHOT_S3_ACCESS_KEY' "${s3_render}" | grep -q 'name: "akernel-snapshot-s3"' + +if helm template akernel-snapshot "${chart}" \ + --set monitor.enabled=false \ + --set core.snapshot.storage.backend=obs >/dev/null 2>&1; then + echo "Helm accepted the removed snapshot OBS backend" >&2 + exit 1 +fi + +if helm template akernel-snapshot "${chart}" \ + --set monitor.enabled=false \ + --set core.snapshot.storage.backend=s3 \ + --set core.snapshot.storage.s3.provider=unknown \ + --set core.snapshot.storage.s3.endpoint=s3.example \ + --set core.snapshot.storage.s3.bucket=akernel-test \ + --set core.snapshot.storage.s3.existingSecret=akernel-snapshot-s3 >/dev/null 2>&1; then + echo "Helm accepted an unknown S3 provider" >&2 + exit 1 +fi + +if helm template akernel-snapshot "${chart}" \ + --set monitor.enabled=false \ + --set core.snapshot.storage.backend=s3 \ + --set core.snapshot.storage.s3.provider=oss \ + --set core.snapshot.storage.s3.endpoint=oss-cname.example \ + --set core.snapshot.storage.s3.region=cn-hangzhou \ + --set core.snapshot.storage.s3.bucket=akernel-test \ + --set core.snapshot.storage.s3.existingSecret=akernel-snapshot-s3 \ + --set core.snapshot.storage.s3.pathStyle=true >/dev/null 2>&1; then + echo "Helm accepted path-style addressing for OSS" >&2 + exit 1 +fi + +echo "Kubernetes snapshot wiring contract passed" diff --git a/deploy/akernel/values.yaml b/deploy/akernel/values.yaml index 77bbe26..0ef06e6 100644 --- a/deploy/akernel/values.yaml +++ b/deploy/akernel/values.yaml @@ -1,5 +1,19 @@ core: createNamespace: false + snapshot: + storage: + backend: datasystem + s3: + provider: generic + endpoint: "" + region: us-east-1 + bucket: "" + existingSecret: "" + accessKeyKey: access-key + secretKeyKey: secret-key + securityTokenKey: security-token + useHttps: true + pathStyle: true monitor: enabled: true diff --git a/deploy/standalone/README.md b/deploy/standalone/README.md index 28f359a..9bfac5e 100644 --- a/deploy/standalone/README.md +++ b/deploy/standalone/README.md @@ -57,9 +57,46 @@ rather than tmpfs. Without `storage_mb`, runsc retains its configured memory-backed overlay while Firecracker uses its configured sparse ext4 default. +Sandbox checkpoints for runsc and Firecracker use the embedded YuanRong +DataSystem by default. Select the unified S3-compatible backend when snapshots +must live in object storage: + +```bash +AKERNEL_SNAPSHOT_STORAGE_BACKEND=s3 \ +AKERNEL_SNAPSHOT_S3_PROVIDER=generic \ +AKERNEL_SNAPSHOT_S3_ENDPOINT=minio.example.internal:9000 \ +AKERNEL_SNAPSHOT_S3_REGION=us-east-1 \ +AKERNEL_SNAPSHOT_S3_BUCKET=akernel-snapshots \ +AKERNEL_SNAPSHOT_S3_ACCESS_KEY='' \ +AKERNEL_SNAPSHOT_S3_SECRET_KEY='' \ +AKERNEL_SNAPSHOT_S3_USE_HTTPS=false \ +AKERNEL_SNAPSHOT_S3_PATH_STYLE=true \ +./start.sh +``` + +The provider is `generic`, `obs`, or `oss`; it selects validation and +addressing defaults while every provider uses the same AWS Signature V4 S3 +protocol client. Private endpoints and CNAMEs are allowed. OSS requires +virtual-hosted addressing (`PATH_STYLE=false`). The optional +`AKERNEL_SNAPSHOT_S3_SECURITY_TOKEN` carries an encrypted temporary token. +The removed OBS-native backend and `AKERNEL_SNAPSHOT_OBS_*` variables are not +accepted. + +Remote snapshots larger than 5 GiB are rejected before upload because this +version does not implement multipart CopyObject. `/home/akernel/checkpoints` +is the node's local staging directory; SDK checkpoint records have no automatic +TTL and remain until `Sandbox.delete_checkpoint()` is called. A restored +sandbox is a new sandbox and receives fresh network routes. + `start.sh` loads the host `tun` module and verifies `/dev/net/tun` before starting the pooled-TAP runtimes. Runc retains its separate veth network path. +The in-sandbox checkpoint endpoint uses +`YR_RRT_CONTROL_SOCKET_PATH=/run/openyuanrong` by default. Set +`YR_RRT_CONTROL_SOCKET_PATH=` explicitly when starting standalone to disable +the RRT control socket; an empty value is preserved rather than replaced by +the default. + ### Network backend Standalone uses the iptables NAT backend by default. Nodes without the diff --git a/deploy/standalone/config/config.json b/deploy/standalone/config/config.json index 5387596..25458c9 100644 --- a/deploy/standalone/config/config.json +++ b/deploy/standalone/config/config.json @@ -232,15 +232,6 @@ "noexec", "nodev" ] - }, - { - "destination": "/etc/resolv.conf", - "type": "bind", - "source": "/etc/resolv.conf", - "options": [ - "bind", - "ro" - ] } ], "linux": { diff --git a/deploy/standalone/start.sh b/deploy/standalone/start.sh index 25a17b8..81a9a6c 100755 --- a/deploy/standalone/start.sh +++ b/deploy/standalone/start.sh @@ -23,6 +23,7 @@ TOKEN_FILE="${DATA_DIR}/token" SANDBOXD_CONFIG_FILE="${DATA_DIR}/sandboxd/config.toml" AKERNEL_NAT_BACKEND="${AKERNEL_NAT_BACKEND:-iptables}" AKERNEL_ENABLE_RUNC="${AKERNEL_ENABLE_RUNC:-false}" +YR_RRT_CONTROL_SOCKET_PATH="${YR_RRT_CONTROL_SOCKET_PATH-/run/openyuanrong}" LITEBUS_DATA_KEY="" # Container runtime command (docker or pouch) @@ -330,12 +331,31 @@ start_node_container() { # FunctionMaster's HTTP provider publishes the per-sandbox routes required # by reverse tunnels; the legacy etcd mode cannot publish those routes. + local snapshot_backend="${AKERNEL_SNAPSHOT_STORAGE_BACKEND:-datasystem}" + local snapshot_docker_env=( + -e AKERNEL_SNAPSHOT_STORAGE_BACKEND="${snapshot_backend}" + ) + if [ "${snapshot_backend}" = "s3" ]; then + snapshot_docker_env+=( + -e AKERNEL_SNAPSHOT_S3_PROVIDER="${AKERNEL_SNAPSHOT_S3_PROVIDER:-}" + -e AKERNEL_SNAPSHOT_S3_ENDPOINT="${AKERNEL_SNAPSHOT_S3_ENDPOINT:-}" + -e AKERNEL_SNAPSHOT_S3_REGION="${AKERNEL_SNAPSHOT_S3_REGION:-}" + -e AKERNEL_SNAPSHOT_S3_BUCKET="${AKERNEL_SNAPSHOT_S3_BUCKET:-}" + -e AKERNEL_SNAPSHOT_S3_ACCESS_KEY + -e AKERNEL_SNAPSHOT_S3_SECRET_KEY + -e AKERNEL_SNAPSHOT_S3_SECURITY_TOKEN + -e AKERNEL_SNAPSHOT_S3_USE_HTTPS="${AKERNEL_SNAPSHOT_S3_USE_HTTPS:-true}" + -e AKERNEL_SNAPSHOT_S3_PATH_STYLE="${AKERNEL_SNAPSHOT_S3_PATH_STYLE:-true}" + ) + fi + "${DOCKER_PREFIX[@]}" ${DOCKER_CMD} run -d \ --name "${NODE_CONTAINER_NAME}" \ --privileged \ --net bridge \ --restart always \ -e AKS_LOCAL_MODE="true" \ + "${snapshot_docker_env[@]}" \ -e TRAEFIK_MODE="http" \ -e TRAEFIK_HTTP_ENTRYPOINT="web" \ -e TRAEFIK_ENABLE_TLS="false" \ @@ -347,6 +367,7 @@ start_node_container() { -e TZ=Asia/Shanghai \ -e ENABLE_TRACE="${ENABLE_TRACE:-false}" \ -e ENABLE_METRICS="${ENABLE_METRICS:-false}" \ + -e YR_RRT_CONTROL_SOCKET_PATH="${YR_RRT_CONTROL_SOCKET_PATH}" \ "${PROXY_RUN_ARGS[@]}" \ "${GPU_RUN_ARGS[@]}" \ --entrypoint=/usr/local/bin/akernel-entrypoint \ diff --git a/deploy/standalone/tests/test-pause-resume-wiring.sh b/deploy/standalone/tests/test-pause-resume-wiring.sh new file mode 100755 index 0000000..bf0d307 --- /dev/null +++ b/deploy/standalone/tests/test-pause-resume-wiring.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# Copyright (c) 2026 Ant Group Corporation. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +helper="${repo_root}/builder/scripts/yr_pause_resume_args.sh" +tmp_dir=$(mktemp -d) +trap 'rm -rf "${tmp_dir}"' EXIT +rrt_capability="${tmp_dir}/rrt-capable" +s3_capability="${tmp_dir}/s3-capable" +checkpoint_dir="${tmp_dir}/checkpoints" +touch "${rrt_capability}" "${s3_capability}" + +source "${helper}" +configure_snapshot_args "${rrt_capability}" "${checkpoint_dir}" true "${s3_capability}" +expected="--snapshot_storage_backend datasystem --checkpoint_dir ${checkpoint_dir} --data_system_enable true" +[[ "${standalone_snapshot_args[*]}" == "${expected}" ]] + +export AKERNEL_SNAPSHOT_STORAGE_BACKEND=s3 +export AKERNEL_SNAPSHOT_S3_PROVIDER=generic +export AKERNEL_SNAPSHOT_S3_ENDPOINT=s3.private.example:9000 +export AKERNEL_SNAPSHOT_S3_REGION=us-east-1 +export AKERNEL_SNAPSHOT_S3_BUCKET=akernel-test +export AKERNEL_SNAPSHOT_S3_ACCESS_KEY=encrypted-access +export AKERNEL_SNAPSHOT_S3_SECRET_KEY=encrypted-secret +export AKERNEL_SNAPSHOT_S3_SECURITY_TOKEN=encrypted-token +export AKERNEL_SNAPSHOT_S3_USE_HTTPS=false +export AKERNEL_SNAPSHOT_S3_PATH_STYLE=true +configure_snapshot_args "${rrt_capability}" "${checkpoint_dir}" false "${s3_capability}" +[[ " ${snapshot_args[*]} " == *" --snapshot_s3_provider generic "* ]] +[[ " ${snapshot_args[*]} " == *" --snapshot_s3_endpoint s3.private.example:9000 "* ]] +[[ " ${snapshot_args[*]} " != *" --snapshot_s3_access_key "* ]] +[[ " ${snapshot_args[*]} " != *" --snapshot_s3_secret_key "* ]] +[[ " ${snapshot_args[*]} " != *" --snapshot_s3_security_token "* ]] +[[ "${SNAPSHOT_S3_ACCESS_KEY}" == "encrypted-access" ]] +[[ "${SNAPSHOT_S3_SECRET_KEY}" == "encrypted-secret" ]] +[[ "${SNAPSHOT_S3_SECURITY_TOKEN}" == "encrypted-token" ]] + +export AKERNEL_SNAPSHOT_S3_PROVIDER=oss +export AKERNEL_SNAPSHOT_S3_PATH_STYLE=true +if configure_snapshot_args "${rrt_capability}" "${checkpoint_dir}" false "${s3_capability}" \ + >/dev/null 2>&1; then + echo "OSS snapshot storage accepted path-style addressing" >&2 + exit 1 +fi + +export AKERNEL_SNAPSHOT_STORAGE_BACKEND=obs +if configure_snapshot_args "${rrt_capability}" "${checkpoint_dir}" false "${s3_capability}" \ + >/dev/null 2>&1; then + echo "removed snapshot OBS backend was accepted" >&2 + exit 1 +fi + +echo "standalone snapshot wiring contract passed" diff --git a/sdk/python/README.md b/sdk/python/README.md index 700f93f..435fa20 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -22,6 +22,7 @@ It supports two backends: - [Filesystem](#filesystem) - [Interactive PTYs](#interactive-ptys) - [Port forwarding](#port-forwarding) + - [Checkpoint and restore](#checkpoint-and-restore) - [Reverse tunnels](#reverse-tunnels) - [Rootfs and mounts](#rootfs-and-mounts) - [Launch from a Dockerfile](#launch-from-a-dockerfile) @@ -99,6 +100,7 @@ Sandbox( detached: bool = False, node_id: str | None = None, *, + failover: bool = False, xpu: str | None = None, storage_mb: int | None = None, network_policy: NetworkPolicy | None = None, @@ -107,6 +109,23 @@ Sandbox( ) ``` +Set `failover=True` to opt into same-node recovery. If the sandbox has a valid +local checkpoint, recovery restores it. If the node explicitly reports that no +anonymous checkpoint exists, AKernel cold-starts the same logical sandbox on +the same physical node with its original creation parameters. Snapshot lookup, +metadata, validation, and restore errors do not silently fall back to a cold +start. The default is `False`; enabling the policy does not create a checkpoint +by itself. + +`sandbox.reload()` follows the same selection rule: it restores the latest +valid local checkpoint, or cold-starts on the original node when no checkpoint +exists, and returns only `True` or `False`. Existing `commands`, `files`, and +PTY factories remain owned by the original `Sandbox` object; no replacement +handle is returned. Completed `CommandResult` values remain readable. A cold +start does not preserve sandbox memory, writable-file changes, or the RRT +process table, so an incomplete command handle from the old runtime fails +instead of being associated with a PID in the new runtime. + ### Experimental GPU and writable storage Request a whole NVIDIA GPU by type, exact product model, and count: @@ -352,6 +371,67 @@ with Sandbox(port_forwardings=[8080]) as sandbox: deployment operator explicitly wants the direct Traefik address instead of the public gateway. +## Checkpoint and restore + +Create an immutable checkpoint of a running sandbox and restore it as a new, +independent sandbox: + +```python +from akernel_sdk import Sandbox + +checkpoint = None +try: + with Sandbox(runtime="runsc") as source: + source.commands.run("printf before > /tmp/state && sync") + checkpoint = source.checkpoint(timeout=180) + source.commands.run("printf after > /tmp/state") + + with Sandbox.restore(checkpoint) as restored: + assert restored.id != source.id + assert restored.commands.run("cat /tmp/state").stdout == "before" +finally: + if checkpoint is not None: + Sandbox.delete_checkpoint(checkpoint) +``` + +`checkpoint()` keeps the source running by default. Set +`leave_running=False` to terminate it after the checkpoint commits. Checkpoints +do not expire and must be removed explicitly with `delete_checkpoint()`; +`list_checkpoints()` returns all checkpoint identities visible to the current +tenant. + +Each restore gets a new sandbox ID, placement, network attachment, and routes. +The runtime, root filesystem, resources, mounts, environment, network policy, +and filesystem/process state come from the checkpoint. v1 does not support +in-place rollback or restore-time resource and configuration overrides. + +The bundled backend supports checkpoints for runsc and Firecracker. A restore +must use compatible runtime binaries, architecture, kernel, and runtime +configuration. The cluster prefers the source node when it is available and +may fall back to another compatible node through the configured snapshot +storage. + +When the deployment uses the S3-compatible snapshot backend, remote checkpoint +objects are limited to 5 GiB. Larger checkpoints fail before upload because +this version does not support multipart CopyObject; the independent DataSystem +backend is unaffected. This is an operator-selected storage capability rather +than an SDK-side size override. + +For a checkpoint created from a sandbox with a reverse tunnel, pass an +explicit `reverse_tunnel` to `restore()` using the same `reverse_port` and +`listen_port`. The target and connection timeout may change. A checkpoint made +without a tunnel rejects adding one during restore. The source tunnel is +briefly disconnected during checkpoint creation and then reconnected. + +Checkpoint/restore is available through the default `openyuanrong-sandbox` +backend. The legacy `openyuanrong-sdk` actor backend reports it as unsupported. +The current official backend package supports the default 180-second +checkpoint timeout. Custom checkpoint timeouts and checkpointing a sandbox +with an active reverse tunnel require a backend release containing the +corresponding YuanRong changes. +See [`examples/checkpoint_restore.py`](./examples/checkpoint_restore.py) for a +runnable example. + ## Reverse tunnels A reverse tunnel lets sandbox code call an HTTP or HTTPS service reachable @@ -519,6 +599,7 @@ Maintained examples are under [`examples/`](./examples): - `basic_usage.py` - `command_stdin.py` +- `checkpoint_restore.py` - `custom_image.py` - `dockerfile_launch.py` - `gpu_sandbox.py` @@ -556,6 +637,7 @@ not part of the default test suite. | `CommandInfo` | `pid`, `command`, `running` | | `EntryInfo` | `name`, `path`, `type`, `size`, `permissions`, `modified_time` | | `SandboxInfo` | `id`, `state`, `cpu`, `memory`, `image`, `xpu`, `storage_mb` | +| `CheckpointInfo` | `id` | | `NodeInfo` | `id`, `status`, `capacity`, `allocatable`, `labels` | | `S3Config` | `endpoint`, `bucket`, `object`, optional credentials | | `Mount` | `target`, one source, and `type` | diff --git a/sdk/python/akernel_sdk/__init__.py b/sdk/python/akernel_sdk/__init__.py index 5e77dd9..78fff0c 100644 --- a/sdk/python/akernel_sdk/__init__.py +++ b/sdk/python/akernel_sdk/__init__.py @@ -24,6 +24,7 @@ ) from ._backends.registry import selected_backend from .types import ( + CheckpointInfo, CommandInfo, CommandResult, EntryInfo, @@ -37,6 +38,7 @@ __all__ = [ "Sandbox", + "CheckpointInfo", "S3Config", "Mount", "NetworkPolicy", diff --git a/sdk/python/akernel_sdk/_backends/base.py b/sdk/python/akernel_sdk/_backends/base.py index e0ec3b7..c1a1f2b 100644 --- a/sdk/python/akernel_sdk/_backends/base.py +++ b/sdk/python/akernel_sdk/_backends/base.py @@ -41,6 +41,7 @@ class Capability(Enum): NODE_PLACEMENT = auto() CUSTOM_REVERSE_TUNNEL_PORTS = auto() REVERSE_WEBSOCKET = auto() + CHECKPOINT_RESTORE = auto() @dataclass(frozen=True) @@ -76,6 +77,7 @@ class SandboxSpec: xpu: str | None storage_mb: int | None network_policy: NetworkPolicy | None + failover: bool extra_config: Mapping[str, object] @@ -149,6 +151,10 @@ def is_running(self) -> bool: ... def get_info(self) -> SandboxInfo: ... + def reload(self) -> bool: ... + + def checkpoint(self, *, timeout: int) -> str: ... + def terminate(self) -> None: ... def close(self) -> None: ... @@ -163,6 +169,17 @@ class Backend(Protocol): def create(self, spec: SandboxSpec) -> BackendSession: ... + def restore( + self, + checkpoint_id: str, + *, + reverse_tunnel: HttpReverseTunnel | None, + ) -> BackendSession: ... + + def list_checkpoints(self) -> list[str]: ... + + def delete_checkpoint(self, checkpoint_id: str) -> None: ... + def delete_named(self, name: str) -> None: ... def close(self) -> None: ... diff --git a/sdk/python/akernel_sdk/_backends/openyuanrong_sandbox.py b/sdk/python/akernel_sdk/_backends/openyuanrong_sandbox.py index 56773b5..57a7571 100644 --- a/sdk/python/akernel_sdk/_backends/openyuanrong_sandbox.py +++ b/sdk/python/akernel_sdk/_backends/openyuanrong_sandbox.py @@ -16,13 +16,20 @@ from __future__ import annotations +import inspect import os from collections.abc import Mapping from typing import Any import yr_sandbox -from ..types import CommandInfo, CommandResult, EntryInfo, SandboxInfo +from ..types import ( + CommandInfo, + CommandResult, + EntryInfo, + HttpReverseTunnel, + SandboxInfo, +) from .base import ( Backend, BackendConfig, @@ -246,6 +253,48 @@ def get_info(self) -> SandboxInfo: storage_mb=self._spec.storage_mb, ) + def reload(self) -> bool: + if self._terminated or self._closed: + return False + try: + return bool(self._sandbox.reload()) + except Exception: + return False + + def checkpoint(self, *, timeout: int) -> str: + if self._terminated or self._closed: + raise BackendOperationError("checkpoint requires a running sandbox") + create_snapshot = self._sandbox.create_snapshot + try: + parameters = inspect.signature(create_snapshot).parameters.values() + supports_timeout = any( + parameter.name == "timeout" + or parameter.kind is inspect.Parameter.VAR_KEYWORD + for parameter in parameters + ) + except (TypeError, ValueError): + # Unknown callable signatures are treated as current backends. If + # they reject the keyword, the normal backend error includes the + # native failure instead of silently changing timeout semantics. + supports_timeout = True + if not supports_timeout and timeout != 180: + raise UnsupportedBackendFeatureError( + "The installed openyuanrong-sandbox backend only supports " + "the default 180-second checkpoint timeout. Upgrade the " + "backend to use a custom timeout." + ) + try: + if supports_timeout: + value = create_snapshot(timeout=timeout) + else: + value = create_snapshot() + except Exception as error: + raise _convert_error("checkpoint sandbox", error) from error + checkpoint_id = str(value.snapshot_id).strip() + if not checkpoint_id: + raise BackendOperationError("checkpoint returned an empty identity") + return checkpoint_id + def terminate(self) -> None: if self._terminated: return @@ -280,6 +329,7 @@ class OpenYuanRongSandboxBackend: { Capability.S3_ROOTFS, Capability.NODE_PLACEMENT, + Capability.CHECKPOINT_RESTORE, } ) @@ -373,6 +423,7 @@ def create(self, spec: SandboxSpec) -> BackendSession: xpu=spec.xpu, storage_mb=spec.storage_mb, network=network, + failover=spec.failover, extra_config=dict(spec.extra_config), create_timeout=create_timeout, ) @@ -380,6 +431,86 @@ def create(self, spec: SandboxSpec) -> BackendSession: raise _convert_error("create sandbox", error) from error return _Session(sandbox, spec) + def restore( + self, + checkpoint_id: str, + *, + reverse_tunnel: HttpReverseTunnel | None, + ) -> BackendSession: + if reverse_tunnel is not None and ( + reverse_tunnel.reverse_port != reverse_tunnel.listen_port - 1 + ): + raise UnsupportedBackendFeatureError( + "Backend 'openyuanrong-sandbox' requires reverse_port to equal " + "listen_port - 1." + ) + try: + sandbox = yr_sandbox.Sandbox.create( + checkpoint_id, + upstream=( + reverse_tunnel.target if reverse_tunnel is not None else None + ), + tunnel_connect_timeout=( + reverse_tunnel.connect_timeout + if reverse_tunnel is not None + else None + ), + proxy_port=( + reverse_tunnel.listen_port + if reverse_tunnel is not None + else _DEFAULT_LISTEN_PORT + ), + ) + except Exception as error: + raise _convert_error("restore checkpoint", error) from error + restored_spec = SandboxSpec( + image=None, + rootfs=None, + runtime="runsc", + cpu=1000, + memory=4096, + cpu_limit=0, + mem_limit=0, + idle_timeout=300, + schedule_timeout=30, + env={}, + name=None, + command_cwd=None, + port_forwardings=(), + mounts=(), + reverse_tunnel=reverse_tunnel, + detached=False, + node_id=None, + xpu=None, + storage_mb=None, + network_policy=None, + failover=False, + extra_config={}, + ) + return _Session(sandbox, restored_spec) + + def list_checkpoints(self) -> list[str]: + checkpoint_ids: list[str] = [] + page_token: str | None = None + try: + while True: + items, next_page_token = yr_sandbox.Sandbox.list_snapshots( + page_token=page_token, + page_size=100, + ) + checkpoint_ids.extend(str(item.snapshot_id) for item in items) + if not next_page_token: + return checkpoint_ids + page_token = next_page_token + except Exception as error: + raise _convert_error("list checkpoints", error) from error + + def delete_checkpoint(self, checkpoint_id: str) -> None: + try: + yr_sandbox.Sandbox.delete_snapshot(checkpoint_id) + except Exception as error: + raise _convert_error("delete checkpoint", error) from error + def delete_named(self, name: str) -> None: sandbox_id = f"{self.namespace}-{name}" try: diff --git a/sdk/python/akernel_sdk/_backends/openyuanrong_sdk.py b/sdk/python/akernel_sdk/_backends/openyuanrong_sdk.py index 51cc488..5b646fb 100644 --- a/sdk/python/akernel_sdk/_backends/openyuanrong_sdk.py +++ b/sdk/python/akernel_sdk/_backends/openyuanrong_sdk.py @@ -20,7 +20,13 @@ from collections.abc import Mapping from typing import Any -from ..types import CommandInfo, CommandResult, EntryInfo, SandboxInfo +from ..types import ( + CommandInfo, + CommandResult, + EntryInfo, + HttpReverseTunnel, + SandboxInfo, +) from . import openyuanrong_sdk_impl as _impl from .base import ( Backend, @@ -29,7 +35,7 @@ Capability, SandboxSpec, ) -from .errors import BackendOperationError +from .errors import BackendOperationError, UnsupportedBackendFeatureError from .openyuanrong_sdk_commands import ( CommandHandle as NativeCommandHandle, ) @@ -238,6 +244,21 @@ def get_info(self) -> SandboxInfo: storage_mb=self._spec.storage_mb, ) + def reload(self) -> bool: + if self._terminated or self._closed: + return False + try: + return bool(_impl.reload_instance(self._instance)) + except Exception: + return False + + def checkpoint(self, *, timeout: int) -> str: + del timeout + raise UnsupportedBackendFeatureError( + "Backend 'openyuanrong-sdk' does not support reusable checkpoints. " + "Use the default 'openyuanrong-sandbox' backend." + ) + def terminate(self) -> None: if self._terminated: return @@ -265,7 +286,11 @@ class OpenYuanRongSdkBackend: name = "openyuanrong-sdk" namespace = _NAMESPACE - capabilities = frozenset(Capability) + capabilities: frozenset[Capability] = frozenset( + capability + for capability in Capability + if capability is not Capability.CHECKPOINT_RESTORE + ) def __init__(self, _config: BackendConfig) -> None: _impl.ensure_initialized() @@ -291,6 +316,7 @@ def create(self, spec: SandboxSpec) -> BackendSession: xpu=spec.xpu, storage_mb=spec.storage_mb, network_policy=spec.network_policy, + failover=spec.failover, extra_config=spec.extra_config, ) try: @@ -333,6 +359,31 @@ def create(self, spec: SandboxSpec) -> BackendSession: _rollback_instance(instance, "session initialization") raise _convert_error("initialize sandbox session", error) from error + def restore( + self, + checkpoint_id: str, + *, + reverse_tunnel: HttpReverseTunnel | None, + ) -> BackendSession: + del checkpoint_id, reverse_tunnel + raise UnsupportedBackendFeatureError( + "Backend 'openyuanrong-sdk' does not support reusable checkpoints. " + "Use the default 'openyuanrong-sandbox' backend." + ) + + def list_checkpoints(self) -> list[str]: + raise UnsupportedBackendFeatureError( + "Backend 'openyuanrong-sdk' does not support reusable checkpoints. " + "Use the default 'openyuanrong-sandbox' backend." + ) + + def delete_checkpoint(self, checkpoint_id: str) -> None: + del checkpoint_id + raise UnsupportedBackendFeatureError( + "Backend 'openyuanrong-sdk' does not support reusable checkpoints. " + "Use the default 'openyuanrong-sandbox' backend." + ) + def delete_named(self, name: str) -> None: try: _impl.delete_named_instance(name) diff --git a/sdk/python/akernel_sdk/_backends/openyuanrong_sdk_impl.py b/sdk/python/akernel_sdk/_backends/openyuanrong_sdk_impl.py index a7cbd8a..6087d4d 100644 --- a/sdk/python/akernel_sdk/_backends/openyuanrong_sdk_impl.py +++ b/sdk/python/akernel_sdk/_backends/openyuanrong_sdk_impl.py @@ -156,6 +156,7 @@ def build_options( xpu: str | None, storage_mb: int | None, network_policy: NetworkPolicy | None, + failover: bool, extra_config: Mapping[str, object], ) -> Any: """Translate the stable SDK configuration to openYuanrong options.""" @@ -174,6 +175,7 @@ def build_options( validate_storage_mb(storage_mb) options = yr.InvokeOptions() + options.failover = failover # A Sandbox is driven by one sequential SDK client. Disabling ordered RPC # execution prevents a missing sequence number from stalling later calls. options.need_order = False @@ -264,6 +266,13 @@ def terminate_instance(handle: Any) -> None: handle.terminate() +def reload_instance(handle: Any) -> bool: + instance_id = global_runtime.get_runtime().get_real_instance_id(handle.instance_id) + if not global_runtime.get_runtime().reload_instance(instance_id): + return False + return ping_instance(handle) + + def delete_named_instance(name: str) -> None: ensure_initialized() handle = yr.get_instance(name, namespace=_NAMESPACE) diff --git a/sdk/python/akernel_sdk/sandbox.py b/sdk/python/akernel_sdk/sandbox.py index f48851a..f0a0d27 100644 --- a/sdk/python/akernel_sdk/sandbox.py +++ b/sdk/python/akernel_sdk/sandbox.py @@ -32,7 +32,14 @@ from .commands import CommandHandle, Commands from .filesystem import Filesystem from .pty import Pty -from .types import HttpReverseTunnel, Mount, NetworkPolicy, S3Config, SandboxInfo +from .types import ( + CheckpointInfo, + HttpReverseTunnel, + Mount, + NetworkPolicy, + S3Config, + SandboxInfo, +) _traefik_internal_ip_cache: str | None = None logger = logging.getLogger(__name__) @@ -135,6 +142,17 @@ def _validate_integer( raise ValueError(f"{name} must be greater than or equal to {minimum}") +def _checkpoint_id(checkpoint: CheckpointInfo | str) -> str: + if isinstance(checkpoint, CheckpointInfo): + return checkpoint.id + if not isinstance(checkpoint, str): + raise TypeError("checkpoint must be a CheckpointInfo or string") + value = checkpoint.strip() + if not value: + raise ValueError("checkpoint must be a non-empty string") + return value + + def _get_traefik_internal_ip(gateway: Endpoint) -> tuple[str, int]: """Resolve Traefik's direct address for ``internal=True`` URLs.""" @@ -186,6 +204,7 @@ def __init__( detached: bool = False, node_id: str | None = None, *, + failover: bool = False, xpu: str | None = None, storage_mb: int | None = None, network_policy: NetworkPolicy | None = None, @@ -213,6 +232,8 @@ def __init__( reverse_tunnel: SDK-side HTTP service exposed inside the sandbox. detached: Keep the sandbox alive when this client closes. node_id: Require placement on a specific AKernel node. + failover: Restore this sandbox on the same node from its latest + local anonymous checkpoint after a sandbox failure. xpu: Experimental whole-device accelerator request in ``type:model:count`` format. Currently only exact-model NVIDIA GPU requests are supported. The backend validates runtime @@ -291,6 +312,8 @@ def __init__( raise ValueError("cwd must be an absolute POSIX path") if not isinstance(detached, bool): raise TypeError("detached must be a boolean") + if not isinstance(failover, bool): + raise TypeError("failover must be a boolean") if node_id is not None: if not isinstance(node_id, str): raise TypeError("node_id must be a string") @@ -362,6 +385,7 @@ def __init__( if network_policy is None or network_policy.is_empty else network_policy ), + failover=failover, extra_config=normalized_extra_config, ) self._session = load_backend().create(spec) @@ -443,6 +467,120 @@ def reverse_tunnel(self) -> HttpReverseTunnel | None: return self._reverse_tunnel + def checkpoint( + self, + *, + timeout: int = 180, + leave_running: bool = True, + ) -> CheckpointInfo: + """Create a reusable checkpoint of this sandbox. + + The checkpoint has no TTL and remains available until explicitly + deleted with :meth:`delete_checkpoint`. A successful checkpoint is an + immutable template; each restore creates a new sandbox identity with + fresh placement and routes. + + Args: + timeout: Positive checkpoint timeout in seconds. + leave_running: Keep this source sandbox running after success. + + Returns: + The stable checkpoint identity. + """ + + _validate_integer("timeout", timeout, minimum=1) + if not isinstance(leave_running, bool): + raise TypeError("leave_running must be a boolean") + if self._closed or self._session is None or not self.is_running(): + raise RuntimeError("checkpoint requires a running sandbox") + checkpoint = CheckpointInfo(self._session.checkpoint(timeout=timeout)) + if not leave_running: + self.kill() + return checkpoint + + @classmethod + def restore( + cls, + checkpoint: CheckpointInfo | str, + *, + reverse_tunnel: HttpReverseTunnel | None = None, + ) -> Sandbox: + """Restore an independent sandbox from a reusable checkpoint. + + Runtime, root filesystem, resources, mounts, environment, network + policy, and exposed-port shape are inherited from the checkpoint. + Resource overrides and in-place rollback are intentionally not part of + v1. A source created with a reverse tunnel requires an explicit tunnel + with the same ports here; its target and connect timeout may differ. + """ + + checkpoint_id = _checkpoint_id(checkpoint) + if reverse_tunnel is not None and not isinstance( + reverse_tunnel, HttpReverseTunnel + ): + raise TypeError("reverse_tunnel must be an HttpReverseTunnel") + session = load_backend().restore( + checkpoint_id, + reverse_tunnel=reverse_tunnel, + ) + restored = cls.__new__(cls) + restored._session = session + restored._startup_command = None + restored._pty = None + restored._closed = False + restored._terminated = False + restored._reverse_tunnel = reverse_tunnel + restored._forwarded_ports = set() + restored._image = None + restored._cpu = 0 + restored._memory = 0 + restored._xpu = None + restored._storage_mb = None + restored._id = "" + try: + restored._id = session.id + restored._files = Filesystem(session.files) + restored._commands = Commands(session.commands) + restored._pty = Pty(restored._id) + info = session.get_info() + restored._image = info.image + restored._cpu = info.cpu if info.cpu is not None else 0 + restored._memory = info.memory if info.memory is not None else 0 + restored._xpu = info.xpu + restored._storage_mb = info.storage_mb + except Exception: + restored._closed = True + try: + session.terminate() + except Exception: + logger.warning( + "failed to roll back a partially initialized restore", + exc_info=True, + ) + try: + session.close() + except Exception: + logger.warning( + "failed to close a partially initialized restore session", + exc_info=True, + ) + raise + return restored + + @classmethod + def list_checkpoints(cls) -> list[CheckpointInfo]: + """List reusable checkpoints visible to the current tenant.""" + + del cls + return [CheckpointInfo(value) for value in load_backend().list_checkpoints()] + + @classmethod + def delete_checkpoint(cls, checkpoint: CheckpointInfo | str) -> None: + """Permanently delete one reusable checkpoint.""" + + del cls + load_backend().delete_checkpoint(_checkpoint_id(checkpoint)) + def get_port_url(self, port: int, *, internal: bool = False) -> str: """Return the gateway URL for a declared sandbox port. @@ -481,6 +619,13 @@ def is_running(self) -> bool: return False return self._session.is_running() + def reload(self) -> bool: + """Restore this sandbox from its latest local anonymous checkpoint.""" + + if self._closed or self._session is None: + return False + return bool(self._session.reload()) + def get_info(self) -> SandboxInfo: """Return current sandbox state and requested resources.""" diff --git a/sdk/python/akernel_sdk/types.py b/sdk/python/akernel_sdk/types.py index 67a43cd..db137da 100644 --- a/sdk/python/akernel_sdk/types.py +++ b/sdk/python/akernel_sdk/types.py @@ -160,6 +160,21 @@ class SandboxInfo: storage_mb: int | None = None +@dataclass(frozen=True) +class CheckpointInfo: + """Stable identity of a reusable sandbox checkpoint.""" + + id: str + + def __post_init__(self) -> None: + if not isinstance(self.id, str): + raise TypeError("id must be a string") + normalized = self.id.strip() + if not normalized: + raise ValueError("id must be a non-empty string") + object.__setattr__(self, "id", normalized) + + @dataclass(frozen=True) class NodeInfo: """Capacity, allocation, and labels advertised by an AKernel node.""" diff --git a/sdk/python/examples/checkpoint_restore.py b/sdk/python/examples/checkpoint_restore.py new file mode 100644 index 0000000..b9359b7 --- /dev/null +++ b/sdk/python/examples/checkpoint_restore.py @@ -0,0 +1,39 @@ +# Copyright (c) 2026 Ant Group Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Create, restore, and explicitly delete a reusable checkpoint.""" + +from akernel_sdk import CheckpointInfo, Sandbox + + +def main() -> None: + checkpoint: CheckpointInfo | None = None + try: + with Sandbox(runtime="runsc") as source: + source.commands.run("printf before > /tmp/checkpoint-state && sync") + checkpoint = source.checkpoint() + source.commands.run("printf after > /tmp/checkpoint-state") + print("source:", source.id, "checkpoint:", checkpoint.id) + + with Sandbox.restore(checkpoint) as restored: + value = restored.commands.run("cat /tmp/checkpoint-state") + print("restored:", restored.id, "state:", value.stdout) + assert value.stdout == "before" + finally: + if checkpoint is not None: + Sandbox.delete_checkpoint(checkpoint) + + +if __name__ == "__main__": + main() diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 166f6d6..f3619e7 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -28,7 +28,7 @@ classifiers = [ "Topic :: System :: Distributed Computing", ] dependencies = [ - "openyuanrong-sandbox==0.9.9", + "openyuanrong-sandbox==0.9.10", "websockets>=10.0", "dockerfile-parse>=2.0.1", ] diff --git a/sdk/python/tests/integration/test_sandbox.py b/sdk/python/tests/integration/test_sandbox.py index 08586e3..1303ac9 100644 --- a/sdk/python/tests/integration/test_sandbox.py +++ b/sdk/python/tests/integration/test_sandbox.py @@ -17,6 +17,7 @@ import unittest from akernel_sdk import Sandbox +from akernel_sdk._backends.errors import BackendOperationError _ENABLED = ( os.environ.get("AKERNEL_RUN_INTEGRATION") == "1" @@ -24,6 +25,9 @@ and bool(os.environ.get("AKERNEL_TOKEN")) ) _RUNTIME = os.environ.get("AKERNEL_TEST_RUNTIME", "runsc") +_RECOVERY_ENABLED = ( + _ENABLED and os.environ.get("AKERNEL_RUN_RECOVERY_INTEGRATION") == "1" +) @unittest.skipUnless( @@ -99,5 +103,90 @@ def test_pty_interrupts_foreground_process(self): self.assertIn(b"PTY_AFTER_INTERRUPT", output) +@unittest.skipUnless( + _ENABLED, + "set AKERNEL_RUN_INTEGRATION=1 and the AKernel SDK environment", +) +class SandboxCheckpointIntegrationTest(unittest.TestCase): + def test_checkpoint_restore_and_delete(self): + source = Sandbox(cpu=1000, memory=2048, runtime=_RUNTIME) + restored = None + checkpoint = None + try: + source_id = source.id + created = source.commands.run( + "printf checkpoint-before > /tmp/akernel-checkpoint-state && sync" + ) + self.assertEqual(created.exit_code, 0) + + checkpoint = source.checkpoint(timeout=180) + self.assertTrue(source.is_running()) + checkpoint_ids = {item.id for item in Sandbox.list_checkpoints()} + self.assertIn(checkpoint.id, checkpoint_ids) + + changed = source.commands.run( + "printf source-after > /tmp/akernel-checkpoint-state && sync" + ) + self.assertEqual(changed.exit_code, 0) + + restored = Sandbox.restore(checkpoint) + self.assertNotEqual(restored.id, source_id) + restored_value = restored.commands.run("cat /tmp/akernel-checkpoint-state") + self.assertEqual(restored_value.exit_code, 0) + self.assertEqual(restored_value.stdout, "checkpoint-before") + self.assertEqual( + source.commands.run("cat /tmp/akernel-checkpoint-state").stdout, + "source-after", + ) + finally: + if restored is not None: + restored.kill() + source.kill() + if checkpoint is not None: + Sandbox.delete_checkpoint(checkpoint) + + +@unittest.skipUnless( + _RECOVERY_ENABLED, + "set AKERNEL_RUN_RECOVERY_INTEGRATION=1 with the SDK environment", +) +class SandboxColdRecoveryIntegrationTest(unittest.TestCase): + def test_reload_without_snapshot_cold_starts_same_logical_sandbox(self): + sandbox = Sandbox( + cpu=1000, + memory=2048, + runtime=_RUNTIME, + failover=True, + ) + try: + logical_id = sandbox.id + facades = (sandbox.commands, sandbox.files, sandbox.pty) + completed = sandbox.commands.run("printf completed-before-cold-start") + sandbox.files.write("/tmp/cold-start-only", "old-runtime") + pending = sandbox.commands.run("sleep 60", background=True) + + self.assertIs(sandbox.reload(), True) + + self.assertEqual(sandbox.id, logical_id) + self.assertEqual((sandbox.commands, sandbox.files, sandbox.pty), facades) + self.assertEqual(completed.stdout, "completed-before-cold-start") + self.assertEqual( + sandbox.commands.run("test ! -e /tmp/cold-start-only").exit_code, + 0, + ) + self.assertEqual( + sandbox.commands.run("printf command-after-cold-start").stdout, + "command-after-cold-start", + ) + try: + old_result = pending.wait(timeout=10) + except BackendOperationError: + pass + else: + self.assertNotEqual(old_result.exit_code, 0) + finally: + sandbox.kill() + + if __name__ == "__main__": unittest.main() diff --git a/sdk/python/tests/unit/test_backends.py b/sdk/python/tests/unit/test_backends.py index c65924c..1dd8e3b 100644 --- a/sdk/python/tests/unit/test_backends.py +++ b/sdk/python/tests/unit/test_backends.py @@ -63,6 +63,7 @@ def _spec(**overrides): "xpu": None, "storage_mb": None, "network_policy": None, + "failover": False, "extra_config": MappingProxyType({}), } values.update(overrides) @@ -197,6 +198,59 @@ def test_extra_config_is_forwarded_to_native_sdk(self): {"featureFlag": True}, ) + def test_failover_is_forwarded_to_native_sandbox_sdk(self): + native = MagicMock() + native.id = "default-failover" + with patch.object( + openyuanrong_sandbox.yr_sandbox, + "Sandbox", + return_value=native, + ) as sandbox_type: + self.backend.create(_spec(failover=True)) + + self.assertIs(sandbox_type.call_args.kwargs["failover"], True) + + def test_reload_cold_start_success_keeps_native_session_and_facades(self): + native = MagicMock() + native.id = "default-reload" + native.reload.return_value = True + with patch.object( + openyuanrong_sandbox.yr_sandbox, + "Sandbox", + return_value=native, + ): + session = self.backend.create(_spec()) + + self.assertIs(session.reload(), True) + native.reload.assert_called_once_with() + + def test_incomplete_old_command_handle_fails_after_cold_start(self): + native = MagicMock() + native.id = "default-cold-start-handle" + native.reload.return_value = True + old_handle = MagicMock() + old_handle.pid = 321 + old_handle.wait.side_effect = RuntimeError( + "process handle is unavailable after cold start" + ) + native.commands.run.return_value = old_handle + with patch.object( + openyuanrong_sandbox.yr_sandbox, + "Sandbox", + return_value=native, + ): + session = self.backend.create(_spec()) + + pid = session.commands.start( + "sleep 60", envs=None, cwd=None, stdin=False + ) + self.assertIs(session.reload(), True) + + with self.assertRaisesRegex( + BackendOperationError, "unavailable after cold start" + ): + session.commands.wait(pid, 30) + def test_explicit_kata_image_is_forwarded_to_native_sdk(self): native = MagicMock() native.id = "default-kata-image" @@ -464,6 +518,118 @@ def test_named_delete_uses_deterministic_sid(self): self.backend.delete_named("worker") delete.assert_called_once_with("default-worker") + def test_checkpoint_delegates_to_reusable_snapshot_api(self): + native = MagicMock() + native.id = "default-source" + native.commands = MagicMock() + native.files = MagicMock() + native.create_snapshot.return_value = SimpleNamespace( + snapshot_id="checkpoint-1" + ) + with patch.object( + openyuanrong_sandbox.yr_sandbox, + "Sandbox", + return_value=native, + ): + session = self.backend.create(_spec()) + + self.assertEqual(session.checkpoint(timeout=240), "checkpoint-1") + + native.create_snapshot.assert_called_once_with(timeout=240) + + def test_checkpoint_uses_official_backend_default_timeout(self): + native = MagicMock() + native.id = "default-source" + native.commands = MagicMock() + native.files = MagicMock() + calls = [] + + def create_snapshot(): + calls.append(True) + return SimpleNamespace(snapshot_id="checkpoint-1") + + native.create_snapshot = create_snapshot + with patch.object( + openyuanrong_sandbox.yr_sandbox, + "Sandbox", + return_value=native, + ): + session = self.backend.create(_spec()) + + self.assertEqual(session.checkpoint(timeout=180), "checkpoint-1") + + self.assertEqual(calls, [True]) + + def test_checkpoint_rejects_custom_timeout_on_official_backend(self): + native = MagicMock() + native.id = "default-source" + native.commands = MagicMock() + native.files = MagicMock() + native.create_snapshot = lambda: SimpleNamespace( + snapshot_id="checkpoint-1" + ) + with patch.object( + openyuanrong_sandbox.yr_sandbox, + "Sandbox", + return_value=native, + ): + session = self.backend.create(_spec()) + + with self.assertRaisesRegex( + UnsupportedBackendFeatureError, + "default 180-second", + ): + session.checkpoint(timeout=240) + + def test_restore_uses_snapshot_template_and_explicit_tunnel(self): + native = MagicMock() + native.id = "default-restored" + native.commands = MagicMock() + native.files = MagicMock() + tunnel = HttpReverseTunnel( + "https://new-target.example", + reverse_port=9000, + listen_port=9001, + connect_timeout=12, + ) + with patch.object( + openyuanrong_sandbox.yr_sandbox, + "Sandbox", + ) as sandbox_type: + sandbox_type.create.return_value = native + session = self.backend.restore( + "checkpoint-1", + reverse_tunnel=tunnel, + ) + + self.assertEqual(session.id, "default-restored") + sandbox_type.create.assert_called_once_with( + "checkpoint-1", + upstream="https://new-target.example", + tunnel_connect_timeout=12, + proxy_port=9001, + ) + + def test_checkpoint_catalog_pages_and_deletes(self): + first = ([SimpleNamespace(snapshot_id="checkpoint-1")], "next") + second = ([SimpleNamespace(snapshot_id="checkpoint-2")], "") + with patch.object( + openyuanrong_sandbox.yr_sandbox.Sandbox, + "list_snapshots", + side_effect=[first, second], + ) as list_snapshots, patch.object( + openyuanrong_sandbox.yr_sandbox.Sandbox, + "delete_snapshot", + ) as delete_snapshot: + self.assertEqual( + self.backend.list_checkpoints(), + ["checkpoint-1", "checkpoint-2"], + ) + self.backend.delete_checkpoint("checkpoint-1") + + self.assertEqual(list_snapshots.call_count, 2) + delete_snapshot.assert_called_once_with("checkpoint-1") + class OpenYuanRongSdkBackendTest(unittest.TestCase): def setUp(self): @@ -477,6 +643,59 @@ def setUp(self): self.addCleanup(initialized.stop) self.backend = openyuanrong_sdk.OpenYuanRongSdkBackend(self.config) + def test_failover_is_forwarded_to_native_invoke_options(self): + instance = MagicMock() + with ( + patch.object( + openyuanrong_sdk._impl, + "build_options", + return_value=MagicMock(), + ) as build_options, + patch.object( + openyuanrong_sdk._impl, + "create_instance", + return_value=instance, + ), + patch.object( + openyuanrong_sdk._impl, + "real_instance_id", + return_value="physical-id", + ), + ): + self.backend.create(_spec(failover=True)) + + self.assertIs(build_options.call_args.kwargs["failover"], True) + + def test_reload_forwards_to_actor_runtime(self): + instance = MagicMock() + with ( + patch.object( + openyuanrong_sdk._impl, + "build_options", + return_value=MagicMock(), + ), + patch.object( + openyuanrong_sdk._impl, + "create_instance", + return_value=instance, + ), + patch.object( + openyuanrong_sdk._impl, + "real_instance_id", + return_value="physical-id", + ), + patch.object( + openyuanrong_sdk._impl, + "reload_instance", + return_value=True, + ) as reload_instance, + ): + session = self.backend.create(_spec()) + result = session.reload() + + self.assertIs(result, True) + reload_instance.assert_called_once_with(instance) + def test_physical_id_failure_rolls_back_created_actor(self): instance = MagicMock() physical_id_error = RuntimeError("physical ID unavailable") @@ -598,6 +817,14 @@ def test_close_finalizes_actor_sdk(self): finalize.assert_called_once_with() + def test_reusable_checkpoint_operations_are_explicitly_unsupported(self): + with self.assertRaises(UnsupportedBackendFeatureError): + self.backend.restore("checkpoint-1", reverse_tunnel=None) + with self.assertRaises(UnsupportedBackendFeatureError): + self.backend.list_checkpoints() + with self.assertRaises(UnsupportedBackendFeatureError): + self.backend.delete_checkpoint("checkpoint-1") + if __name__ == "__main__": unittest.main() diff --git a/sdk/python/tests/unit/test_openyuanrong_sdk_impl.py b/sdk/python/tests/unit/test_openyuanrong_sdk_impl.py index a766a68..5cc3127 100644 --- a/sdk/python/tests/unit/test_openyuanrong_sdk_impl.py +++ b/sdk/python/tests/unit/test_openyuanrong_sdk_impl.py @@ -42,6 +42,7 @@ def build_options(self, **overrides): "xpu": None, "storage_mb": None, "network_policy": None, + "failover": False, "extra_config": {}, } values.update(overrides) @@ -150,6 +151,11 @@ def test_extra_config_uses_custom_extension_wire_format(self): {"featureFlag": True, "labels": ["one", "two"]}, ) + def test_failover_uses_typed_native_option(self): + options = self.build_options(failover=True) + + self.assertIs(options.failover, True) + def test_node_info_conversion(self): node = _impl._to_node_info( { diff --git a/sdk/python/tests/unit/test_sandbox.py b/sdk/python/tests/unit/test_sandbox.py index 14a1386..50f6f9f 100644 --- a/sdk/python/tests/unit/test_sandbox.py +++ b/sdk/python/tests/unit/test_sandbox.py @@ -18,6 +18,7 @@ from unittest.mock import MagicMock, patch from akernel_sdk import ( + CheckpointInfo, DockerfileLaunch, HttpReverseTunnel, NetworkPolicy, @@ -73,11 +74,43 @@ def test_default_constructor_and_info(self): self.assertIsNone(spec.xpu) self.assertIsNone(spec.storage_mb) self.assertIsNone(spec.network_policy) + self.assertFalse(spec.failover) self.assertEqual(dict(spec.extra_config), {}) sandbox.kill() self.session.terminate.assert_called_once_with() self.session.close.assert_called_once_with() + def test_failover_is_typed_and_forwarded(self): + sandbox = Sandbox(failover=True) + spec = self.backend.create.call_args.args[0] + + self.assertTrue(spec.failover) + sandbox.kill() + + def test_failover_rejects_non_boolean_values(self): + with self.assertRaisesRegex(TypeError, "failover"): + Sandbox(failover=1) + self.backend.create.assert_not_called() + + def test_reload_cold_start_success_returns_true_without_replacing_facades(self): + self.session.reload.return_value = True + sandbox = Sandbox() + before = (sandbox.commands, sandbox.files, sandbox.pty, sandbox._session) + + self.assertIs(sandbox.reload(), True) + self.assertEqual( + (sandbox.commands, sandbox.files, sandbox.pty, sandbox._session), + before, + ) + self.session.reload.assert_called_once_with() + + def test_reload_returns_false_after_close(self): + sandbox = Sandbox() + sandbox.kill() + + self.assertIs(sandbox.reload(), False) + self.session.reload.assert_not_called() + def test_extra_config_is_validated_and_defensively_copied(self): labels = ["worker"] requested = {"featureFlag": True, "nested": {"labels": labels}} @@ -169,6 +202,89 @@ def test_named_delete_hides_backend_namespace(self): Sandbox.delete("worker") self.backend.delete_named.assert_called_once_with("worker") + def test_checkpoint_returns_public_identity_and_keeps_source_running(self): + self.session.checkpoint.return_value = "checkpoint-1" + sandbox = Sandbox() + + checkpoint = sandbox.checkpoint(timeout=240) + + self.assertEqual(checkpoint, CheckpointInfo("checkpoint-1")) + self.session.checkpoint.assert_called_once_with(timeout=240) + self.session.terminate.assert_not_called() + sandbox.kill() + + def test_checkpoint_can_terminate_source_after_success(self): + self.session.checkpoint.return_value = "checkpoint-1" + sandbox = Sandbox() + + checkpoint = sandbox.checkpoint(leave_running=False) + + self.assertEqual(checkpoint.id, "checkpoint-1") + self.session.terminate.assert_called_once_with() + self.session.close.assert_called_once_with() + + def test_checkpoint_validates_arguments_and_running_state(self): + sandbox = Sandbox() + for timeout in (True, 0, -1, 1.5): + with self.subTest(timeout=timeout), self.assertRaises( + (TypeError, ValueError) + ): + sandbox.checkpoint(timeout=timeout) + with self.assertRaisesRegex(TypeError, "leave_running"): + sandbox.checkpoint(leave_running=1) + self.session.is_running.return_value = False + with self.assertRaisesRegex(RuntimeError, "running sandbox"): + sandbox.checkpoint() + sandbox.kill() + + def test_restore_builds_facades_around_new_backend_session(self): + restored_session = MagicMock() + restored_session.id = "restored-physical-id" + restored_session.commands = MagicMock() + restored_session.files = MagicMock() + restored_session.get_info.return_value = SandboxInfo( + id="restored-physical-id", + state="running", + cpu=2000, + memory=8192, + image="base-image", + ) + self.backend.restore.return_value = restored_session + tunnel = HttpReverseTunnel("http://127.0.0.1:9000") + + restored = Sandbox.restore( + CheckpointInfo("checkpoint-1"), reverse_tunnel=tunnel + ) + + self.backend.restore.assert_called_once_with( + "checkpoint-1", reverse_tunnel=tunnel + ) + self.assertEqual(restored.id, "restored-physical-id") + self.assertIs(restored.reverse_tunnel, tunnel) + self.assertEqual(restored.get_info().cpu, 2000) + restored.kill() + restored_session.terminate.assert_called_once_with() + restored_session.close.assert_called_once_with() + + def test_list_and_delete_checkpoints_hide_backend_details(self): + self.backend.list_checkpoints.return_value = ["checkpoint-1", "checkpoint-2"] + + self.assertEqual( + Sandbox.list_checkpoints(), + [CheckpointInfo("checkpoint-1"), CheckpointInfo("checkpoint-2")], + ) + Sandbox.delete_checkpoint(CheckpointInfo("checkpoint-1")) + Sandbox.delete_checkpoint(" checkpoint-2 ") + + self.assertEqual( + self.backend.delete_checkpoint.call_args_list, + [unittest.mock.call("checkpoint-1"), unittest.mock.call("checkpoint-2")], + ) + with self.assertRaises(ValueError): + Sandbox.delete_checkpoint(" ") + with self.assertRaises(TypeError): + Sandbox.restore(object()) # type: ignore[arg-type] + def test_rootfs_requires_s3_config(self): with self.assertRaisesRegex(TypeError, "S3Config"): Sandbox(rootfs={"type": "s3"}) diff --git a/sdk/python/tests/unit/test_types.py b/sdk/python/tests/unit/test_types.py index e9766c4..2a8f4a8 100644 --- a/sdk/python/tests/unit/test_types.py +++ b/sdk/python/tests/unit/test_types.py @@ -19,7 +19,13 @@ from pathlib import Path import akernel_sdk -from akernel_sdk import DockerContextEntry, HttpReverseTunnel, Mount, S3Config +from akernel_sdk import ( + CheckpointInfo, + DockerContextEntry, + HttpReverseTunnel, + Mount, + S3Config, +) class PublicTypesTest(unittest.TestCase): @@ -69,6 +75,7 @@ def test_public_exports_are_minimal(self): set(akernel_sdk.__all__), { "Sandbox", + "CheckpointInfo", "S3Config", "Mount", "NetworkPolicy", @@ -120,6 +127,14 @@ def test_docker_context_entry_is_public_and_immutable(self): with self.assertRaisesRegex(AttributeError, "cannot assign"): entry.mode = 0o700 # type: ignore[misc] + def test_checkpoint_info_is_public_normalized_and_immutable(self): + checkpoint = CheckpointInfo(" checkpoint-1 ") + self.assertEqual(checkpoint.id, "checkpoint-1") + with self.assertRaisesRegex(AttributeError, "cannot assign"): + checkpoint.id = "changed" # type: ignore[misc] + with self.assertRaises(ValueError): + CheckpointInfo(" ") + def test_s3_config_serialization(self): config = S3Config( endpoint="https://s3.example.com", diff --git a/src/sandboxd b/src/sandboxd index 1918fad..446b3ab 160000 --- a/src/sandboxd +++ b/src/sandboxd @@ -1 +1 @@ -Subproject commit 1918fadb03b59bc6f540196e14b90a91bdf31b7d +Subproject commit 446b3aba74332e84c6260c5551994157e1d9a518 diff --git a/src/yuanrong b/src/yuanrong index 177592c..abb0051 160000 --- a/src/yuanrong +++ b/src/yuanrong @@ -1 +1 @@ -Subproject commit 177592c7eb208698bd9b048a818b9cfb02060545 +Subproject commit abb0051c79c7ce1e032b53f1261f52b9102a2b78