diff --git a/.dockerignore b/.dockerignore index 9550d54..58e47ae 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,6 @@ # Git metadata is not part of the image build. Component versions come from # their source manifests, and exact revisions remain traceable through the -# parent repository's submodule gitlinks. +# parent repository's submodule gitlinks or checksum-pinned source archives. .git **/.git diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5de3225..e9687fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -142,7 +142,7 @@ jobs: run: | gateway_ip="$(docker inspect \ --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' \ - akernel-traefik)" + akernel-node)" test -n "${gateway_ip}" token="$(cat deploy/standalone/data/token)" export AKERNEL_TOKEN="${token}" @@ -170,7 +170,7 @@ jobs: run: | gateway_ip="$(docker inspect \ --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' \ - akernel-traefik)" + akernel-node)" test -n "${gateway_ip}" token="$(cat deploy/standalone/data/token)" export AKERNEL_TOKEN="${token}" @@ -186,7 +186,7 @@ jobs: if: failure() run: | docker ps -a - for container in akernel-node akernel-traefik; do + for container in akernel-node; do if docker container inspect "${container}" >/dev/null 2>&1; then echo "=== ${container}: docker logs ===" docker logs --tail 500 "${container}" 2>&1 || true @@ -204,5 +204,5 @@ jobs: if [[ -x ./deploy/standalone/stop.sh ]]; then ./deploy/standalone/stop.sh else - docker rm -f akernel-traefik akernel-node >/dev/null 2>&1 || true + docker rm -f akernel-node >/dev/null 2>&1 || true fi diff --git a/AGENTS.md b/AGENTS.md index fca1d2a..69b972d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -123,6 +123,14 @@ runtimes and `openyuanrong_sdk`. `builder/node.Dockerfile` then compiles the node components and produces the AKernel all-in-one image using the selected runtime image and its matching service configuration. +The image also builds Edge, Node Proxy and the forwarding helper from a +checksum-pinned YuanRong source archive with its Cargo lockfile; their binaries +are installed under `data_plane/bin` for the Go CLI. The default Core wheel +and data-plane source are pinned to the matching YuanRong `2b54c26885c6` +build so the Core package includes Node Proxy address +registration and Edge process scripts. The RRT binary and sandbox SDK retain +their matching `0.10.2rc2` command protocol. + The control-plane and RRT release version is independent of the optional actor-based `openyuanrong_sdk` installed in the Python runtime profile. This actor backend is deprecated and retained only for compatibility with existing @@ -175,7 +183,10 @@ source revisions are traceable through the AKernel commit's submodule gitlinks. Use [`deploy/README.md`](./deploy/README.md) as the deployment entry point. AKernel supports standalone, existing Kubernetes clusters via Helm, and -Terraform-based cloud provisioning. +Terraform-based cloud provisioning. The core chart enables Edge and Node Proxy +by default with an image containing the data-plane binaries and Go CLI support; +configure its TLS Secret and allowed CIDRs. +See `deploy/README.md` for ingress migration and SDK endpoint configuration. Aliyun's aggregate Pod PID budget is configurable independently of the per-sandbox limit; see `deploy/terraform/aliyun/README.md#pod-pid-budget`. @@ -380,13 +391,13 @@ export AKERNEL_SERVER_ADDRESS="" export AKERNEL_TOKEN="" ``` -When the public Traefik dual-entrypoint mode is enabled, a host/IP-only +With the default Edge ingress, a host/IP-only `AKERNEL_SERVER_ADDRESS` uses HTTPS/WSS on 443 for the frontend API and exec websocket, and HTTP on 80 for sandbox port URLs. For standalone deployments, -use the Traefik container IP printed by `deploy/standalone/start.sh`: +use the AKernel container IP printed by `deploy/standalone/start.sh`: ```bash -export AKERNEL_SERVER_ADDRESS= +export AKERNEL_SERVER_ADDRESS= ``` No separate `AKERNEL_GATEWAY_ADDRESS` is required for the default standalone diff --git a/assets/akernel-architecture.svg b/assets/akernel-architecture.svg index dd2d904..400bb2e 100644 --- a/assets/akernel-architecture.svg +++ b/assets/akernel-architecture.svg @@ -1,3 +1,3 @@ SDK/CLItraefikfrontend & schedulerlocal gatewaylocal gatewaysandboxdsandboxdsandboxsandboxgVisorsandboxsandboxgVisorOTel CollectorOTel CollectorGrafanaprometheuslokitempoOCI RegistryObject StoragePrivate K8s / Cloud VendorBrowser \ No newline at end of file + @font-face { font-family: "Lilita One"; src: url(data:font/woff2;base64,d09GMgABAAAAAAa4AA4AAAAACxQAAAZiAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGhYbgQAcGAZgAIEUEQgKi0CJQgsqAAE2AiQDUAQgBYMkByAb+QijopRuRsn+eRgbsx7b2ZmN2i1VDxbGuVyjXbC0oyZh2i53OblHQ/yu//k8eFk/yVDVqCnlLYS3NwurCwCHFl6+A6GJ0W3NYX7jvB0Qy2kgiyTRTn5uLtFtzw8IFfG3HQi5/A8H+PtL09RKmVgfT4QiHpFW/qbSvk1p1Bo1b2AJUAQzoUK1WjmwowZy40cB06Nb1CsHKc2Ec9UgAAgQVJIgQEDurgfwoGWPqCtuawAHDAD4JYja8pY+hlYSDJAamYMkqNkphLKnRIACEyywwQDn/7u4MZlHFgETbA6c/5/3yG/w5yTMdDR/HyX69ClYAiSx/CZiAuSeRgPG5h8k7nL1C0gEugTWSv6paQhG63izZwFQc9yBpcSvDmRoJOCTqYwCBYKLGdC9ZjUJCgLQnHgLEydZpmLlqtVp1DEA1DJRg5bfGZZddtpBu2yz0XLLLLbQAvMiEGgRIP6XGBwFlrVmJcRFIIMQrfFYoECEiNQppL1tZe9CJuFys4xlHDnNUXH1uJw4zOYwMs0NNaJeTxJPUx2TMDuVk87SXEqe0VkOqDsYFIpvk3f6I7pLuVbUdZ81aA05x3Re//s6i+weIVvja7lLniGXPvRvE8yFuttg0y0RTQJklqcl1ST05OgKLloWsKi/qovXyeOgtSj70jf0MaC7Mgw95pqucJhCh+jNmzMxLH9qg72mn7eJip9aCfDHbG69R72bdsPjQJHWipaQO82qJCseTxB+IoMvmY5J2UB+TNK/MDrYanRhx2jQrZVy1T/+PPo5RaDjoTm18XYv0Zyhj5BXAPwx3YZVJAkQOVkZ2gEMTon55fRI5CxXUfq2vKReHuMDqZpGPkkIt2sylv+M0LgavqXIIT5g/VOKHGM3+KgvgqyrWUVtAXECuJKwy67SrXw4Lb38toii/dmEovnHxQRg5Br6fuyvWKQuUB6pGUQt6NQbyHUJF+b8j/S7Ajl5M7Qk9ExlL+lrStmxmm/B/Z4CucbNLkf7LGWZBd8cRpdKxva+Ht5xcEBsmG//oNpTXWWY+Xece6G3k8bUndR4hps61j7Qfyt3kziEW+lh5pEG60j3Chnz1Bupb2TKyugUb4Oxu6UCgSpaYdXpkDp05c60dQeH4KdQXNgi5y2QnpCi+//61mEVJLHW56Q+6Xj9mqW8+buk+aYEI4k8xUhT4w1TB3e3DQhIet7plLxZEMuJFc8Xmv1OJmzBW8fM5Kgb4zO2J2/P79BksZkbpgWVpXNG8wJQW7MxiDPyeEp+3nxTqVWkaZjWU/C66WZxTGll1JLDXjylyNiSe1y/w+JdgtoZNlvFSapIFktsZtSndrjvZ29iM25sWX/7qO+9wwtUW+lZ3uVuf6wkq7joVz45NmL4+8GTTtSVxyzeFaFl0ZHvMbDHb4KBrY28X9/Y0wSPmZzt7Nfs7Sy/mDWRd088YnMrksBeKS5OJ9n4y2RHl5KccKWyVy003qeK3zzp6bqEglp7wWqBiasZzzkMRv996Rt1pXfUF/HHuOIyYEfzDrDBzsslWPEqiaWmLthhm1l0pLOF714rY1mSG0MktPRHyOZkZyetntirz7UttjZhRpWxUt82qmW5q0ouCO4KK/4akFxdUhvV6mSrXgsZU5BeSnFL3BQRslK+R7ZipSSYnyEdK42C1Vv+Gr65K214JYS3ihetNA0xuehnGvYNRyOL8wY0gfnB6iAzKs0j0XpAsdXYF9UPBsIiu6kJgP9nFEDJ4Lqf1ZIH8EGQo9tHuW+Zksc47K8nHP55DGnK9+tqyQGwlkmJX1j+qPwcAgCBIeUBH9cNi/2/cvjUCwBOO8SfIlPHM6d2vjkADkgAjYOaMxIHqHcA6XwPBILZFY4D2AOQAQDpC1Uu0FYH13qcY9PwSSZEpAeU98yXSeFwkteeFRQRUN5YijvE6Jogj4Z1XLBAAFBIwAIAEAhGgI1cFAgG91D3eK4moNCjJiFCnZpCMPtWBqxmQknmZDlvBcSrVqdam2JJGpTLVK5Fq2qNGtBcOHLmzFWsdsUaJIxCm0YMKu8NNvJFb4PRunks5cqZS/j9qcpValenWAvy58M0atKtRbVKVdrQrJWy2cBLX6ErrZvR7hoQBY856zXTcVCqioKjEjfVw4a9jjoPJ1eZLoZan93coVwZWuTGQEvUAvWg/jTTSydNUQj0/b+VKwAAAA==); }SDK/CLIedgefrontend & schedulerlocal gatewaylocal gatewaysandboxdsandboxdsandboxsandboxgVisorsandboxsandboxgVisorOTel CollectorOTel CollectorGrafanaprometheuslokitempoOCI RegistryObject StoragePrivate K8s / Cloud VendorBrowser \ No newline at end of file diff --git a/builder/node.Dockerfile b/builder/node.Dockerfile index 19669be..be84f62 100644 --- a/builder/node.Dockerfile +++ b/builder/node.Dockerfile @@ -10,12 +10,17 @@ ARG AKERNEL_ENABLE_RUNC=false ARG AKERNEL_ENABLE_FIRECRACKER=true ARG SANDBOXD_BUILD_IMAGE=golang:1.25.5-bookworm ARG DISTILL_FS_BUILD_IMAGE=rust:1.85.0-bookworm -ARG OPEN_YR_VERSION=0.10.2rc2 +ARG DATA_PLANE_BUILD_IMAGE=rust:1.95.0-bookworm +ARG DATA_PLANE_SOURCE_URL=https://codeload.github.com/openYuanrong-mirror/yuanrong/tar.gz/2b54c26885c67535e7b5b80608812f0d5ed33244 +ARG DATA_PLANE_SOURCE_SHA256=c57b633b3fe50802ec9fcd70cb1da24ac80bea54984ed8d5d2cff15e7781dcfb +ARG OPEN_YR_VERSION=0.7.0+2b54c26885c6 ARG OPEN_YR_CORE_WHEEL_URL= ARG OPEN_YR_CORE_WHEEL_SHA256= -ARG OPEN_YR_RELEASE_BASE_URL=https://openyuanrong.obs.cn-southwest-2.myhuaweicloud.com/release -ARG OPEN_YR_CORE_AMD64_SHA256=8cdefba9a415a7a35b6f39bf847e7fb933ad6ca55b27b7dec2f377ece47198c4 -ARG OPEN_YR_CORE_ARM64_SHA256=2e9d2d18922b87721fcc3e92fa959cdd2ce026a51b18512c7c71ab24ac6a6eaa +# Core includes the matching process scripts and FunctionProxy registration. +ARG OPEN_YR_CORE_AMD64_URL=https://openyuanrong.obs.cn-southwest-2.myhuaweicloud.com/daily/20260909023812/linux/amd64/openyuanrong_core-0.7.0%2B2b54c26885c6-py3-none-manylinux_2_31_x86_64.whl +ARG OPEN_YR_CORE_ARM64_URL=https://openyuanrong.obs.cn-southwest-2.myhuaweicloud.com/daily/20260909024016/linux/arm64/openyuanrong_core-0.7.0%2B2b54c26885c6-py3-none-manylinux_2_31_aarch64.whl +ARG OPEN_YR_CORE_AMD64_SHA256=b459e916e75d5ef6c59988ad7b4108fd15a19c8fd102f2ca7ab98b5e9fc7a09c +ARG OPEN_YR_CORE_ARM64_SHA256=0678dc164e371471665fa2468967c456055c5cbd2f24bb130384189f75fc7a2a ARG GVISOR_DOWNLOAD_IMAGE=ubuntu:24.04 ARG GVISOR_RELEASE ARG GVISOR_AMD64_URL @@ -38,6 +43,29 @@ ARG OTELCOL_CONTRIB_URL=https://github.com/open-telemetry/opentelemetry-collecto ARG AKERNEL_VERSION=unknown ARG AKERNEL_REVISION=unknown +# The data plane is packaged independently from the Core wheel. Build the +# checksum-pinned source with its lockfile and include it in every node image. +FROM ${DATA_PLANE_BUILD_IMAGE} AS data-plane-builder +ARG DATA_PLANE_SOURCE_URL +ARG DATA_PLANE_SOURCE_SHA256 +RUN apt-get update && \ + apt-get install -y --no-install-recommends ca-certificates curl protobuf-compiler && \ + rm -rf /var/lib/apt/lists/* +WORKDIR /src/data-plane-gateway +RUN set -eux; \ + curl -fSL --retry 5 --retry-delay 2 \ + "${DATA_PLANE_SOURCE_URL}" -o /tmp/data-plane-source.tar.gz; \ + echo "${DATA_PLANE_SOURCE_SHA256} /tmp/data-plane-source.tar.gz" | sha256sum -c -; \ + tar -xzf /tmp/data-plane-source.tar.gz --strip-components=2 \ + --wildcards '*/data-plane-gateway/*'; \ + rm /tmp/data-plane-source.tar.gz; \ + cargo build --locked --release --all-features --bins; \ + mkdir -p /output/bin; \ + for binary in yr-edge-frontend yr-node-proxy yr-data-plane-forward; do \ + install -m 0755 "target/release/${binary}" "/output/bin/${binary}"; \ + strip "/output/bin/${binary}"; \ + done + FROM ${GVISOR_DOWNLOAD_IMAGE} AS gvisor-runtime ARG GVISOR_RELEASE ARG GVISOR_AMD64_URL @@ -223,7 +251,8 @@ ARG AKERNEL_REVISION ARG OPEN_YR_VERSION ARG OPEN_YR_CORE_WHEEL_URL ARG OPEN_YR_CORE_WHEEL_SHA256 -ARG OPEN_YR_RELEASE_BASE_URL +ARG OPEN_YR_CORE_AMD64_URL +ARG OPEN_YR_CORE_ARM64_URL ARG OPEN_YR_CORE_AMD64_SHA256 ARG OPEN_YR_CORE_ARM64_SHA256 ARG GVISOR_RELEASE @@ -299,21 +328,21 @@ ENV YR_INSTALLATION_DIR=/home/yuanrong # Install the complete, language-runtime-free openYuanRong control plane from # its checksum-pinned core wheel. A URL and checksum pair may override the -# release asset when validating an unreleased daily build. +# selected artifact when validating another build. RUN set -eux; \ case "${TARGETARCH:-}" in \ - amd64) wheel_arch=x86_64; wheel_platform=amd64; release_sha="${OPEN_YR_CORE_AMD64_SHA256}" ;; \ - arm64) wheel_arch=aarch64; wheel_platform=arm64; release_sha="${OPEN_YR_CORE_ARM64_SHA256}" ;; \ + amd64) wheel_arch=x86_64; default_core_url="${OPEN_YR_CORE_AMD64_URL}"; release_sha="${OPEN_YR_CORE_AMD64_SHA256}" ;; \ + arm64) wheel_arch=aarch64; default_core_url="${OPEN_YR_CORE_ARM64_URL}"; release_sha="${OPEN_YR_CORE_ARM64_SHA256}" ;; \ "") \ case "$(uname -m)" in \ - x86_64) wheel_arch=x86_64; wheel_platform=amd64; release_sha="${OPEN_YR_CORE_AMD64_SHA256}" ;; \ - aarch64) wheel_arch=aarch64; wheel_platform=arm64; release_sha="${OPEN_YR_CORE_ARM64_SHA256}" ;; \ + x86_64) wheel_arch=x86_64; default_core_url="${OPEN_YR_CORE_AMD64_URL}"; release_sha="${OPEN_YR_CORE_AMD64_SHA256}" ;; \ + aarch64) wheel_arch=aarch64; default_core_url="${OPEN_YR_CORE_ARM64_URL}"; release_sha="${OPEN_YR_CORE_ARM64_SHA256}" ;; \ *) echo "unsupported openYuanRong target architecture: $(uname -m)" >&2; exit 1 ;; \ esac ;; \ *) echo "unsupported openYuanRong target architecture: ${TARGETARCH}" >&2; exit 1 ;; \ esac; \ wheel_name="openyuanrong_core-${OPEN_YR_VERSION}-py3-none-manylinux_2_31_${wheel_arch}.whl"; \ - wheel_url="${OPEN_YR_RELEASE_BASE_URL}/${OPEN_YR_VERSION}/linux/${wheel_platform}/${wheel_name}"; \ + wheel_url="${default_core_url}"; \ wheel_sha="${release_sha}"; \ if [ -n "${OPEN_YR_CORE_WHEEL_URL}" ]; then \ test -n "${OPEN_YR_CORE_WHEEL_SHA256}"; \ @@ -343,6 +372,7 @@ RUN set -eux; \ COPY --from=runtime-image /yr-runtime-rootfs.img ${YR_INSTALLATION_DIR}/yr-runtime-rootfs.img +COPY --from=data-plane-builder /output/ ${YR_INSTALLATION_DIR}/data_plane/ COPY --from=gvisor-runtime /gvisor/runsc /usr/local/bin/runsc COPY --from=sandboxd-builder /src/sandboxd/output/sandboxd /usr/local/bin/sandboxd COPY --from=sandboxd-builder /src/sandboxd/output/sbox /usr/local/bin/sbox diff --git a/builder/scripts/akernel-entrypoint.sh b/builder/scripts/akernel-entrypoint.sh index 1df691c..6a4a8e5 100644 --- a/builder/scripts/akernel-entrypoint.sh +++ b/builder/scripts/akernel-entrypoint.sh @@ -28,7 +28,36 @@ fi case "${role}" in master|frontend) /usr/local/bin/ensure-component-cert - exec /bin/bash /home/yuanrong/entrypoint.sh "$@" + # The shared image uses systemd's stop signal for node/standalone. + # Translate it for the CLI and keep PID 1 alive until cleanup finishes. + child_pid="" + stop_requested=false + stop_control_plane() { + stop_requested=true + if [ -n "$child_pid" ]; then + kill -TERM "$child_pid" 2>/dev/null || true + fi + } + trap stop_control_plane TERM INT RTMIN+3 + /bin/bash /home/yuanrong/entrypoint.sh "$@" & + child_pid=$! + if [ "$stop_requested" = true ]; then + stop_control_plane + fi + # A trapped signal interrupts wait before the child has exited. + status=0 + while true; do + if wait "$child_pid"; then + status=0 + break + else + status=$? + fi + if ! kill -0 "$child_pid" 2>/dev/null; then + break + fi + done + exit "$status" ;; node) /bin/bash /root/prepare_node.sh diff --git a/builder/scripts/edge-config.sh b/builder/scripts/edge-config.sh new file mode 100644 index 0000000..3866ae5 --- /dev/null +++ b/builder/scripts/edge-config.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Copyright (c) 2026 Ant Group Corporation. +# SPDX-License-Identifier: Apache-2.0 + +configure_edge() { + EDGE_ARGS=() + if [ "${ENABLE_EDGE_FRONTEND:-false}" != true ]; then + return + fi + export EDGE_ADVERTISE_IP="${INSTANCE_IP:-${YR_NODE_IP:-127.0.0.1}}" + export YR_DATA_PLANE_EDGE_FRONTEND_PROXY_ROUTES_FILE=/run/akernel/edge-proxy-routes.json + mkdir -p /run/akernel/edge-http || return 1 + python3 - <<'PY' || return 1 +import json +import os +from pathlib import Path + +Path('/run/akernel/edge-http/internal-stats').write_text(json.dumps({ + 'pod_ip': os.environ['EDGE_ADVERTISE_IP'], + 'http_port': int(os.environ.get('EDGE_PLAIN_PORT', '80')), + 'https_port': int(os.environ.get('EDGE_TLS_PORT', '443')), +})) +routes = [{'name': 'internal-stats', 'path_prefix': '/internal-stats', + 'upstream': 'http://127.0.0.1:18081', 'strip_prefix': False}] +if os.environ.get('EDGE_GRAFANA_URL'): + routes.append({'name': 'grafana', 'path_prefix': '/grafana', + 'upstream': os.environ['EDGE_GRAFANA_URL'], 'strip_prefix': False}) +Path(os.environ['YR_DATA_PLANE_EDGE_FRONTEND_PROXY_ROUTES_FILE']).write_text(json.dumps(routes)) +PY + python3 -m http.server 18081 --bind 127.0.0.1 --directory /run/akernel/edge-http & + EDGE_ARGS=( + --enable_edge_frontend true + --edge_frontend_tls_bind "0.0.0.0:${EDGE_TLS_PORT:-443}" + --edge_frontend_plain_bind "0.0.0.0:${EDGE_PLAIN_PORT:-80}" + --edge_frontend_health_bind "0.0.0.0:${EDGE_HEALTH_PORT:-18080}" + --edge_frontend_tls_cert "${EDGE_TLS_CERT:-/home/yuanrong/.cert/module.crt}" + --edge_frontend_tls_key "${EDGE_TLS_KEY:-/home/yuanrong/.cert/module.key}" + --edge_frontend_control_plane_address "${EDGE_ADVERTISE_IP}:8888" + --edge_frontend_iam_address 127.0.0.1:31113 + --edge_frontend_validate_iam true + --edge_frontend_allowed_client_cidrs "${EDGE_ALLOWED_CLIENT_CIDRS:-0.0.0.0/0}" + --data_plane_log_dir "${DATA_PLANE_LOG_DIR:-/var/log/akernel-edge}" + --data_plane_log_stdout true + --edge_frontend_access_log_enabled true + ) + FRONTEND_SSL_ENABLE=false + IAM_SSL_ENABLE=false +} diff --git a/builder/scripts/master_entrypoint.sh b/builder/scripts/master_entrypoint.sh index 862d1b6..104d2bc 100644 --- a/builder/scripts/master_entrypoint.sh +++ b/builder/scripts/master_entrypoint.sh @@ -50,13 +50,6 @@ else echo "otelcol watchdog skipped" fi -# Set enable_traefik_provider based on TRAEFIK_MODE -if [ "${TRAEFIK_MODE:-etcd}" = "http" ]; then - ENABLE_TRAEFIK_PROVIDER=true -else - ENABLE_TRAEFIK_PROVIDER=false -fi - if [ -z "${LITEBUS_DATA_KEY:-}" ]; then echo "LITEBUS_DATA_KEY is required for akernel master/frontend" >&2 exit 1 @@ -68,7 +61,10 @@ if [ ! -x "${YR_BIN}" ]; then exit 1 fi -exec "${YR_BIN}" start --master --block true \ +. /root/edge-config.sh +configure_edge || exit 1 + +exec "${YR_BIN}" start --master --block true "${EDGE_ARGS[@]}" \ -e -c 0 -m 8000 -s 4096 -n $HOSTNAME \ -d $DEPLOY_PATH \ --fs_health_check_retry_interval 1 \ @@ -79,7 +75,7 @@ exec "${YR_BIN}" start --master --block true \ --enable_iam_server ${ENABLE_IAM_SERVER:-true} \ --iam_token_expired_time_span 604800 \ --ssl_base_path=/home/yuanrong/.cert/ \ - --frontend_ssl_enable=true \ + --frontend_ssl_enable=${FRONTEND_SSL_ENABLE:-true} \ --frontend_client_auth_type NoClientCert \ --enable_function_token_auth ${ENABLE_FUNCTION_TOKEN_AUTH:-true} \ --enable_inherit_env false \ @@ -98,12 +94,9 @@ exec "${YR_BIN}" start --master --block true \ --ds_rpc_thread_num 128 \ --function_proxy_merge_process_enable true \ --force_low_reliability_instance true \ - --enable_traefik_provider=${ENABLE_TRAEFIK_PROVIDER} \ - --traefik_http_entry_point=${TRAEFIK_HTTP_ENTRYPOINT:-websecure} \ - --traefik_enable_tls=${TRAEFIK_ENABLE_TLS:-false} \ - --traefik_forward_timeout_ms=3000 \ + --enable_traefik_provider=false \ --frontend_lease_bypass true \ - --iam_ssl_enable true \ + --iam_ssl_enable ${IAM_SSL_ENABLE:-true} \ --ssl_root_file ca.crt \ --ssl_cert_file module.crt \ --ssl_key_file module.key \ diff --git a/builder/scripts/yr_node_bootstrap.sh b/builder/scripts/yr_node_bootstrap.sh index 4b8b8e4..15fa567 100755 --- a/builder/scripts/yr_node_bootstrap.sh +++ b/builder/scripts/yr_node_bootstrap.sh @@ -45,15 +45,59 @@ echo "Using ${YR_NODE_IP} as the YuanRong node address" CHECKPOINT_DIR="/home/akernel/checkpoints" mkdir -p "${CHECKPOINT_DIR}" -# Select the legacy etcd registry or the FunctionMaster HTTP provider. -if [ "${TRAEFIK_MODE:-etcd}" = "etcd" ]; then - ENABLE_TRAEFIK_REGISTRY=${ENABLE_TRAEFIK_REGISTRY:-true} - ENABLE_TRAEFIK_PROVIDER=false -else - ENABLE_TRAEFIK_REGISTRY=false - ENABLE_TRAEFIK_PROVIDER=true +. /root/edge-config.sh +configure_edge || exit 1 + +NODE_PROXY_ARGS=() +if [ "${ENABLE_NODE_PROXY:-false}" = "true" ]; then + NODE_PROXY_ARGS=( + --enable_node_proxy true + --node_proxy_bind "0.0.0.0:${NODE_PROXY_PORT:-9443}" + --node_proxy_advertise_address "${YR_NODE_IP}:${NODE_PROXY_PORT:-9443}" + --node_proxy_health_bind "0.0.0.0:${NODE_PROXY_HEALTH_PORT:-18443}" + --node_proxy_security_mode network + --node_proxy_allowed_target_cidrs "${NODE_PROXY_ALLOWED_TARGET_CIDRS:?required for Node Proxy}" + --node_proxy_allowed_edge_cidrs "${NODE_PROXY_ALLOWED_EDGE_CIDRS:?required for Node Proxy}" + --data_plane_log_dir "${DATA_PLANE_LOG_DIR:-/var/log/akernel-edge}" + --data_plane_log_stdout true + ) fi +run_yuanrong() { + local child_pid="" + local stop_requested=false + local status=0 + stop_yuanrong() { + if [ "$stop_requested" = false ]; then + stop_requested=true + if [ -n "$child_pid" ]; then + kill -TERM "$child_pid" 2>/dev/null || true + fi + fi + } + trap stop_yuanrong TERM INT + "$@" & + child_pid=$! + if [ "$stop_requested" = true ]; then + kill -TERM "$child_pid" 2>/dev/null || true + fi + # A signal interrupts wait; keep the service MainPID alive until the CLI + # has finished its ordered shutdown, including sandbox cleanup. + while true; do + if wait "$child_pid"; then + status=0 + break + else + status=$? + fi + if ! kill -0 "$child_pid" 2>/dev/null; then + break + fi + done + trap - TERM INT + return "$status" +} + if [ "x${AKS_LOCAL_MODE}" == "xtrue" ]; then if [ -z "${LITEBUS_DATA_KEY:-}" ] && [ -r /home/akernel/iam-seed ]; then LITEBUS_DATA_KEY="$(tr -d '[:space:]' < /home/akernel/iam-seed)" @@ -63,7 +107,7 @@ if [ "x${AKS_LOCAL_MODE}" == "xtrue" ]; then echo "LITEBUS_DATA_KEY is required in standalone mode" >&2 exit 1 fi - /usr/bin/yr start --master \ + run_yuanrong /usr/bin/yr start --master "${NODE_PROXY_ARGS[@]}" "${EDGE_ARGS[@]}" \ --ip_address "${YR_NODE_IP}" \ --port_policy FIX \ --enable_function_scheduler=false \ @@ -72,7 +116,7 @@ if [ "x${AKS_LOCAL_MODE}" == "xtrue" ]; then --enable_iam_server=true \ --iam_token_expired_time_span 604800 \ --ssl_base_path=/home/yuanrong/.cert/ \ - --frontend_ssl_enable=true \ + --frontend_ssl_enable="${FRONTEND_SSL_ENABLE:-true}" \ --frontend_client_auth_type NoClientCert \ --enable_function_token_auth true \ --ds_node_timeout_s 30 \ @@ -87,13 +131,8 @@ if [ "x${AKS_LOCAL_MODE}" == "xtrue" ]; then --npu_collection_mode off \ --enable_distributed_master false \ --metrics_collector_type external \ - --enable_traefik_registry=${ENABLE_TRAEFIK_REGISTRY} \ - --enable_traefik_provider=${ENABLE_TRAEFIK_PROVIDER} \ - --traefik_enable_tls=${TRAEFIK_ENABLE_TLS:-false} \ - --traefik_etcd_prefix=traefik \ - --traefik_lease_ttl=300000 \ - --traefik_http_entrypoint=${TRAEFIK_HTTP_ENTRYPOINT:-websecure} \ - --traefik_http_entry_point=${TRAEFIK_HTTP_ENTRYPOINT:-websecure} \ + --enable_traefik_registry=false \ + --enable_traefik_provider=false \ --enable_metrics ${ENABLE_METRICS} \ --metrics_config_file "/home/yuanrong/metrics/metrics_config.json" \ --enable_trace ${ENABLE_TRACE} \ @@ -102,7 +141,7 @@ if [ "x${AKS_LOCAL_MODE}" == "xtrue" ]; then --function_proxy_merge_process_enable true \ --fc_agent_mgr_retry_times 30 \ --fc_agent_mgr_retry_cycle 60000 \ - --iam_ssl_enable true \ + --iam_ssl_enable "${IAM_SSL_ENABLE:-true}" \ --ssl_root_file ca.crt \ --ssl_cert_file module.crt \ --ssl_key_file module.key \ @@ -115,7 +154,7 @@ if [ "x${AKS_LOCAL_MODE}" == "xtrue" ]; then --enable_sandbox_router true \ --enable_direct_routing false else - /usr/bin/yr start \ + run_yuanrong /usr/bin/yr start "${NODE_PROXY_ARGS[@]}" "${EDGE_ARGS[@]}" \ --ip_address "${YR_NODE_IP}" \ --port_policy FIX \ --ds_node_timeout_s 30 \ @@ -136,11 +175,7 @@ else --enable_trace ${ENABLE_TRACE} \ --trace_config "$(cat /home/yuanrong/trace/trace_config.json)" \ -n ${HOSTNAME} \ - --enable_traefik_registry=${ENABLE_TRAEFIK_REGISTRY} \ - --traefik_enable_tls=${TRAEFIK_ENABLE_TLS:-false} \ - --traefik_etcd_prefix=traefik \ - --traefik_lease_ttl=300000 \ - --traefik_http_entrypoint=${TRAEFIK_HTTP_ENTRYPOINT:-websecure} \ + --enable_traefik_registry=false \ --log_root "${YR_LOG_PATH}" \ --fc_agent_mgr_retry_times 30 \ --fc_agent_mgr_retry_cycle 60000 \ diff --git a/builder/systemd_services/yuanrong.service b/builder/systemd_services/yuanrong.service index 4572f5b..6b9ef36 100644 --- a/builder/systemd_services/yuanrong.service +++ b/builder/systemd_services/yuanrong.service @@ -4,7 +4,7 @@ Description=yuanrong.service [Service] #Type=simple PIDFile=/run/yuanrong.pid -PassEnvironment=ETCD_PORT ETCD_PEER_PORT ETCD_ADDRESS HOSTNAME AKS_LOCAL_MODE AKERNEL_NODE_IP INSTANCE_IP LITEBUS_DATA_KEY YR_LOG_PATH YR_INSTALLATION_DIR YR_RRT_CONTROL_SOCKET_PATH YR_IMAGE_PROCESS_CONFIG ENABLE_METRICS ENABLE_TRACE TRAEFIK_MODE TRAEFIK_ENABLE_TLS TRAEFIK_HTTP_ENTRYPOINT +PassEnvironment=ENABLE_NODE_PROXY NODE_PROXY_PORT NODE_PROXY_HEALTH_PORT NODE_PROXY_ALLOWED_TARGET_CIDRS NODE_PROXY_ALLOWED_EDGE_CIDRS DATA_PLANE_LOG_DIR ETCD_PORT ETCD_PEER_PORT ETCD_ADDRESS HOSTNAME AKS_LOCAL_MODE AKERNEL_NODE_IP INSTANCE_IP LITEBUS_DATA_KEY YR_LOG_PATH YR_INSTALLATION_DIR YR_RRT_CONTROL_SOCKET_PATH YR_IMAGE_PROCESS_CONFIG ENABLE_METRICS ENABLE_TRACE ENABLE_EDGE_FRONTEND EDGE_TLS_PORT EDGE_PLAIN_PORT EDGE_HEALTH_PORT EDGE_TLS_CERT EDGE_TLS_KEY EDGE_ALLOWED_CLIENT_CIDRS EDGE_GRAFANA_URL Environment="CONTAINER_EP=unix:///run/sandboxd/sandboxd.sock" Environment="RUNTIME_HOME_DIR=/home/yuanrong/runtime" Environment="YR_NOSET_CUDA_VISIBLE_DEVICES=1" @@ -12,7 +12,8 @@ ExecStartPre=-/bin/mkdir -p /home/yuanrong/runtime ExecStart=/usr/bin/bash /home/yuanrong/yr_node_bootstrap.sh ExecReload=/bin/kill -15 $MAINPID -KillMode=control-group +# The bootstrap forwards TERM and waits for the CLI's ordered cleanup. +KillMode=mixed Restart=always Delegate=yes UMask=000 diff --git a/builder/tests/test_node_supervisor.py b/builder/tests/test_node_supervisor.py new file mode 100644 index 0000000..4325306 --- /dev/null +++ b/builder/tests/test_node_supervisor.py @@ -0,0 +1,64 @@ +"""Exercise the real bootstrap supervisor with a blocked CLI cleanup.""" +import os +from pathlib import Path +import signal +import subprocess +import tempfile +import time +import unittest + + +class NodeSupervisorTest(unittest.TestCase): + def test_signal_waits_for_cleanup(self): + source = (Path(__file__).resolve().parents[1] / "scripts/yr_node_bootstrap.sh").read_text() + function = source.split("run_yuanrong() {", 1)[1].split('\nif [ "x${AKS_LOCAL_MODE}"', 1)[0] + function = "run_yuanrong() {" + function + for sig in (signal.SIGTERM, signal.SIGINT): + with self.subTest(signal=sig), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + child = root / "cli.sh" + child.write_text('''#!/bin/bash +cleanup() { + echo signal >> "$1/signals" + touch "$1/draining" + while [ ! -f "$1/release" ]; do sleep 0.02; done + touch "$1/cleaned" + exit 7 +} +trap 'cleanup "$1"' TERM +touch "$1/ready" +while :; do sleep 0.02; done +''') + supervisor = root / "supervisor.sh" + supervisor.write_text(function + '\nrun_yuanrong /bin/bash "$1" "$2"\n') + process = subprocess.Popen( + ["/bin/bash", str(supervisor), str(child), directory], start_new_session=True + ) + try: + self.wait_file(root / "ready", process) + process.send_signal(sig) + self.wait_file(root / "draining", process) + process.send_signal(sig) + time.sleep(0.15) + self.assertIsNone(process.poll(), "Bash exited before CLI cleanup") + (root / "release").touch() + self.assertEqual(process.wait(timeout=5), 7) + self.assertTrue((root / "cleaned").exists()) + self.assertEqual((root / "signals").read_text().splitlines(), ["signal"]) + finally: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=5) + + def wait_file(self, path, process): + deadline = time.monotonic() + 5 + while not path.exists(): + self.assertIsNone(process.poll(), "Supervisor exited while waiting for " + path.name) + self.assertLess(time.monotonic(), deadline) + time.sleep(0.01) + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/README.md b/deploy/README.md index 9ba6a48..2c5658d 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -179,7 +179,7 @@ start/stop instructions. The umbrella chart in [`akernel/`](./akernel/) bundles two subcharts: - **core** — scheduler + node-side components (etcd, master, frontend, node - DaemonSet, and Traefik) + DaemonSet, and Edge ingress) - **monitor** — observability stack (Prometheus, Grafana, Loki, Tempo) ```bash @@ -213,40 +213,18 @@ image: Each component can still override `master.image`, `frontend.image`, or `node.image` when a split-image deployment is required. -### Public Traefik entrypoints +### Public Edge entrypoints -For cloud deployments, use Traefik with two public entrypoints: - -```yaml -traefik: - enabled: true - enableWebEntrypoint: true - ports: - websecure: 443 - web: 80 -``` - -The `websecure` entrypoint serves the AKernel frontend API and exec websocket -over HTTPS/WSS. The `web` entrypoint serves function port-forwarding traffic -over plain HTTP/WS. With this layout the Python SDK only needs the LoadBalancer -host or IP: +The core chart starts Edge alongside Frontend and Node Proxy alongside each +node by default. Edge serves API and exec WebSocket traffic over HTTPS/WSS on +443, and published sandbox ports and reverse tunnels over HTTP/WS on 80. ```bash -export AKERNEL_SERVER_ADDRESS= +export AKERNEL_SERVER_ADDRESS= ``` -Do not set `traefik.tls.enabled` just to make port 443 work. The frontend -router is already configured as a TLS router; `traefik.tls.enabled` only mounts -a custom default certificate Secret. When it is `false`, Traefik uses its -default certificate. - -The legacy single-entrypoint mode is still available by setting -`traefik.enableWebEntrypoint=false`. In that mode API, exec, and function -traffic share `traefik.ports.tcp`, so SDK clients should use an explicit port: - -```bash -export AKERNEL_SERVER_ADDRESS=: -``` +See [Edge and Node Proxy ingress](#edge-and-node-proxy-ingress) for certificate, +network allowlist, and existing LoadBalancer migration settings. ### IAM token signing seed @@ -274,8 +252,8 @@ helm template akernel ./akernel \ The all-in-one image does not contain a TLS private key. The core chart creates one deployment-specific Secret and mounts the same certificate into master and frontend Pods. This certificate protects the openYuanrong frontend and IAM -service connections; it is separate from the certificate served by Traefik's -public `websecure` entrypoint. +service connections and is also the default Edge ingress certificate. Set +`dataPlane.edge.tlsSecretName` to use a dedicated ingress certificate. Regular `helm install` and `helm upgrade` reuse the existing Secret. For a render-and-apply workflow, create it once before rendering so a new certificate @@ -314,7 +292,7 @@ Per-vendor details are in [`terraform/huaweicloud/README.md`](./terraform/huaweicloud/README.md). The Alibaba Cloud Terraform defaults follow the recommended public layout: -frontend enabled, Traefik `websecure:443` plus `web:80`, and Grafana exposed +frontend enabled, Edge HTTPS 443 plus HTTP 80, and Grafana exposed through its own LoadBalancer when `install_monitor=true`. Set `install_dragonfly=true` to install the pinned official Dragonfly chart and inject its seed-client proxy into the node runtime configuration. @@ -326,7 +304,7 @@ loop-backed filestore. See the Aliyun guide for capacity, opt-out, and node replacement details. Only the AKernel all-in-one image is pushed to the registry selected by -`make config`. etcd, Traefik, Grafana, Prometheus, Loki, Tempo, and BusyBox use +`make config`. etcd, Grafana, Prometheus, Loki, Tempo, and BusyBox use their pinned official public images by default. Set the per-component image overrides when a private cluster requires mirrored third-party images. @@ -339,3 +317,57 @@ deploy/ ├── terraform/ # multi-cloud provisioning (aliyun, huaweicloud, shared) └── scripts/ # deployment and image helper scripts ``` + +## Edge and Node Proxy ingress + +Node shutdown is supervised by systemd. The YuanRong bootstrap forwards the +stop signal to the Go CLI and waits for its ordered cleanup before exiting; +sandboxd stops after YuanRong. The YuanRong service uses `KillMode=mixed` so +the bootstrap controls the initial shutdown of its child processes. + +The image must contain the YuanRong Edge and Node Proxy binaries and Go CLI +data-plane deployment support. The builder pins Core and the data-plane source to the matching +`2b54c26885c6` build, including `config.sh`/`deploy.sh` and FunctionProxy +address registration. RRT and the sandbox SDK use `0.10.2rc2`. Both data-plane +components are enabled by default. Edge uses the +component TLS certificate unless an existing Secret (`tls.crt` and `tls.key`) +is selected with `dataPlane.edge.tlsSecretName`. Configure the allowed client, +Edge Pod and sandbox destination CIDRs for the deployment: + +```yaml +dataPlane: + enabled: true + edge: + tlsSecretName: akernel-edge-tls + allowedClientCIDRs: "0.0.0.0/0" + service: + name: akernel-edge + type: LoadBalancer + nodeProxy: + allowedEdgeCIDRs: "192.168.0.0/16" + allowedTargetCIDRs: "10.88.0.0/16" +``` + +Replace the CIDRs with those of the deployment. Node Proxy defaults allow +RFC1918 private networks; narrow them to the actual Pod and sandbox ranges. Edge exposes HTTPS/WSS on +service port 443 and HTTP/WS on port 80; API and authenticated direct routes +use TLS. The local Frontend and IAM hops use HTTP with IAM validation enabled. +Node Proxy uses network security mode and only accepts connections from the +configured Edge CIDRs. Node readiness includes Node Proxy; nodes roll one at +a time. Frontend readiness includes Edge route readiness. + +During migration, `dataPlane.edge.service.name` can match the existing ingress +Service name to retain its LoadBalancer identity and public address. Transfer +its annotations, clusterIP and loadBalancerIP as well. The Service then selects +Frontend/Edge pods; the Traefik Deployment and configuration are removed. +Existing SDK API and gateway address settings select TLS or plain WebSocket. +The `/internal-stats` endpoint reports the Edge Pod address and listener ports +for SDK `internal=True` URLs. Set `dataPlane.edge.grafanaURL` to the Grafana +HTTP service URL to expose its configured `/grafana` subpath through Edge. + +Cloud profiles use `edge_*` and `node_proxy_allowed_*` Terraform variables. +Migrate existing ingress Service annotations, name and IP to the corresponding +`edge_service_*` variables before planning an upgrade. + +`make print-env` discovers the Service labeled `app.kubernetes.io/component=edge`. +Set `AKERNEL_GATEWAY_SERVICE` to select a specific gateway Service. diff --git a/deploy/akernel/charts/core/templates/edge/secret.yaml b/deploy/akernel/charts/core/templates/edge/secret.yaml new file mode 100644 index 0000000..94141b3 --- /dev/null +++ b/deploy/akernel/charts/core/templates/edge/secret.yaml @@ -0,0 +1,11 @@ +{{- if and .Values.dataPlane.enabled .Values.dataPlane.edge.tls.createSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ required "dataPlane.edge.tlsSecretName is required when creating its Secret" .Values.dataPlane.edge.tlsSecretName }} + namespace: {{ .Release.Namespace }} +type: kubernetes.io/tls +data: + tls.crt: {{ required "dataPlane.edge.tls.cert is required" .Values.dataPlane.edge.tls.cert | b64enc | quote }} + tls.key: {{ required "dataPlane.edge.tls.key is required" .Values.dataPlane.edge.tls.key | b64enc | quote }} +{{- end }} diff --git a/deploy/akernel/charts/core/templates/edge/service.yaml b/deploy/akernel/charts/core/templates/edge/service.yaml new file mode 100644 index 0000000..449d259 --- /dev/null +++ b/deploy/akernel/charts/core/templates/edge/service.yaml @@ -0,0 +1,33 @@ +{{- if .Values.dataPlane.enabled }} +{{- if not .Values.frontend.enabled }} +{{- fail "dataPlane.enabled requires frontend.enabled" }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.dataPlane.edge.service.name }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: edge + {{- with .Values.dataPlane.edge.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.dataPlane.edge.service.type }} + {{- with .Values.dataPlane.edge.service.clusterIP }} + clusterIP: {{ . }} + {{- end }} + {{- with .Values.dataPlane.edge.service.loadBalancerIP }} + loadBalancerIP: {{ . | quote }} + {{- end }} + ports: + - name: websecure + port: {{ .Values.dataPlane.edge.service.httpsPort }} + targetPort: websecure + - name: web + port: {{ .Values.dataPlane.edge.service.httpPort }} + targetPort: web + selector: + app: akernel-frontend +{{- end }} diff --git a/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml b/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml index 2b02967..9aa2ca8 100644 --- a/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml +++ b/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml @@ -87,6 +87,28 @@ spec: command: - /usr/local/bin/akernel-entrypoint env: + {{- if .Values.dataPlane.enabled }} + - name: ENABLE_EDGE_FRONTEND + value: "true" + - name: EDGE_GRAFANA_URL + value: {{ .Values.dataPlane.edge.grafanaURL | quote }} + - name: INSTANCE_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: EDGE_TLS_CERT + value: /etc/akernel-edge-tls/tls.crt + - name: EDGE_TLS_KEY + value: /etc/akernel-edge-tls/tls.key + - name: EDGE_TLS_PORT + value: {{ .Values.dataPlane.edge.tlsPort | quote }} + - name: EDGE_PLAIN_PORT + value: {{ .Values.dataPlane.edge.plainPort | quote }} + - name: EDGE_HEALTH_PORT + value: {{ .Values.dataPlane.edge.healthPort | quote }} + - name: EDGE_ALLOWED_CLIENT_CIDRS + value: {{ required "dataPlane.edge.allowedClientCIDRs is required" .Values.dataPlane.edge.allowedClientCIDRs | quote }} + {{- end }} - name: AKERNEL_ROLE value: "frontend" - name: LITEBUS_DATA_KEY @@ -129,22 +151,30 @@ spec: - name: ENABLE_TRACE value: {{ ne (.Values.monitoring.tempoEndpoint | default "") "" | ternary "true" "false" | quote }} ports: + {{- if .Values.dataPlane.enabled }} + - name: websecure + containerPort: {{ .Values.dataPlane.edge.tlsPort }} + - name: web + containerPort: {{ .Values.dataPlane.edge.plainPort }} + - name: edge-health + containerPort: {{ .Values.dataPlane.edge.healthPort }} + {{- end }} - name: http containerPort: 8888 livenessProbe: httpGet: path: /healthz port: 8888 - scheme: HTTPS + scheme: {{ .Values.dataPlane.enabled | ternary "HTTP" "HTTPS" }} initialDelaySeconds: 15 periodSeconds: 20 timeoutSeconds: 5 failureThreshold: 3 readinessProbe: httpGet: - path: /healthz - port: 8888 - scheme: HTTPS + path: {{ .Values.dataPlane.enabled | ternary "/readyz" "/healthz" }} + port: {{ .Values.dataPlane.enabled | ternary (.Values.dataPlane.edge.healthPort | int) 8888 }} + scheme: {{ .Values.dataPlane.enabled | ternary "HTTP" "HTTPS" }} initialDelaySeconds: 5 periodSeconds: 10 timeoutSeconds: 3 @@ -152,6 +182,11 @@ spec: resources: {{- toYaml .Values.frontend.resources | nindent 12 }} volumeMounts: + {{- if .Values.dataPlane.enabled }} + - name: edge-tls + mountPath: /etc/akernel-edge-tls + readOnly: true + {{- end }} - name: component-tls mountPath: /home/yuanrong/.cert readOnly: true @@ -161,6 +196,19 @@ spec: readOnly: true {{- end }} volumes: + {{- if .Values.dataPlane.enabled }} + - name: edge-tls + secret: + secretName: {{ .Values.dataPlane.edge.tlsSecretName | default (include "core.componentTLSSecretName" .) }} + {{- if not .Values.dataPlane.edge.tlsSecretName }} + items: + - key: module.crt + path: tls.crt + - key: module.key + path: tls.key + {{- end }} + defaultMode: 0400 + {{- end }} - name: component-tls secret: secretName: {{ include "core.componentTLSSecretName" . }} diff --git a/deploy/akernel/charts/core/templates/master/akernel_master.yaml b/deploy/akernel/charts/core/templates/master/akernel_master.yaml index de37591..12532ef 100644 --- a/deploy/akernel/charts/core/templates/master/akernel_master.yaml +++ b/deploy/akernel/charts/core/templates/master/akernel_master.yaml @@ -112,12 +112,6 @@ spec: value: {{ ne (.Values.monitoring.prometheusEndpoint | default "") "" | ternary "true" "false" | quote }} - name: ENABLE_TRACE value: {{ ne (.Values.monitoring.tempoEndpoint | default "") "" | ternary "true" "false" | quote }} - - name: TRAEFIK_MODE - value: {{ .Values.traefik.mode | default "http" | quote }} - - name: TRAEFIK_ENABLE_TLS - value: {{ .Values.traefik.enableWebEntrypoint | default false | ternary false (.Values.traefik.enableTLS | default false) | quote }} - - name: TRAEFIK_HTTP_ENTRYPOINT - value: {{ .Values.traefik.enableWebEntrypoint | default false | ternary "web" "websecure" | quote }} ports: - name: http containerPort: 8888 diff --git a/deploy/akernel/charts/core/templates/node/daemonset.yaml b/deploy/akernel/charts/core/templates/node/daemonset.yaml index 01eb11d..187f72d 100644 --- a/deploy/akernel/charts/core/templates/node/daemonset.yaml +++ b/deploy/akernel/charts/core/templates/node/daemonset.yaml @@ -21,11 +21,11 @@ spec: {{- if .Values.kruise.enabled }} rollingUpdate: rollingUpdateType: Standard - maxUnavailable: 200 + maxUnavailable: 1 partition: 0 {{- else }} rollingUpdate: - maxUnavailable: 200 + maxUnavailable: 1 {{- end }} type: RollingUpdate selector: @@ -65,6 +65,15 @@ spec: imagePullPolicy: {{ include "core.image.pullPolicy" (dict "root" . "image" .Values.node.image) }} command: - /usr/local/bin/akernel-entrypoint + {{- if .Values.dataPlane.enabled }} + readinessProbe: + httpGet: + path: /readyz + port: {{ .Values.dataPlane.nodeProxy.healthPort }} + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + {{- end }} lifecycle: postStart: exec: @@ -72,10 +81,26 @@ spec: - bash - -c - | - cp /etc/resolv.conf /etc/resolv_akernel.conf && sed -i 's/127.0.0.1/10.88.0.1/g' /etc/resolv_akernel.conf + sandbox_resolver_ip={{ first (splitList "/" (.Values.node.sandboxIPRange | default "10.88.0.1/16")) | quote }} + awk -v resolver="${sandbox_resolver_ip}" ' + $1 == "nameserver" && $2 ~ /^127\./ { $2 = resolver } + { print } + ' /etc/resolv.conf > /etc/resolv_akernel.conf resources: {{- toYaml .Values.node.resources | nindent 10 }} env: + {{- if .Values.dataPlane.enabled }} + - name: ENABLE_NODE_PROXY + value: "true" + - name: NODE_PROXY_PORT + value: {{ .Values.dataPlane.nodeProxy.port | quote }} + - name: NODE_PROXY_HEALTH_PORT + value: {{ .Values.dataPlane.nodeProxy.healthPort | quote }} + - name: NODE_PROXY_ALLOWED_TARGET_CIDRS + value: {{ required "dataPlane.nodeProxy.allowedTargetCIDRs is required" .Values.dataPlane.nodeProxy.allowedTargetCIDRs | quote }} + - name: NODE_PROXY_ALLOWED_EDGE_CIDRS + value: {{ required "dataPlane.nodeProxy.allowedEdgeCIDRs is required" .Values.dataPlane.nodeProxy.allowedEdgeCIDRs | quote }} + {{- end }} - name: AKERNEL_ROLE value: "node" - name: RUNSC_AKERNEL @@ -126,12 +151,6 @@ spec: value: {{ ne (.Values.monitoring.prometheusEndpoint | default "") "" | ternary "true" "false" | quote }} - name: ENABLE_TRACE value: {{ ne (.Values.monitoring.tempoEndpoint | default "") "" | ternary "true" "false" | quote }} - - name: TRAEFIK_MODE - value: {{ .Values.traefik.mode | default "http" | quote }} - - name: TRAEFIK_ENABLE_TLS - value: {{ .Values.traefik.enableWebEntrypoint | default false | ternary false (.Values.traefik.enableTLS | default false) | quote }} - - name: TRAEFIK_HTTP_ENTRYPOINT - value: {{ .Values.traefik.enableWebEntrypoint | default false | ternary "web" "websecure" | quote }} securityContext: privileged: true volumeMounts: diff --git a/deploy/akernel/charts/core/templates/traefik/configmap.yaml b/deploy/akernel/charts/core/templates/traefik/configmap.yaml deleted file mode 100644 index 585259d..0000000 --- a/deploy/akernel/charts/core/templates/traefik/configmap.yaml +++ /dev/null @@ -1,151 +0,0 @@ -{{- if .Values.traefik.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: traefik-static - namespace: {{ .Release.Namespace }} -data: - traefik.yml: | - entryPoints: - {{- if .Values.traefik.enableWebEntrypoint }} - websecure: - address: ":{{ .Values.traefik.ports.websecure }}" - web: - address: ":{{ .Values.traefik.ports.web }}" - {{- else }} - websecure: - address: ":{{ .Values.traefik.ports.tcp }}" - {{- end }} - traefik: - address: ":{{ .Values.traefik.ports.dashboard }}" - - {{- $masterEtcd := .Values.master.etcd | default dict }} - {{- $etcdHost := get $masterEtcd "host" | default (printf "akernel-etcd.%s.svc.cluster.local" .Release.Namespace) }} - {{- $etcdPort := get $masterEtcd "port" | default "2379" }} - {{- $masterHost := .Values.master.host | default (printf "akernel-master.%s.svc.cluster.local" .Release.Namespace) }} - {{- $masterPort := .Values.master.port | default "22770" }} - providers: - file: - directory: /etc/traefik/dynamic - watch: true - {{- if eq (.Values.traefik.mode | default "http") "http" }} - http: - endpoint: "http://{{ $masterHost }}:{{ $masterPort }}/global-scheduler/traefik/config" - pollInterval: "1s" - {{- else }} - etcd: - endpoints: - - "{{ $etcdHost }}:{{ $etcdPort }}" - rootKey: "traefik" - {{- end }} - - ping: {} - - api: - dashboard: true - insecure: true - - log: - level: INFO - filePath: /var/logs/traefik.log - - accessLog: - format: json - fields: - names: - # Exec places authentication data in the websocket query string. - # Keep status and timing fields, but never persist the request path. - RequestPath: drop ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: traefik-dynamic - namespace: {{ .Release.Namespace }} -data: - config.yml: | - {{- if .Values.traefik.tls.enabled }} - tls: - stores: - default: - defaultCertificate: - certFile: /openyuanrong/cert/module.crt - keyFile: /openyuanrong/cert/module.key - certificates: - - certFile: /openyuanrong/cert/module.crt - keyFile: /openyuanrong/cert/module.key - {{- end }} - - http: - routers: - frontend-terminal: - entryPoints: - - websecure - rule: "PathPrefix(`/terminal`) || PathPrefix(`/api/instances`) || PathPrefix(`/api/jobs`) || PathPrefix(`/functions`) || PathPrefix(`/api-docs`) || PathPrefix(`/admin/v1/functions`) || PathPrefix(`/serverless/v1/functions`) || PathPrefix(`/serverless/v1/stream`) || PathPrefix(`/serverless/v1/componentshealth`) || PathPrefix(`/serverless/v1/posix`) || PathPrefix(`/serverless/v2`) || PathPrefix(`/frontend/v1/instance`) || PathPrefix(`/datasystem/v1`) || PathPrefix(`/app/v1`) || PathPrefix(`/client/v1/lease`) || PathPrefix(`/invocations`) || PathPrefix(`/global-scheduler`) || Path(`/healthz`)" - service: {{ if .Values.frontend.enabled }}akernel-frontend{{ else }}akernel-master{{ end }} - tls: {} - sandbox-router: - entryPoints: - - websecure - rule: "PathPrefix(`/api/sandbox`) || PathPrefix(`/direct/`) || Path(`/direct`)" - priority: 100 - service: {{ if .Values.frontend.enabled }}akernel-frontend{{ else }}akernel-master{{ end }} - tls: {} - {{- if .Values.traefik.grafana.enabled }} - grafana: - entryPoints: - - websecure - rule: "PathPrefix(`/grafana`)" - service: grafana - tls: {} - {{- end }} - {{- if .Values.traefik.internalStats.enabled }} - internal-stats: - entryPoints: - - websecure - rule: "Path(`/internal-stats`)" - service: internal-stats - middlewares: - - internal-stats-strip - tls: {} - {{- end }} - - services: - {{ if .Values.frontend.enabled }}akernel-frontend{{ else }}akernel-master{{ end }}: - loadBalancer: - serversTransport: frontend-transport - servers: - - url: "https://{{ if .Values.frontend.enabled }}akernel-frontend{{ else }}akernel-master{{ end }}:8888" - {{- if .Values.traefik.grafana.enabled }} - grafana: - loadBalancer: - servers: - - url: "{{ .Values.traefik.grafana.url }}" - {{- end }} - {{- if .Values.traefik.internalStats.enabled }} - internal-stats: - loadBalancer: - servers: - - url: "http://localhost:{{ .Values.traefik.internalStats.port }}" - {{- end }} - - {{- if .Values.traefik.internalStats.enabled }} - middlewares: - internal-stats-strip: - stripPrefix: - prefixes: - - "/internal-stats" - {{- end }} - - serversTransports: - frontend-transport: - insecureSkipVerify: true - {{- if .Values.traefik.frontendTransport.disableHTTP2 }} - # Force HTTP/1.1 to the frontend backend. With an HTTP/2 backend, - # Traefik multiplexes all requests over a single connection; the 1s - # full-config reloads (390+ churning per-sandbox routes) drain and - # rebuild that connection, stalling in-flight requests for 7-10s. - # An HTTP/1.1 connection pool is unaffected. - disableHTTP2: true - {{- end }} -{{- end }} diff --git a/deploy/akernel/charts/core/templates/traefik/deployment.yaml b/deploy/akernel/charts/core/templates/traefik/deployment.yaml deleted file mode 100644 index 0c7373a..0000000 --- a/deploy/akernel/charts/core/templates/traefik/deployment.yaml +++ /dev/null @@ -1,147 +0,0 @@ -{{- if .Values.traefik.enabled }} -{{- if .Values.kruise.enabled }} -apiVersion: apps.kruise.io/v1alpha1 -kind: CloneSet -{{- else }} -apiVersion: apps/v1 -kind: Deployment -{{- end }} -metadata: - name: traefik - namespace: {{ .Release.Namespace }} - labels: - app: traefik -spec: - {{- if .Values.kruise.enabled }} - updateStrategy: - type: InPlaceIfPossible - {{- else }} - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - maxSurge: 1 - {{- end }} - replicas: {{ .Values.traefik.replicas | default 1 }} - selector: - matchLabels: - app: traefik - template: - metadata: - {{- with .Values.traefik.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - app: traefik - {{- with .Values.traefik.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - {{- if .Values.kruise.enabled }} - readinessGates: - - conditionType: InPlaceUpdateReady - {{- end }} - {{- if .Values.traefik.affinity }} - affinity: - {{- toYaml .Values.traefik.affinity | nindent 8 }} - {{- else if gt (int (.Values.traefik.replicas | default 1)) 1 }} - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - traefik - topologyKey: kubernetes.io/hostname - {{- end }} - {{- with .Values.traefik.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: traefik - image: "{{ .Values.traefik.image.repository }}:{{ .Values.traefik.image.tag }}" - imagePullPolicy: {{ .Values.traefik.image.pullPolicy }} - args: - - --configFile=/etc/traefik/traefik.yml - ports: - {{- if .Values.traefik.enableWebEntrypoint }} - - name: websecure - containerPort: {{ .Values.traefik.ports.websecure }} - - name: web - containerPort: {{ .Values.traefik.ports.web }} - {{- else }} - - name: tcp - containerPort: {{ .Values.traefik.ports.tcp }} - {{- end }} - - name: dashboard - containerPort: {{ .Values.traefik.ports.dashboard }} - livenessProbe: - httpGet: - path: /ping - port: dashboard - initialDelaySeconds: 10 - periodSeconds: 15 - timeoutSeconds: 3 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /ping - port: dashboard - initialDelaySeconds: 5 - periodSeconds: 10 - timeoutSeconds: 3 - failureThreshold: 3 - volumeMounts: - - name: static-config - mountPath: /etc/traefik/traefik.yml - subPath: traefik.yml - readOnly: true - - name: dynamic-config - mountPath: /etc/traefik/dynamic - readOnly: true - - name: logs - mountPath: /var/logs - {{- if .Values.traefik.tls.enabled }} - - name: tls-cert - mountPath: /openyuanrong/cert - readOnly: true - {{- end }} - {{- if .Values.traefik.internalStats.enabled }} - - name: internal-stats - image: "{{ .Values.traefik.internalStats.image }}" - command: ['sh', '-c', 'mkdir -p /var/www && echo "{\"pod_ip\":\"$POD_IP\"}" > /var/www/index.html && httpd -f -p {{ .Values.traefik.internalStats.port }} -h /var/www'] - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - ports: - - name: internal-stats - containerPort: {{ .Values.traefik.internalStats.port }} - {{- end }} - volumes: - - name: static-config - configMap: - name: traefik-static - - name: dynamic-config - configMap: - name: traefik-dynamic - - name: logs - emptyDir: {} - {{- if .Values.traefik.tls.enabled }} - - name: tls-cert - secret: - secretName: {{ .Values.traefik.tls.secretName | default "traefik-tls" }} - optional: false - {{- end }} -{{- end }} diff --git a/deploy/akernel/charts/core/templates/traefik/secret.yaml b/deploy/akernel/charts/core/templates/traefik/secret.yaml deleted file mode 100644 index c64c39d..0000000 --- a/deploy/akernel/charts/core/templates/traefik/secret.yaml +++ /dev/null @@ -1,11 +0,0 @@ -{{- if and .Values.traefik.enabled .Values.traefik.tls.enabled .Values.traefik.tls.createSecret }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ .Values.traefik.tls.secretName | default "traefik-tls" }} - namespace: {{ .Release.Namespace }} -type: Opaque -data: - module.crt: {{ .Values.traefik.tls.moduleCrt | default "" | b64enc | quote }} - module.key: {{ .Values.traefik.tls.moduleKey | default "" | b64enc | quote }} -{{- end }} diff --git a/deploy/akernel/charts/core/templates/traefik/service.yaml b/deploy/akernel/charts/core/templates/traefik/service.yaml deleted file mode 100644 index 5696fcb..0000000 --- a/deploy/akernel/charts/core/templates/traefik/service.yaml +++ /dev/null @@ -1,39 +0,0 @@ -{{- if .Values.traefik.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: traefik - namespace: {{ .Release.Namespace }} - {{- with .Values.traefik.service.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - type: {{ .Values.traefik.service.type }} - {{- with .Values.traefik.service.clusterIP }} - clusterIP: {{ . }} - {{- end }} - {{- if and (eq .Values.traefik.service.type "LoadBalancer") .Values.traefik.service.loadBalancerIP }} - loadBalancerIP: {{ .Values.traefik.service.loadBalancerIP | quote }} - {{- end }} - ports: - {{- if .Values.traefik.enableWebEntrypoint }} - - name: websecure - port: {{ .Values.traefik.ports.websecure }} - targetPort: websecure - - name: web - port: {{ .Values.traefik.ports.web }} - targetPort: web - {{- else }} - - name: tcp - port: {{ .Values.traefik.ports.tcp }} - targetPort: tcp - {{- end }} - {{- if ne .Values.traefik.service.type "LoadBalancer" }} - - name: dashboard - port: {{ .Values.traefik.ports.dashboard }} - targetPort: dashboard - {{- end }} - selector: - app: traefik -{{- end }} diff --git a/deploy/akernel/charts/core/values.yaml b/deploy/akernel/charts/core/values.yaml index ab638fd..bafacaa 100644 --- a/deploy/akernel/charts/core/values.yaml +++ b/deploy/akernel/charts/core/values.yaml @@ -525,47 +525,31 @@ node: provider="kubernetes" sock_path="/var/run/resource.sock" -traefik: - enabled: false - mode: "http" - enableTLS: false - # When enabled, creates a separate "web" entrypoint (no TLS) on its own port - # for port forwarding traffic, while frontend API stays on "websecure" with TLS. - enableWebEntrypoint: true - # Transport options for the frontend (akernel-master) backend. - frontendTransport: - # Force HTTP/1.1 to the frontend backend. The HTTP provider reloads the - # full config every 1s, and the per-sandbox routes churn constantly; each - # reload drains Traefik's backend connections. An HTTP/2 backend uses a - # single multiplexed connection, so a reload stalls all in-flight requests - # for 7-10s. An HTTP/1.1 connection pool avoids this. Keep true unless the - # frontend backend requires HTTP/2 (e.g. gRPC over :8888). - disableHTTP2: true - replicas: 1 - image: - repository: traefik - tag: v3.6.8 - pullPolicy: IfNotPresent - service: - type: LoadBalancer - clusterIP: "" - annotations: {} - loadBalancerIP: "" - ports: - tcp: 8888 - websecure: 443 - web: 80 - dashboard: 9990 - tls: - enabled: false - createSecret: false - secretName: "traefik-tls" - moduleCrt: "" - moduleKey: "" - internalStats: - enabled: false - image: busybox:1.37.0-musl - port: 8079 - grafana: - enabled: false - url: "http://grafana.akernel-monitor.svc:3000" +# Edge handles API, direct, tunnel and published-port ingress. It is started +# with Frontend by the Go CLI; Node Proxy runs in each node network namespace. +dataPlane: + enabled: true + edge: + tlsPort: 443 + plainPort: 80 + healthPort: 18080 + tlsSecretName: "" + tls: + createSecret: false + cert: "" + key: "" + grafanaURL: "" + allowedClientCIDRs: "0.0.0.0/0" + service: + name: akernel-edge + type: LoadBalancer + annotations: {} + clusterIP: "" + loadBalancerIP: "" + httpsPort: 443 + httpPort: 80 + nodeProxy: + port: 9443 + healthPort: 18443 + allowedTargetCIDRs: "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" + allowedEdgeCIDRs: "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" diff --git a/deploy/akernel/charts/monitor/templates/prometheus/configmap.yaml b/deploy/akernel/charts/monitor/templates/prometheus/configmap.yaml index 82a80b4..2cb1ea2 100644 --- a/deploy/akernel/charts/monitor/templates/prometheus/configmap.yaml +++ b/deploy/akernel/charts/monitor/templates/prometheus/configmap.yaml @@ -66,7 +66,7 @@ data: regex: Running - source_labels: [__meta_kubernetes_pod_container_name] action: keep - regex: {{ $controlPlanePods.targetContainerRegex | default "akernel-master|akernel-frontend|etcd|traefik|manager|scheduler|seed-client|mysql|redis" | quote }} + regex: {{ $controlPlanePods.targetContainerRegex | default "akernel-master|akernel-frontend|etcd|manager|scheduler|seed-client|mysql|redis" | quote }} - source_labels: [__meta_kubernetes_pod_container_port_name, __meta_kubernetes_pod_container_port_number] action: keep regex: {{ printf "(%s);.*|;(%s)" ($controlPlanePods.targetPortNameRegex | default "http|client|dashboard|mysql|redis") ($controlPlanePods.targetPortNumberRegex | default "8000|4002|3306|6379") | quote }} @@ -103,7 +103,7 @@ data: target_label: component replacement: frontend - source_labels: [__meta_kubernetes_pod_label_app] - regex: (etcd|traefik) + regex: (etcd) action: replace target_label: component replacement: $1 @@ -168,7 +168,7 @@ data: regex: Running - source_labels: [__meta_kubernetes_pod_container_name] action: keep - regex: {{ $controlPlanePods.targetContainerRegex | default "akernel-master|akernel-frontend|etcd|traefik|manager|scheduler|seed-client|mysql|redis" | quote }} + regex: {{ $controlPlanePods.targetContainerRegex | default "akernel-master|akernel-frontend|etcd|manager|scheduler|seed-client|mysql|redis" | quote }} - source_labels: [__meta_kubernetes_pod_container_port_name, __meta_kubernetes_pod_container_port_number] action: keep regex: {{ printf "(%s);.*|;(%s)" ($controlPlanePods.targetPortNameRegex | default "http|client|dashboard|mysql|redis") ($controlPlanePods.targetPortNumberRegex | default "8000|4002|3306|6379") | quote }} @@ -205,7 +205,7 @@ data: target_label: component replacement: frontend - source_labels: [__meta_kubernetes_pod_label_app] - regex: (etcd|traefik) + regex: (etcd) action: replace target_label: component replacement: $1 diff --git a/deploy/akernel/charts/monitor/values.yaml b/deploy/akernel/charts/monitor/values.yaml index ebca533..1519df0 100644 --- a/deploy/akernel/charts/monitor/values.yaml +++ b/deploy/akernel/charts/monitor/values.yaml @@ -200,7 +200,7 @@ prometheusServer: namespaces: - "akernel" labelSelector: "" - targetContainerRegex: "akernel-master|akernel-frontend|etcd|traefik|manager|scheduler|seed-client|mysql|redis" + targetContainerRegex: "akernel-master|akernel-frontend|etcd|manager|scheduler|seed-client|mysql|redis" targetPortNameRegex: "http|client|dashboard|mysql|redis" targetPortNumberRegex: "8000|4002|3306|6379" kubelet: diff --git a/deploy/scripts/configure.sh b/deploy/scripts/configure.sh index 69f1915..2b8c8b4 100755 --- a/deploy/scripts/configure.sh +++ b/deploy/scripts/configure.sh @@ -332,16 +332,6 @@ frontend_replicas = 1 frontend_cpu = "1" frontend_memory = "2Gi" -traefik_enabled = true -install_traefik = true -traefik_service_type = "LoadBalancer" -traefik_enable_web_entrypoint = true -traefik_websecure_port = 443 -traefik_web_port = 80 -traefik_tls_enabled = false -traefik_tls_create_secret = false -traefik_internal_stats_enabled = true - install_prereqs = false install_monitor = ${install_monitor} @@ -387,15 +377,6 @@ frontend_replicas = 1 frontend_cpu = "1" frontend_memory = "2Gi" -install_traefik = true -traefik_public_access = true -traefik_enable_web_entrypoint = true -traefik_websecure_port = 443 -traefik_web_port = 80 -traefik_tls_enabled = false -traefik_tls_create_secret = false -traefik_internal_stats_enabled = true - install_prereqs = false install_monitor = ${install_monitor} diff --git a/deploy/scripts/print-sdk-env.sh b/deploy/scripts/print-sdk-env.sh index da184ef..5e0307e 100755 --- a/deploy/scripts/print-sdk-env.sh +++ b/deploy/scripts/print-sdk-env.sh @@ -52,14 +52,29 @@ get_lb_host() { printf '%s' "${host}" } -traefik_host="$(get_lb_host "${core_ns}" traefik)" -[[ -n "${traefik_host}" ]] || die "traefik LoadBalancer address is not ready" +gateway_service="${AKERNEL_GATEWAY_SERVICE:-}" +if [[ -z "${gateway_service}" ]]; then + gateway_service="$(kubectl --kubeconfig "${kubeconfig}" -n "${core_ns}" get svc \ + -l app.kubernetes.io/component=edge -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" +fi +[[ -n "${gateway_service}" ]] || die "Edge gateway Service not found; set AKERNEL_GATEWAY_SERVICE" +gateway_host="$(get_lb_host "${core_ns}" "${gateway_service}")" +[[ -n "${gateway_host}" ]] || die "${gateway_service} LoadBalancer address is not ready" + +https_port="$(kubectl --kubeconfig "${kubeconfig}" -n "${core_ns}" get svc "${gateway_service}" -o 'jsonpath={.spec.ports[?(@.name=="websecure")].port}')" +http_port="$(kubectl --kubeconfig "${kubeconfig}" -n "${core_ns}" get svc "${gateway_service}" -o 'jsonpath={.spec.ports[?(@.name=="web")].port}')" +[[ -n "${https_port}" && -n "${http_port}" ]] || die "Edge Service must expose websecure and web ports" token="$("${AKERNEL_REPO_ROOT}/deploy/scripts/generate-token.py" --env "${env_name}" --write-file "${dir}/token")" sdk_env="${dir}/sdk.env" { - printf 'export AKERNEL_SERVER_ADDRESS=%q\n' "${traefik_host}" + if [[ "${https_port}" == 443 && "${http_port}" == 80 ]]; then + printf 'export AKERNEL_SERVER_ADDRESS=%q\n' "${gateway_host}" + else + printf 'export AKERNEL_SERVER_ADDRESS=%q\n' "https://${gateway_host}:${https_port}" + printf 'export AKERNEL_GATEWAY_ADDRESS=%q\n' "http://${gateway_host}:${http_port}" + fi printf 'export AKERNEL_TOKEN=%q\n' "${token}" } > "${sdk_env}" chmod 600 "${sdk_env}" diff --git a/deploy/standalone/README.md b/deploy/standalone/README.md index b24912b..f0e54bd 100644 --- a/deploy/standalone/README.md +++ b/deploy/standalone/README.md @@ -2,15 +2,10 @@ This directory contains scripts and configurations for running AKernel in standalone mode using Docker or Pouch, without Kubernetes. The deployment uses -two containers on the default container bridge: - -- `akernel-node` runs the AKernel all-in-one image. -- `akernel-traefik` runs the official Traefik image as the external gateway. - -Keeping the gateway in a separate network namespace allows sandboxd's normal -`PREROUTING` rules to handle gateway traffic. The all-in-one frontend sends -traffic from the node network namespace, so the standalone sandboxd config -also enables its local-output DNAT support. +one privileged `akernel-node` container. The Go CLI starts Edge and Node Proxy +alongside the control plane, while systemd supervises YuanRong and sandboxd. +The standalone sandboxd config enables local-output DNAT for traffic from the +shared node network namespace. The default runtime is gVisor `runsc`. The bundled image also contains Kata Containers and Firecracker. Both `Sandbox(runtime="kata")` and @@ -108,8 +103,8 @@ initialize ACLs while pre-ACL sandboxes remain in its store. ``` deploy/standalone/ ├── README.md # This file -├── start.sh # Start AKernel and Traefik containers -├── stop.sh # Stop AKernel and Traefik containers +├── start.sh # Start the AKernel container +├── stop.sh # Stop the AKernel container └── config/ # Configuration files ├── config.json # OCI runtime configuration ├── oss_auths.json # OSS authentication (edit as needed) @@ -154,6 +149,10 @@ Edit `config/oss.json` and `config/registry.json` to point to your actual OSS an ### 3. Start AKernel +When upgrading an existing standalone installation, run `stop.sh` first. It +also removes the gateway container from earlier releases. The new deployment +uses the AKernel container IP for SDK access. + ```bash cd deploy/standalone ./start.sh @@ -165,16 +164,13 @@ This will: - Use `akerneldev/all-in-one:latest` if `IMAGE` is not set, reusing a local copy when present and otherwise pulling it from Docker Hub - Start the privileged AKernel all-in-one container -- Start an independent Traefik container for the HTTPS API and HTTP sandbox - port-forwarding gateway -- Configure Traefik to poll FunctionMaster's HTTP provider for per-sandbox - tunnel routes, including custom tunnel ports +- Start Edge and Node Proxy through the Go CLI inside the all-in-one container - Generate a deployment-specific IAM signing seed and a 24-hour SDK token - Generate a sandboxd config using `AKERNEL_NAT_BACKEND` (`iptables` by default) -- Print the Traefik container IP to use as `AKERNEL_SERVER_ADDRESS` +- Print the AKernel container IP to use as `AKERNEL_SERVER_ADDRESS` -No host ports are published. On Linux, the host accesses Traefik directly +No host ports are published. On Linux, the host accesses Edge directly through its Docker bridge IP. ### 4. Check Status @@ -183,8 +179,8 @@ through its Docker bridge IP. # View AKernel logs sudo docker logs -f akernel-node -# View gateway logs -sudo docker logs -f akernel-traefik +# View data-plane logs +sudo docker exec akernel-node ls /var/log/akernel-edge # Enter the container sudo docker exec -it akernel-node bash @@ -205,20 +201,20 @@ sudo docker exec akernel-node systemctl status ### SDK Connection -Traefik listens on port 443 for the AKernel API and port 80 for sandbox port -forwarding. These ports are not published on the host. Use the Traefik +Edge listens on port 443 for the AKernel API and port 80 for sandbox port +forwarding. These ports are not published on the host. Use the AKernel container IP printed by `start.sh`, or retrieve it later: ```bash -TRAEFIK_IP=$(docker inspect \ +NODE_IP=$(docker inspect \ --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' \ - akernel-traefik) + akernel-node) ``` Set the SDK environment: ```bash -export AKERNEL_SERVER_ADDRESS="${TRAEFIK_IP}" +export AKERNEL_SERVER_ADDRESS="${NODE_IP}" export AKERNEL_TOKEN="$(cat data/token)" ``` @@ -240,13 +236,6 @@ variable to test another registry, tag, or locally built image: IMAGE=":" ./start.sh ``` -The gateway defaults to `traefik:v3.6.8`. Override it independently when -needed: - -```bash -TRAEFIK_IMAGE="traefik:v3.6.8" ./start.sh -``` - ### Data Directory Location By default, data is stored in `./data`. To change this, edit `start.sh`: diff --git a/deploy/standalone/start.sh b/deploy/standalone/start.sh index 8faed46..da1e12c 100755 --- a/deploy/standalone/start.sh +++ b/deploy/standalone/start.sh @@ -11,13 +11,10 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" CONFIG_DIR="${SCRIPT_DIR}/config" DATA_DIR="${SCRIPT_DIR}/data" -FRONTEND_PORT="8888" ETCD_PORT="${ETCD_PORT:-2379}" ETCD_PEER_PORT="${ETCD_PEER_PORT:-2378}" -NODE_CONTAINER_NAME="akernel-node" -TRAEFIK_CONTAINER_NAME="akernel-traefik" +NODE_CONTAINER_NAME="${NODE_CONTAINER_NAME:-akernel-node}" IMAGE="${IMAGE:-akerneldev/all-in-one:latest}" -TRAEFIK_IMAGE="${TRAEFIK_IMAGE:-traefik:v3.6.8}" IAM_SEED_FILE="${DATA_DIR}/iam-seed" TOKEN_FILE="${DATA_DIR}/token" SANDBOXD_CONFIG_FILE="${DATA_DIR}/sandboxd/config.toml" @@ -162,7 +159,7 @@ configure_auth() { # Stop and remove existing container cleanup_existing() { local container - for container in "${NODE_CONTAINER_NAME}" "${TRAEFIK_CONTAINER_NAME}"; do + for container in "${NODE_CONTAINER_NAME}"; do if "${DOCKER_PREFIX[@]}" ${DOCKER_CMD} container inspect "${container}" &> /dev/null; then log_warn "Existing container '${container}' found; run stop.sh first" exit 1 @@ -349,12 +346,9 @@ prepare_host_network_modules() { log_info "Loaded host filter, bridge, conntrack, and ipset modules for the iptables ACL backend" } -# Start the AKernel all-in-one container. Traefik runs separately so traffic -# from the gateway enters this network namespace through PREROUTING. +# Start the all-in-one container with Edge and Node Proxy supervised by Go CLI. start_node_container() { log_info "Starting container: ${NODE_CONTAINER_NAME}" - # FunctionMaster's HTTP provider publishes the per-sandbox routes required - # by reverse tunnels; the legacy etcd mode cannot publish those routes. "${DOCKER_PREFIX[@]}" ${DOCKER_CMD} run -d \ --name "${NODE_CONTAINER_NAME}" \ @@ -364,9 +358,11 @@ start_node_container() { -e AKS_LOCAL_MODE="true" \ -e YR_RRT_CONTROL_SOCKET_PATH="/run/akernel" \ -e YR_IMAGE_PROCESS_CONFIG="${YR_IMAGE_PROCESS_CONFIG}" \ - -e TRAEFIK_MODE="http" \ - -e TRAEFIK_HTTP_ENTRYPOINT="web" \ - -e TRAEFIK_ENABLE_TLS="false" \ + -e ENABLE_EDGE_FRONTEND=true \ + -e EDGE_GRAFANA_URL="${EDGE_GRAFANA_URL:-}" \ + -e ENABLE_NODE_PROXY=true \ + -e NODE_PROXY_ALLOWED_EDGE_CIDRS="${NODE_PROXY_ALLOWED_EDGE_CIDRS:-127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16}" \ + -e NODE_PROXY_ALLOWED_TARGET_CIDRS="${NODE_PROXY_ALLOWED_TARGET_CIDRS:-10.0.0.0/8,172.16.0.0/12,192.168.0.0/16}" \ -e ETCD_PORT="${ETCD_PORT}" \ -e ETCD_PEER_PORT="${ETCD_PEER_PORT}" \ -e NODE_NAME="$(hostname)" \ @@ -415,86 +411,28 @@ container_ip() { --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$1" } -write_traefik_config() { - local node_ip="$1" - local traefik_dir="${DATA_DIR}/traefik" - mkdir -p "${traefik_dir}" - - cat > "${traefik_dir}/dynamic.yml" < /dev/null; then - log_info "Traefik gateway is ready" + if curl --noproxy '*' -fkSs "https://${gateway_ip}/healthz" > /dev/null; then + log_info "Edge gateway is ready" return 0 fi if ! "${DOCKER_PREFIX[@]}" ${DOCKER_CMD} inspect \ - --format '{{.State.Running}}' "${TRAEFIK_CONTAINER_NAME}" 2> /dev/null \ + --format '{{.State.Running}}' "${NODE_CONTAINER_NAME}" 2> /dev/null \ | grep -q true; then - log_error "Traefik exited during startup" - "${DOCKER_PREFIX[@]}" ${DOCKER_CMD} logs "${TRAEFIK_CONTAINER_NAME}" || true + log_error "Edge exited during startup" + "${DOCKER_PREFIX[@]}" ${DOCKER_CMD} logs "${NODE_CONTAINER_NAME}" || true return 1 fi if [[ ${i} -eq ${retries} ]]; then - log_error "Traefik gateway did not become ready" + log_error "Edge gateway did not become ready" return 1 fi sleep ${delay} @@ -504,21 +442,17 @@ wait_for_gateway() { # Show status show_status() { local node_ip="$1" - local traefik_ip="$2" echo "" log_info "Container status:" "${DOCKER_PREFIX[@]}" ${DOCKER_CMD} ps -a \ - --filter "name=${NODE_CONTAINER_NAME}" \ - --filter "name=${TRAEFIK_CONTAINER_NAME}" + --filter "name=${NODE_CONTAINER_NAME}" echo "" log_info "Useful commands:" echo " AKernel logs: ${DOCKER_CMD} logs -f ${NODE_CONTAINER_NAME}" - echo " Traefik logs: ${DOCKER_CMD} logs -f ${TRAEFIK_CONTAINER_NAME}" echo " Enter AKernel: ${DOCKER_CMD} exec -it ${NODE_CONTAINER_NAME} bash" echo " AKernel IP: ${node_ip}" - echo " Traefik IP: ${traefik_ip}" echo " SDK token: ${TOKEN_FILE}" } @@ -527,7 +461,6 @@ check_prerequisites cleanup_existing configure_auth ensure_image "${IMAGE}" -ensure_image "${TRAEFIK_IMAGE}" configure_container_proxy configure_gpu configure_network @@ -539,18 +472,9 @@ if [[ -z "${NODE_IP}" ]]; then log_error "Could not determine the AKernel container IP" exit 1 fi -write_traefik_config "${NODE_IP}" -TRAEFIK_PROVIDER_ENDPOINT="http://${NODE_IP}:22770/global-scheduler/traefik/config" -log_info "Using FunctionMaster route provider: ${TRAEFIK_PROVIDER_ENDPOINT}" -start_traefik_container "${TRAEFIK_PROVIDER_ENDPOINT}" -TRAEFIK_IP="$(container_ip "${TRAEFIK_CONTAINER_NAME}")" -if [[ -z "${TRAEFIK_IP}" ]]; then - log_error "Could not determine the Traefik container IP" - exit 1 -fi -wait_for_gateway "${TRAEFIK_IP}" -show_status "${NODE_IP}" "${TRAEFIK_IP}" +wait_for_gateway "${NODE_IP}" +show_status "${NODE_IP}" log_info "AKernel started successfully in standalone mode" -log_info "Set AKERNEL_SERVER_ADDRESS=${TRAEFIK_IP}" +log_info "Set AKERNEL_SERVER_ADDRESS=${NODE_IP}" log_info "Set AKERNEL_TOKEN=\$(cat ${TOKEN_FILE})" diff --git a/deploy/standalone/stop.sh b/deploy/standalone/stop.sh index eb19800..38f2781 100755 --- a/deploy/standalone/stop.sh +++ b/deploy/standalone/stop.sh @@ -7,7 +7,11 @@ set -e -CONTAINER_NAMES=("akernel-traefik" "akernel-node") +CONTAINER_NAMES=("${NODE_CONTAINER_NAME:-akernel-node}") +# Remove the gateway container left by earlier standalone releases on upgrade. +if [[ "${NODE_CONTAINER_NAME:-akernel-node}" == akernel-node ]]; then + CONTAINER_NAMES=("akernel-traefik" "${CONTAINER_NAMES[@]}") +fi # Container runtime command (docker or pouch) DOCKER_CMD="" @@ -52,8 +56,7 @@ else exit 1 fi -# Stop the gateway before the AKernel container so no new requests arrive -# while the runtime is shutting down. +# The Go CLI supervises Edge, Node Proxy, and runtime shutdown. for container in "${CONTAINER_NAMES[@]}"; do if "${DOCKER_PREFIX[@]}" ${DOCKER_CMD} container inspect "${container}" &> /dev/null; then log_info "Stopping container: ${container}" diff --git a/deploy/terraform/aliyun/README.md b/deploy/terraform/aliyun/README.md index 228f6ff..d197a8f 100644 --- a/deploy/terraform/aliyun/README.md +++ b/deploy/terraform/aliyun/README.md @@ -179,28 +179,22 @@ image registry settings before applying the plan. ## Public endpoint model The default cloud deployment uses a split frontend plus a two-entrypoint -Traefik LoadBalancer: +Edge LoadBalancer: - `websecure:443` routes frontend API and exec websocket traffic over TLS. - `web:80` routes function port-forwarding traffic over plain HTTP/WS. -Use the Traefik LoadBalancer host or IP directly with the SDK: +Use the Edge LoadBalancer host or IP directly with the SDK: ```bash -export AKERNEL_SERVER_ADDRESS= +export AKERNEL_SERVER_ADDRESS= ``` -`traefik_tls_enabled` is only for mounting a custom default certificate. It is -not required for the `websecure` router on port 443; Traefik serves its default -certificate when the variable is `false`. - -To use the legacy single-entrypoint mode, set -`traefik_enable_web_entrypoint=false` and configure `traefik_tcp_port`. In that -mode SDK clients must include the port explicitly: - -```bash -export AKERNEL_SERVER_ADDRESS=: -``` +Edge uses the chart's component certificate by default. Set +`edge_tls_secret_name` for a dedicated TLS Secret, or also set +`edge_tls_create_secret`, `edge_tls_cert`, and `edge_tls_key` to create it. +Customize public ports with `edge_http_port` and `edge_https_port`; configure +the SDK API and gateway addresses explicitly when using custom ports. Enable OSS auth injection for AKernel node secret: diff --git a/deploy/terraform/aliyun/main.tf b/deploy/terraform/aliyun/main.tf index c23bf15..c2b8ed0 100644 --- a/deploy/terraform/aliyun/main.tf +++ b/deploy/terraform/aliyun/main.tf @@ -102,7 +102,7 @@ locals { slb_security_group_annotations = length(var.security_group_id) > 0 ? { "service.beta.kubernetes.io/alibaba-cloud-loadbalancer-security-group-ids" = var.security_group_id } : {} - effective_traefik_service_annotations = merge(local.slb_security_group_annotations, var.traefik_service_annotations) + effective_edge_service_annotations = merge(local.slb_security_group_annotations, var.edge_service_annotations) effective_storage_class = var.storage_class effective_monitor_storage_class = length(var.monitor_storage_class) > 0 ? var.monitor_storage_class : local.effective_storage_class @@ -128,11 +128,9 @@ locals { auths = { for host, cred in var.registry_auths : host => { auth = base64encode("${cred.username}:${cred.password}") } } } - etcd_image_repo = length(var.etcd_image_repository) > 0 ? var.etcd_image_repository : "public.ecr.aws/bitnami/etcd" - master_image_repo = length(var.master_image_repository) > 0 ? var.master_image_repository : "${local.acr_registry}/all-in-one" - node_image_repo = length(var.node_image_repository) > 0 ? var.node_image_repository : "${local.acr_registry}/all-in-one" - traefik_image_repo = length(var.traefik_image_repository) > 0 ? var.traefik_image_repository : "traefik" - traefik_internal_stats_image = length(var.traefik_internal_stats_image) > 0 ? var.traefik_internal_stats_image : "${local.acr_registry}/busybox:1.37.0-musl" + etcd_image_repo = length(var.etcd_image_repository) > 0 ? var.etcd_image_repository : "public.ecr.aws/bitnami/etcd" + master_image_repo = length(var.master_image_repository) > 0 ? var.master_image_repository : "${local.acr_registry}/all-in-one" + node_image_repo = length(var.node_image_repository) > 0 ? var.node_image_repository : "${local.acr_registry}/all-in-one" core_values = templatefile("${path.module}/values-akernel.yaml.tmpl", { acr_registry = local.acr_registry @@ -146,12 +144,9 @@ locals { master_image_tag = var.master_image_tag node_image_repository = local.node_image_repo node_image_tag = var.node_image_tag - traefik_image_repository = local.traefik_image_repo - traefik_image_tag = var.traefik_image_tag iam_litebus_data_key = var.iam_litebus_data_key enable_kruise = var.install_prereqs - master_service_type = (var.master_public_access_8888 && !var.traefik_enabled) ? var.master_service_type : "ClusterIP" - traefik_enabled = var.traefik_enabled + master_service_type = var.master_public_access_8888 ? var.master_service_type : "ClusterIP" sandboxd_nat_backend = var.sandboxd_nat_backend enable_runc = var.enable_runc node_secret_create = var.node_secret_create @@ -189,23 +184,20 @@ locals { frontend_cpu = var.frontend_cpu frontend_memory = var.frontend_memory - install_traefik = var.install_traefik - traefik_replicas = var.traefik_replicas - traefik_tcp_port = var.traefik_tcp_port - traefik_enable_web_entrypoint = var.traefik_enable_web_entrypoint - traefik_web_port = var.traefik_web_port - traefik_websecure_port = var.traefik_websecure_port - traefik_service_type = var.traefik_service_type - traefik_service_annotations = local.effective_traefik_service_annotations - traefik_tls_enabled = var.traefik_tls_enabled - traefik_tls_create_secret = var.traefik_tls_create_secret - traefik_tls_cert = var.traefik_tls_cert - traefik_tls_key = var.traefik_tls_key - traefik_internal_stats = var.traefik_internal_stats_enabled - traefik_internal_stats_image = local.traefik_internal_stats_image - traefik_grafana_enabled = var.install_monitor - traefik_grafana_url = var.install_monitor ? "http://grafana.${var.monitor_namespace}.svc:3000" : "" - + edge_service_name = var.edge_service_name + edge_service_type = var.edge_service_type + edge_service_loadbalancer_ip = var.edge_service_loadbalancer_ip + edge_http_port = var.edge_http_port + edge_https_port = var.edge_https_port + edge_tls_secret_name = var.edge_tls_secret_name + edge_tls_create_secret = var.edge_tls_create_secret + edge_tls_cert = var.edge_tls_cert + edge_tls_key = var.edge_tls_key + edge_allowed_client_cidrs = var.edge_allowed_client_cidrs + node_proxy_allowed_target_cidrs = var.node_proxy_allowed_target_cidrs + node_proxy_allowed_edge_cidrs = var.node_proxy_allowed_edge_cidrs + edge_service_annotations = local.effective_edge_service_annotations + edge_grafana_url = var.install_monitor && !var.grafana_public_access ? "http://grafana.${var.monitor_namespace}.svc:3000" : "" }) monitor_image_registry = var.monitor_image_registry diff --git a/deploy/terraform/aliyun/terraform.tfvars.example b/deploy/terraform/aliyun/terraform.tfvars.example index eb411f9..113e2c9 100644 --- a/deploy/terraform/aliyun/terraform.tfvars.example +++ b/deploy/terraform/aliyun/terraform.tfvars.example @@ -97,8 +97,6 @@ core_namespace = "akernel" # master_image_tag = "" # node_image_repository = "my-registry.com/akernel/all-in-one" # node_image_tag = "" -# traefik_image_repository = "my-registry.com/traefik" -# traefik_image_tag = "v3.6.8" # Optional mirror registry for Grafana, Prometheus, Loki, Tempo, and BusyBox. # Leave empty to use the projects' official public images. @@ -139,28 +137,13 @@ core_namespace = "akernel" # Optional: master replicas (for HA when frontend is enabled). # master_replicas = 1 -# Frontend Deployment. Traefik routes API traffic to akernel-frontend when it is enabled. +# Frontend Deployment. Edge runs alongside akernel-frontend when it is enabled. frontend_enabled = true frontend_replicas = 1 frontend_cpu = "1" frontend_memory = "2Gi" -# Traefik ingress controller (deployed via core chart). -traefik_enabled = true -install_traefik = true -traefik_service_type = "LoadBalancer" -traefik_enable_web_entrypoint = true -traefik_websecure_port = 443 -traefik_web_port = 80 -# traefik_replicas = 1 -# Optional: mount a custom default certificate. The websecure router still -# serves TLS when this is false, using Traefik's default certificate. -traefik_tls_enabled = false -traefik_tls_create_secret = false -# traefik_tls_cert = "" -# traefik_tls_key = "" -# traefik_internal_stats_enabled = true -# traefik_internal_stats_image = "registry-vpc.cn-hangzhou.aliyuncs.com/my-namespace/busybox:1.37.0-musl" +# Edge ingress is deployed by the core chart (HTTP 80 and HTTPS 443). # Install prerequisite components (OpenKruise) before AKernel charts. install_prereqs = false @@ -266,3 +249,9 @@ dragonfly_server_node_pool = { # Example for using existing cluster instead of creating ACK: # create_cluster = false # kubeconfig_path = "/abs/path/to/kubeconfig" + +# Edge ingress and Node Proxy are enabled by default. +edge_service_type = "LoadBalancer" +# edge_tls_secret_name = "edge-tls" +# node_proxy_allowed_edge_cidrs = "10.0.0.0/8" +# node_proxy_allowed_target_cidrs = "172.16.0.0/12" diff --git a/deploy/terraform/aliyun/values-akernel.yaml.tmpl b/deploy/terraform/aliyun/values-akernel.yaml.tmpl index 0ce6d32..b2052b6 100644 --- a/deploy/terraform/aliyun/values-akernel.yaml.tmpl +++ b/deploy/terraform/aliyun/values-akernel.yaml.tmpl @@ -221,42 +221,23 @@ node: provider="kubernetes" sock_path="/var/run/resource.sock" -traefik: - enabled: ${install_traefik} - replicas: ${traefik_replicas} - enableWebEntrypoint: ${traefik_enable_web_entrypoint} - image: - repository: "${traefik_image_repository}" - tag: "${traefik_image_tag}" - ports: -%{ if traefik_enable_web_entrypoint ~} - web: ${traefik_web_port} - websecure: ${traefik_websecure_port} -%{ else ~} - tcp: ${traefik_tcp_port} -%{ endif ~} - service: - type: "${traefik_service_type}" -%{ if length(traefik_service_annotations) > 0 ~} - annotations: -%{ for k, v in traefik_service_annotations ~} - ${k}: "${v}" -%{ endfor ~} -%{ endif ~} - tls: - enabled: ${traefik_tls_enabled} - createSecret: ${traefik_tls_create_secret} -%{ if traefik_tls_cert != "" ~} - moduleCrt: ${jsonencode(traefik_tls_cert)} -%{ endif ~} -%{ if traefik_tls_key != "" ~} - moduleKey: ${jsonencode(traefik_tls_key)} -%{ endif ~} - internalStats: - enabled: ${traefik_internal_stats} - image: "${traefik_internal_stats_image}" - grafana: - enabled: ${traefik_grafana_enabled} -%{ if traefik_grafana_url != "" ~} - url: "${traefik_grafana_url}" -%{ endif ~} +dataPlane: + enabled: true + edge: + tlsSecretName: "${edge_tls_secret_name}" + tls: + createSecret: ${edge_tls_create_secret} + cert: ${jsonencode(edge_tls_cert)} + key: ${jsonencode(edge_tls_key)} + allowedClientCIDRs: "${edge_allowed_client_cidrs}" + grafanaURL: "${edge_grafana_url}" + service: + name: "${edge_service_name}" + type: "${edge_service_type}" + annotations: ${jsonencode(edge_service_annotations)} + loadBalancerIP: "${edge_service_loadbalancer_ip}" + httpPort: ${edge_http_port} + httpsPort: ${edge_https_port} + nodeProxy: + allowedTargetCIDRs: "${node_proxy_allowed_target_cidrs}" + allowedEdgeCIDRs: "${node_proxy_allowed_edge_cidrs}" diff --git a/deploy/terraform/aliyun/variables.tf b/deploy/terraform/aliyun/variables.tf index d390e09..b971c3f 100644 --- a/deploy/terraform/aliyun/variables.tf +++ b/deploy/terraform/aliyun/variables.tf @@ -408,6 +408,10 @@ variable "frontend_enabled" { type = bool description = "Whether to enable frontend Deployment (splits from master for independent scaling)." default = true + validation { + condition = var.frontend_enabled + error_message = "Edge ingress requires frontend_enabled=true." + } } variable "frontend_replicas" { @@ -440,18 +444,6 @@ variable "node_image_tag" { default = "" } -variable "traefik_image_repository" { - type = string - description = "Image repository for Traefik. Empty uses the official traefik image." - default = "" -} - -variable "traefik_image_tag" { - type = string - description = "Image tag for Traefik." - default = "v3.6.8" -} - variable "monitor_image_registry" { type = string description = "Optional mirror registry prefix for Grafana, Prometheus, Loki, Tempo, and BusyBox. Empty uses their official public images." @@ -508,102 +500,10 @@ variable "master_service_type" { variable "master_public_access_8888" { type = bool - description = "Whether to expose akernel-master port 8888 publicly. Ignored (forced ClusterIP) when traefik_enabled=true." - default = false -} - - -variable "traefik_enabled" { - type = bool - description = "Whether to deploy Traefik as the cluster ingress. When true, akernel-master is kept ClusterIP and Traefik exposes port 8888 via LoadBalancer." - default = true -} - -variable "install_traefik" { - type = bool - description = "Whether to enable Traefik ingress controller in the core chart." - default = true -} - -variable "traefik_replicas" { - type = number - description = "Number of Traefik replicas." - default = 1 -} - -variable "traefik_tcp_port" { - type = number - description = "TCP port for Traefik websecure entrypoint (legacy single-entrypoint mode; ignored when traefik_enable_web_entrypoint=true)." - default = 8888 -} - -variable "traefik_enable_web_entrypoint" { - type = bool - description = "Enable dual entrypoints: 'websecure' (TLS, frontend API) on traefik_websecure_port and 'web' (plain HTTP, port forwarding) on traefik_web_port. When false, falls back to legacy single-entrypoint mode using traefik_tcp_port." - default = true -} - -variable "traefik_web_port" { - type = number - description = "Port for Traefik 'web' (plain HTTP) entrypoint. Only used when traefik_enable_web_entrypoint=true." - default = 80 -} - -variable "traefik_websecure_port" { - type = number - description = "Port for Traefik 'websecure' (TLS) entrypoint. Only used when traefik_enable_web_entrypoint=true." - default = 443 -} - -variable "traefik_service_type" { - type = string - description = "Service type for Traefik. Use 'LoadBalancer' for cloud deployments." - default = "LoadBalancer" -} - -variable "traefik_tls_enabled" { - type = bool - description = "Whether to mount a custom default certificate for Traefik. The websecure router still serves TLS when this is false." - default = false -} - -variable "traefik_tls_cert" { - type = string - description = "TLS certificate content (PEM) for Traefik. Only used when traefik_tls_enabled=true and traefik_tls_create_secret=true." - default = "" - sensitive = true -} - -variable "traefik_tls_key" { - type = string - description = "TLS private key content (PEM) for Traefik. Only used when traefik_tls_enabled=true and traefik_tls_create_secret=true." - default = "" - sensitive = true -} - -variable "traefik_tls_create_secret" { - type = bool - description = "Whether to create TLS secret from traefik_tls_cert/traefik_tls_key." - default = false -} - -variable "traefik_service_annotations" { - type = map(string) - description = "Extra annotations for Traefik LoadBalancer Service." - default = {} -} - -variable "traefik_internal_stats_enabled" { - type = bool - description = "Whether to enable the /internal-stats endpoint on Traefik." + description = "Whether to expose akernel-master port 8888 publicly. Defaults to internal access; SDK ingress is provided by Edge." default = true } -variable "traefik_internal_stats_image" { - type = string - description = "Optional BusyBox image for the Traefik /internal-stats sidecar. Empty uses the deployment ACR VPC endpoint." - default = "" -} variable "monitor_namespace" { type = string @@ -955,3 +855,83 @@ variable "tempo_resources" { }) default = {} } + +variable "edge_service_name" { + type = string + description = "Name of the Edge ingress Service. Reuse the existing ingress Service name during migration." + default = "akernel-edge" +} + +variable "edge_service_type" { + type = string + description = "Kubernetes Service type for Edge ingress." + default = "LoadBalancer" +} + +variable "edge_service_loadbalancer_ip" { + type = string + description = "Optional existing ingress LoadBalancer IP." + default = "" +} + +variable "edge_http_port" { + type = number + description = "External HTTP and WS sandbox-port Service port." + default = 80 +} + +variable "edge_https_port" { + type = number + description = "External HTTPS and WSS API Service port." + default = 443 +} + +variable "edge_tls_secret_name" { + type = string + description = "TLS Secret containing tls.crt and tls.key. Empty uses the component certificate." + default = "" +} + +variable "edge_tls_create_secret" { + type = bool + description = "Create the named TLS Secret from edge_tls_cert and edge_tls_key." + default = false +} + +variable "edge_tls_cert" { + type = string + description = "PEM certificate for the Edge TLS Secret." + default = "" + sensitive = true +} + +variable "edge_tls_key" { + type = string + description = "PEM private key for the Edge TLS Secret." + default = "" + sensitive = true +} + +variable "edge_allowed_client_cidrs" { + type = string + description = "Comma-separated client CIDRs allowed to reach Edge." + default = "0.0.0.0/0" +} + +variable "node_proxy_allowed_target_cidrs" { + type = string + description = "Comma-separated sandbox target CIDRs allowed by Node Proxy." + default = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" +} + +variable "node_proxy_allowed_edge_cidrs" { + type = string + description = "Comma-separated Edge Pod CIDRs allowed by Node Proxy." + default = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" +} + +variable "edge_service_annotations" { + type = map(string) + description = "Additional annotations for the Edge Service." + default = {} +} diff --git a/deploy/terraform/huaweicloud/README.md b/deploy/terraform/huaweicloud/README.md index b03ef62..0fc66fd 100644 --- a/deploy/terraform/huaweicloud/README.md +++ b/deploy/terraform/huaweicloud/README.md @@ -3,7 +3,7 @@ This Terraform module creates a Huawei Cloud CCE cluster and installs the AKernel core and optional monitor charts. It follows the same deployment contract as the Aliyun module: one all-in-one AKernel image, a generated IAM -seed, dual-entrypoint Traefik, optional public Grafana, and local state under +seed, Edge ingress and Node Proxy, optional public Grafana, and local state under `.akernel//`. ## Pod PID budget @@ -65,15 +65,15 @@ login is required. The two settings are mutually exclusive. The generated profile enables the public CCE API endpoint and node-subnet SNAT so Terraform can reach the cluster and worker nodes can pull public images. -Traefik is exposed through a public ELB with two entrypoints: +Edge is exposed through a public ELB with two entrypoints: - `websecure:443` serves the authenticated frontend API and exec websocket. - `web:80` serves sandbox port-forwarding traffic. -The SDK therefore needs only the Traefik ELB address: +The SDK therefore needs only the Edge ELB address: ```bash -export AKERNEL_SERVER_ADDRESS= +export AKERNEL_SERVER_ADDRESS= ``` Grafana uses a separate public ELB when monitoring and public Grafana access @@ -83,7 +83,7 @@ are enabled. Its generated administrator password is stored at ## Images Master, frontend, and node use the same configured AKernel all-in-one image. -etcd, Traefik, Grafana, Prometheus, Loki, Tempo, and BusyBox use pinned official +etcd, Grafana, Prometheus, Loki, Tempo, and BusyBox use pinned official public images by default. Use the component image variables or `monitor_image_registry` only when the cluster requires private mirrors. diff --git a/deploy/terraform/huaweicloud/main.tf b/deploy/terraform/huaweicloud/main.tf index da5e56b..5f0997f 100644 --- a/deploy/terraform/huaweicloud/main.tf +++ b/deploy/terraform/huaweicloud/main.tf @@ -55,11 +55,11 @@ locals { eip_type = var.master_elb_eip_type }) } : {} - huaweicloud_traefik_elb_annotations = var.traefik_public_access ? { + huaweicloud_edge_elb_annotations = var.edge_public_access ? { "kubernetes.io/elb.class" = "union" "kubernetes.io/elb.autocreate" = jsonencode({ type = "public" - bandwidth_name = "${var.cluster_name}-traefik-elb" + bandwidth_name = "${var.cluster_name}-edge-elb" bandwidth_chargemode = var.master_elb_bandwidth_charge_mode bandwidth_size = var.master_elb_bandwidth_size bandwidth_sharetype = "PER" @@ -104,8 +104,6 @@ locals { master_image_tag = var.master_image_tag node_image_repository = var.node_image_repository node_image_tag = var.node_image_tag - traefik_image_repository = var.traefik_image_repository - traefik_image_tag = var.traefik_image_tag iam_litebus_data_key = var.iam_litebus_data_key enable_kruise = var.install_prereqs master_service_type = var.master_public_access_8888 ? var.master_service_type : "ClusterIP" @@ -146,23 +144,21 @@ locals { frontend_cpu = var.frontend_cpu frontend_memory = var.frontend_memory - install_traefik = var.install_traefik - traefik_replicas = var.traefik_replicas - traefik_tcp_port = var.traefik_tcp_port - traefik_enable_web_entrypoint = var.traefik_enable_web_entrypoint - traefik_web_port = var.traefik_web_port - traefik_websecure_port = var.traefik_websecure_port - traefik_service_type = var.traefik_service_type - traefik_service_annotations = local.huaweicloud_traefik_elb_annotations - traefik_service_loadbalancer_ip = "" - traefik_tls_enabled = var.traefik_tls_enabled - traefik_tls_create_secret = var.traefik_tls_create_secret - traefik_tls_cert = var.traefik_tls_cert - traefik_tls_key = var.traefik_tls_key - traefik_internal_stats = var.traefik_internal_stats_enabled - traefik_internal_stats_image = var.traefik_internal_stats_image - traefik_grafana_enabled = var.install_monitor - traefik_grafana_url = var.install_monitor ? "http://grafana.${var.monitor_namespace}.svc:3000" : "" + edge_service_name = var.edge_service_name + edge_service_type = var.edge_service_type + edge_service_loadbalancer_ip = var.edge_service_loadbalancer_ip + edge_http_port = var.edge_http_port + edge_https_port = var.edge_https_port + edge_tls_secret_name = var.edge_tls_secret_name + edge_tls_create_secret = var.edge_tls_create_secret + edge_tls_cert = var.edge_tls_cert + edge_tls_key = var.edge_tls_key + edge_allowed_client_cidrs = var.edge_allowed_client_cidrs + node_proxy_allowed_target_cidrs = var.node_proxy_allowed_target_cidrs + node_proxy_allowed_edge_cidrs = var.node_proxy_allowed_edge_cidrs + edge_service_annotations = merge(local.huaweicloud_edge_elb_annotations, var.edge_service_annotations) + edge_grafana_url = var.install_monitor && !var.grafana_public_access ? "http://grafana.${var.monitor_namespace}.svc:3000" : "" + }) monitor_values = templatefile("${path.module}/values-monitor.yaml.tmpl", { diff --git a/deploy/terraform/huaweicloud/terraform.tfvars.example b/deploy/terraform/huaweicloud/terraform.tfvars.example index c0f9e1c..d3f6488 100644 --- a/deploy/terraform/huaweicloud/terraform.tfvars.example +++ b/deploy/terraform/huaweicloud/terraform.tfvars.example @@ -82,8 +82,6 @@ master_image_repository = "swr.cn-north-4.myhuaweicloud.com/akernel/all-in-one master_image_tag = "" node_image_repository = "swr.cn-north-4.myhuaweicloud.com/akernel/all-in-one" node_image_tag = "" -# traefik_image_repository = "traefik" -# traefik_image_tag = "v3.6.8" # Generate a unique seed for each deployment. Guided `make config` writes this # automatically; direct Terraform users can use `openssl rand -hex 32`. @@ -93,9 +91,6 @@ node_image_tag = "" # matching AKERNEL_ENABLE_RUNC=true profile setting. enable_runc = false -# Enable the /internal-stats endpoint sidecar on Traefik. -# traefik_internal_stats_enabled = true -# traefik_internal_stats_image = "my-registry.example.com/busybox:1.37.0-musl" # Optional mirror registry for Grafana, Prometheus, Loki, Tempo, and BusyBox. # Leave empty to use the projects' official public images. @@ -154,15 +149,12 @@ enable_runc = false # Optional: master replicas (for HA when frontend is enabled). # master_replicas = 1 -# Optional: enable frontend Deployment (splits from master for independent scaling). -# When enabled, Traefik routes API traffic to akernel-frontend instead of akernel-master. -# frontend_enabled = false +# Frontend runs Edge and scales independently from master. +# frontend_enabled = true # frontend_replicas = 2 # frontend_cpu = "4" # frontend_memory = "8Gi" -# Optional: traefik replicas. -# traefik_replicas = 1 # Restrict SSH CIDRs in production. allowed_ssh_cidrs = ["0.0.0.0/0"] @@ -195,3 +187,9 @@ prereq_kruise_chart_version = "1.8.3" # password = "" # } # } + +# Edge ingress and Node Proxy are enabled by default. +edge_service_type = "LoadBalancer" +# edge_tls_secret_name = "edge-tls" +# node_proxy_allowed_edge_cidrs = "10.0.0.0/8" +# node_proxy_allowed_target_cidrs = "172.16.0.0/12" diff --git a/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl b/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl index 89c3602..aae86a7 100644 --- a/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl +++ b/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl @@ -50,10 +50,9 @@ master: cpu: "${master_cpu}" memory: "${master_memory}" ephemeral-storage: "${master_ephemeral}" -%{ if frontend_enabled ~} - frontend: - enabled: true + enabled: ${frontend_enabled} +%{ if frontend_enabled ~} replicas: ${frontend_replicas} image: repository: "${master_image_repository}" @@ -199,38 +198,23 @@ node: provider="kubernetes" sock_path="/var/run/resource.sock" -traefik: - enabled: ${install_traefik} - replicas: ${traefik_replicas} - enableWebEntrypoint: ${traefik_enable_web_entrypoint} - image: - repository: "${traefik_image_repository}" - tag: "${traefik_image_tag}" - ports: -%{ if traefik_enable_web_entrypoint ~} - web: ${traefik_web_port} - websecure: ${traefik_websecure_port} -%{ else ~} - tcp: ${traefik_tcp_port} -%{ endif ~} - service: - type: "${traefik_service_type}" - annotations: ${jsonencode(traefik_service_annotations)} - loadBalancerIP: "${traefik_service_loadbalancer_ip}" - tls: - enabled: ${traefik_tls_enabled} - createSecret: ${traefik_tls_create_secret} -%{ if traefik_tls_cert != "" ~} - moduleCrt: ${jsonencode(traefik_tls_cert)} -%{ endif ~} -%{ if traefik_tls_key != "" ~} - moduleKey: ${jsonencode(traefik_tls_key)} -%{ endif ~} - internalStats: - enabled: ${traefik_internal_stats} - image: "${traefik_internal_stats_image}" - grafana: - enabled: ${traefik_grafana_enabled} -%{ if traefik_grafana_url != "" ~} - url: "${traefik_grafana_url}" -%{ endif ~} +dataPlane: + enabled: true + edge: + tlsSecretName: "${edge_tls_secret_name}" + tls: + createSecret: ${edge_tls_create_secret} + cert: ${jsonencode(edge_tls_cert)} + key: ${jsonencode(edge_tls_key)} + allowedClientCIDRs: "${edge_allowed_client_cidrs}" + grafanaURL: "${edge_grafana_url}" + service: + name: "${edge_service_name}" + type: "${edge_service_type}" + annotations: ${jsonencode(edge_service_annotations)} + loadBalancerIP: "${edge_service_loadbalancer_ip}" + httpPort: ${edge_http_port} + httpsPort: ${edge_https_port} + nodeProxy: + allowedTargetCIDRs: "${node_proxy_allowed_target_cidrs}" + allowedEdgeCIDRs: "${node_proxy_allowed_edge_cidrs}" diff --git a/deploy/terraform/huaweicloud/variables.tf b/deploy/terraform/huaweicloud/variables.tf index 5c668bb..2a45f4f 100644 --- a/deploy/terraform/huaweicloud/variables.tf +++ b/deploy/terraform/huaweicloud/variables.tf @@ -448,18 +448,6 @@ variable "node_image_tag" { default = "" } -variable "traefik_image_repository" { - type = string - description = "Image repository for Traefik." - default = "traefik" -} - -variable "traefik_image_tag" { - type = string - description = "Image tag for Traefik." - default = "v3.6.8" -} - variable "monitor_image_registry" { type = string description = "Optional mirror registry prefix for Grafana, Prometheus, Loki, Tempo, and BusyBox. Empty uses their official public images." @@ -560,7 +548,11 @@ variable "master_replicas" { variable "frontend_enabled" { type = bool description = "Whether to enable frontend Deployment (splits from master for independent scaling)." - default = false + default = true + validation { + condition = var.frontend_enabled + error_message = "Edge ingress requires frontend_enabled=true." + } } variable "frontend_replicas" { @@ -581,92 +573,6 @@ variable "frontend_memory" { default = "8Gi" } -variable "install_traefik" { - type = bool - description = "Whether to enable Traefik ingress controller in the core chart." - default = true -} - -variable "traefik_replicas" { - type = number - description = "Number of Traefik replicas." - default = 1 -} - -variable "traefik_tcp_port" { - type = number - description = "TCP port for Traefik websecure entrypoint (legacy single-entrypoint mode; ignored when traefik_enable_web_entrypoint=true)." - default = 8888 -} - -variable "traefik_enable_web_entrypoint" { - type = bool - description = "Enable dual entrypoints: 'websecure' (TLS, frontend API) on traefik_websecure_port and 'web' (plain HTTP, port forwarding) on traefik_web_port. When false, falls back to legacy single-entrypoint mode using traefik_tcp_port." - default = false -} - -variable "traefik_web_port" { - type = number - description = "Port for Traefik 'web' (plain HTTP) entrypoint. Only used when traefik_enable_web_entrypoint=true." - default = 80 -} - -variable "traefik_websecure_port" { - type = number - description = "Port for Traefik 'websecure' (TLS) entrypoint. Only used when traefik_enable_web_entrypoint=true." - default = 443 -} - -variable "traefik_service_type" { - type = string - description = "Service type for Traefik. Use 'LoadBalancer' for cloud deployments." - default = "LoadBalancer" -} - -variable "traefik_public_access" { - type = bool - description = "Whether Traefik receives a public Huawei Cloud LoadBalancer." - default = true -} - -variable "traefik_tls_enabled" { - type = bool - description = "Whether to enable TLS for Traefik." - default = false -} - -variable "traefik_tls_cert" { - type = string - description = "TLS certificate content (PEM) for Traefik. Only used when traefik_tls_enabled=true and traefik_tls_create_secret=true." - default = "" - sensitive = true -} - -variable "traefik_tls_key" { - type = string - description = "TLS private key content (PEM) for Traefik. Only used when traefik_tls_enabled=true and traefik_tls_create_secret=true." - default = "" - sensitive = true -} - -variable "traefik_tls_create_secret" { - type = bool - description = "Whether to create TLS secret from traefik_tls_cert/traefik_tls_key." - default = false -} - -variable "traefik_internal_stats_enabled" { - type = bool - description = "Whether to enable the /internal-stats endpoint on Traefik." - default = false -} - -variable "traefik_internal_stats_image" { - type = string - description = "BusyBox image for the Traefik /internal-stats sidecar." - default = "busybox:1.37.0-musl" -} - variable "monitor_namespace" { type = string description = "Namespace for monitor resources." @@ -989,3 +895,89 @@ variable "tempo_resources" { }) default = {} } + +variable "edge_service_name" { + type = string + description = "Name of the Edge ingress Service. Reuse the existing ingress Service name during migration." + default = "akernel-edge" +} + +variable "edge_service_type" { + type = string + description = "Kubernetes Service type for Edge ingress." + default = "LoadBalancer" +} + +variable "edge_service_loadbalancer_ip" { + type = string + description = "Optional existing ingress LoadBalancer IP." + default = "" +} + +variable "edge_http_port" { + type = number + description = "External HTTP and WS sandbox-port Service port." + default = 80 +} + +variable "edge_https_port" { + type = number + description = "External HTTPS and WSS API Service port." + default = 443 +} + +variable "edge_tls_secret_name" { + type = string + description = "TLS Secret containing tls.crt and tls.key. Empty uses the component certificate." + default = "" +} + +variable "edge_tls_create_secret" { + type = bool + description = "Create the named TLS Secret from edge_tls_cert and edge_tls_key." + default = false +} + +variable "edge_tls_cert" { + type = string + description = "PEM certificate for the Edge TLS Secret." + default = "" + sensitive = true +} + +variable "edge_tls_key" { + type = string + description = "PEM private key for the Edge TLS Secret." + default = "" + sensitive = true +} + +variable "edge_allowed_client_cidrs" { + type = string + description = "Comma-separated client CIDRs allowed to reach Edge." + default = "0.0.0.0/0" +} + +variable "node_proxy_allowed_target_cidrs" { + type = string + description = "Comma-separated sandbox target CIDRs allowed by Node Proxy." + default = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" +} + +variable "node_proxy_allowed_edge_cidrs" { + type = string + description = "Comma-separated Edge Pod CIDRs allowed by Node Proxy." + default = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" +} + +variable "edge_service_annotations" { + type = map(string) + description = "Additional annotations for the Edge Service." + default = {} +} + +variable "edge_public_access" { + type = bool + description = "Create a public Huawei Cloud ELB for Edge." + default = true +} diff --git a/sdk/python/README.md b/sdk/python/README.md index c18d5ae..1ba6c16 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -428,7 +428,7 @@ with Sandbox(port_forwardings=[8080]) as sandbox: ``` `get_port_url()` rejects undeclared ports. Pass `internal=True` only when a -deployment operator explicitly wants the direct Traefik address instead of the +deployment operator explicitly wants the direct Edge address instead of the public gateway. ## Local failover and reload @@ -483,7 +483,7 @@ with Sandbox(reverse_tunnel=tunnel) as sandbox: ) ``` -`reverse_port` carries the WebSocket tunnel through Traefik. `listen_port` is +`reverse_port` carries the WebSocket tunnel through Edge. `listen_port` is the loopback HTTP listener used inside the sandbox. Consequently, `sandbox.reverse_tunnel.url` is always `http://127.0.0.1:`, even when `target` uses HTTPS. diff --git a/sdk/python/akernel_sdk/_addresses.py b/sdk/python/akernel_sdk/_addresses.py index 6b6a375..3b4a1f9 100644 --- a/sdk/python/akernel_sdk/_addresses.py +++ b/sdk/python/akernel_sdk/_addresses.py @@ -121,7 +121,7 @@ def gateway_endpoint_from_env() -> Endpoint: """Return the public port-forwarding gateway endpoint. An explicit gateway override is parsed as plain HTTP by default because - standalone exposes Traefik's web entrypoint without TLS. Without an + standalone exposes Edge's HTTP listener without TLS. Without an explicit gateway, host-only server addresses use public 80, while host:port server addresses reuse the API port with plain HTTP. """ diff --git a/sdk/python/akernel_sdk/sandbox.py b/sdk/python/akernel_sdk/sandbox.py index d939611..cd9eb93 100644 --- a/sdk/python/akernel_sdk/sandbox.py +++ b/sdk/python/akernel_sdk/sandbox.py @@ -40,7 +40,7 @@ SandboxInfo, ) -_traefik_internal_ip_cache: str | None = None +_gateway_internal_address_cache: dict[tuple[str, Endpoint], tuple[str, int]] = {} logger = logging.getLogger(__name__) @@ -141,14 +141,14 @@ def _validate_integer( raise ValueError(f"{name} must be greater than or equal to {minimum}") -def _get_traefik_internal_ip(gateway: Endpoint) -> tuple[str, int]: - """Resolve Traefik's direct address for ``internal=True`` URLs.""" - - global _traefik_internal_ip_cache - if _traefik_internal_ip_cache is not None: - return _traefik_internal_ip_cache, gateway.port +def _get_gateway_internal_address(gateway: Endpoint) -> tuple[str, int]: + """Resolve the gateway's direct address for ``internal=True`` URLs.""" server = api_endpoint_from_env() + cache_key = (server.base_url(), gateway) + if cache_key in _gateway_internal_address_cache: + return _gateway_internal_address_cache[cache_key] + context = ssl.create_default_context() context.check_hostname = False context.verify_mode = ssl.CERT_NONE @@ -161,8 +161,13 @@ def _get_traefik_internal_ip(gateway: Endpoint) -> tuple[str, int]: pod_ip = payload.get("pod_ip") if not isinstance(pod_ip, str) or not pod_ip: raise RuntimeError("/internal-stats response does not contain pod_ip") - _traefik_internal_ip_cache = pod_ip - return pod_ip, gateway.port + port_key = "https_port" if gateway.use_tls else "http_port" + port = payload.get(port_key, gateway.port) + if isinstance(port, bool) or not isinstance(port, int) or not 1 <= port <= 65535: + raise RuntimeError(f"/internal-stats response contains invalid {port_key}") + address = (pod_ip, port) + _gateway_internal_address_cache[cache_key] = address + return address class Sandbox: @@ -538,7 +543,7 @@ def get_port_url(self, port: int, *, internal: bool = False) -> str: Args: port: Port included in ``port_forwardings`` at sandbox creation. - internal: Resolve Traefik's directly reachable address instead of + internal: Resolve the gateway's directly reachable address instead of the public gateway address. Raises: @@ -554,7 +559,7 @@ def get_port_url(self, port: int, *, internal: bool = False) -> str: gateway = gateway_endpoint_from_env() if internal: - pod_ip, gateway_port = _get_traefik_internal_ip(gateway) + pod_ip, gateway_port = _get_gateway_internal_address(gateway) direct = Endpoint( host=pod_ip, port=gateway_port, diff --git a/sdk/python/tests/unit/test_gateway_metadata.py b/sdk/python/tests/unit/test_gateway_metadata.py new file mode 100644 index 0000000..5bf4fd7 --- /dev/null +++ b/sdk/python/tests/unit/test_gateway_metadata.py @@ -0,0 +1,53 @@ +# Copyright (c) 2026 Ant Group Corporation. +# SPDX-License-Identifier: Apache-2.0 + +import json +import unittest +from unittest.mock import MagicMock, patch + +from akernel_sdk import sandbox +from akernel_sdk._addresses import Endpoint + + +class GatewayMetadataTest(unittest.TestCase): + def setUp(self): + sandbox._gateway_internal_address_cache.clear() + self.addCleanup(sandbox._gateway_internal_address_cache.clear) + + def resolve(self, payload, gateway): + response = MagicMock() + response.__enter__.return_value.read.return_value = json.dumps(payload).encode() + with patch.object(sandbox.urllib.request, "urlopen", return_value=response): + return sandbox._get_gateway_internal_address(gateway) + + @patch.object(sandbox, "api_endpoint_from_env", + return_value=Endpoint("public.example", 443, "https", False)) + def test_internal_ports_follow_listener_instead_of_service_port(self, _server): + payload = {"pod_ip": "10.0.0.4", "http_port": 8080, "https_port": 8443} + for scheme, expected in (("http", 8080), ("https", 8443)): + with self.subTest(scheme=scheme): + gateway = Endpoint("public.example", 10000, scheme, True) + self.assertEqual(self.resolve(payload, gateway), ("10.0.0.4", expected)) + + @patch.object(sandbox, "api_endpoint_from_env", + return_value=Endpoint("public.example", 443, "https", False)) + def test_legacy_metadata_uses_configured_gateway_port(self, _server): + gateway = Endpoint("public.example", 8888, "http", True) + self.assertEqual(self.resolve({"pod_ip": "10.0.0.4"}, gateway), + ("10.0.0.4", 8888)) + + @patch.object(sandbox, "api_endpoint_from_env") + def test_cache_is_scoped_to_api_endpoint(self, server): + gateway = Endpoint("public.example", 80, "http", False) + for index in (1, 2): + server.return_value = Endpoint(f"cluster{index}", 443, "https", False) + ip = f"10.0.0.{index}" + self.assertEqual(self.resolve({"pod_ip": ip}, gateway), (ip, 80)) + + @patch.object(sandbox, "api_endpoint_from_env", + return_value=Endpoint("public.example", 443, "https", False)) + def test_invalid_port_is_rejected(self, _server): + gateway = Endpoint("public.example", 80, "http", False) + for port in (0, 65536, True, "8080"): + with self.subTest(port=port), self.assertRaises(RuntimeError): + self.resolve({"pod_ip": "10.0.0.4", "http_port": port}, gateway) diff --git a/src/sandboxd b/src/sandboxd index b892414..b5d25ee 160000 --- a/src/sandboxd +++ b/src/sandboxd @@ -1 +1 @@ -Subproject commit b892414a3b21f3c2a4481322767362af14fc701f +Subproject commit b5d25eede172983cbfef9975e919219a7643e658