From c101d7360045bb5ed7d817a4343ce5a24382546f Mon Sep 17 00:00:00 2001 From: robbluo Date: Wed, 9 Sep 2026 17:44:29 +0800 Subject: [PATCH 1/2] feat(deploy): add Edge ingress and supervised shutdown Configure Edge and Node Proxy through the Go CLI, expose their Helm service and readiness settings, and discover the gateway for SDK exports. Keep deployment supervisors alive while the CLI performs ordered cleanup. Translate the shared image stop signal for the control plane and forward node termination through Bash with mixed service cleanup. Add regression coverage for signal forwarding, cleanup waiting, and exit-code propagation. Signed-off-by: robbluo --- AGENTS.md | 5 +- builder/scripts/akernel-entrypoint.sh | 31 ++++++++- builder/scripts/master_entrypoint.sh | 30 +++++++-- builder/scripts/yr_node_bootstrap.sh | 59 ++++++++++++++++- builder/systemd_services/yuanrong.service | 5 +- builder/tests/test_node_supervisor.py | 64 +++++++++++++++++++ deploy/README.md | 46 +++++++++++++ .../charts/core/templates/edge/service.yaml | 36 +++++++++++ .../templates/frontend/akernel_frontend.yaml | 45 +++++++++++-- .../core/templates/master/akernel_master.yaml | 2 + .../charts/core/templates/node/daemonset.yaml | 33 +++++++++- deploy/akernel/charts/core/values.yaml | 24 +++++++ deploy/scripts/print-sdk-env.sh | 12 +++- 13 files changed, 371 insertions(+), 21 deletions(-) create mode 100644 builder/tests/test_node_supervisor.py create mode 100644 deploy/akernel/charts/core/templates/edge/service.yaml diff --git a/AGENTS.md b/AGENTS.md index fca1d2a..9fe10a4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -175,7 +175,10 @@ source revisions are traceable through the AKernel commit's submodule gitlinks. Use [`deploy/README.md`](./deploy/README.md) as the deployment entry point. AKernel supports standalone, existing Kubernetes clusters via Helm, and -Terraform-based cloud provisioning. +Terraform-based cloud provisioning. The core chart can use Edge and Node Proxy +through `dataPlane.enabled` with an image containing the data-plane binaries; +configure its TLS Secret and allowed CIDRs and disable `traefik.enabled`. +See `deploy/README.md` for ingress migration and SDK endpoint configuration. Aliyun's aggregate Pod PID budget is configurable independently of the per-sandbox limit; see `deploy/terraform/aliyun/README.md#pod-pid-budget`. diff --git a/builder/scripts/akernel-entrypoint.sh b/builder/scripts/akernel-entrypoint.sh index 1df691c..6a4a8e5 100644 --- a/builder/scripts/akernel-entrypoint.sh +++ b/builder/scripts/akernel-entrypoint.sh @@ -28,7 +28,36 @@ fi case "${role}" in master|frontend) /usr/local/bin/ensure-component-cert - exec /bin/bash /home/yuanrong/entrypoint.sh "$@" + # The shared image uses systemd's stop signal for node/standalone. + # Translate it for the CLI and keep PID 1 alive until cleanup finishes. + child_pid="" + stop_requested=false + stop_control_plane() { + stop_requested=true + if [ -n "$child_pid" ]; then + kill -TERM "$child_pid" 2>/dev/null || true + fi + } + trap stop_control_plane TERM INT RTMIN+3 + /bin/bash /home/yuanrong/entrypoint.sh "$@" & + child_pid=$! + if [ "$stop_requested" = true ]; then + stop_control_plane + fi + # A trapped signal interrupts wait before the child has exited. + status=0 + while true; do + if wait "$child_pid"; then + status=0 + break + else + status=$? + fi + if ! kill -0 "$child_pid" 2>/dev/null; then + break + fi + done + exit "$status" ;; node) /bin/bash /root/prepare_node.sh diff --git a/builder/scripts/master_entrypoint.sh b/builder/scripts/master_entrypoint.sh index 862d1b6..eed7915 100644 --- a/builder/scripts/master_entrypoint.sh +++ b/builder/scripts/master_entrypoint.sh @@ -51,7 +51,7 @@ else fi # Set enable_traefik_provider based on TRAEFIK_MODE -if [ "${TRAEFIK_MODE:-etcd}" = "http" ]; then +if [ "${ENABLE_TRAEFIK:-true}" = "true" ] && [ "${TRAEFIK_MODE:-etcd}" = "http" ]; then ENABLE_TRAEFIK_PROVIDER=true else ENABLE_TRAEFIK_PROVIDER=false @@ -68,7 +68,29 @@ if [ ! -x "${YR_BIN}" ]; then exit 1 fi -exec "${YR_BIN}" start --master --block true \ +EDGE_ARGS=() +if [ "${ENABLE_EDGE_FRONTEND:-false}" = "true" ]; then + EDGE_ARGS=( + --enable_edge_frontend true + --edge_frontend_tls_bind "0.0.0.0:${EDGE_TLS_PORT:-8443}" + --edge_frontend_plain_bind "0.0.0.0:${EDGE_PLAIN_PORT:-8080}" + --edge_frontend_health_bind "0.0.0.0:${EDGE_HEALTH_PORT:-18080}" + --edge_frontend_tls_cert "${EDGE_TLS_CERT:-/etc/akernel-edge-tls/tls.crt}" + --edge_frontend_tls_key "${EDGE_TLS_KEY:-/etc/akernel-edge-tls/tls.key}" + --edge_frontend_control_plane_address "${INSTANCE_IP:?required for Edge}:8888" + --edge_frontend_iam_address 127.0.0.1:31113 + --edge_frontend_validate_iam true + --edge_frontend_allowed_client_cidrs "${EDGE_ALLOWED_CLIENT_CIDRS:?required for Edge}" + --data_plane_log_dir "${DATA_PLANE_LOG_DIR:-/var/log/akernel-edge}" + --data_plane_log_stdout true + --edge_frontend_access_log_enabled true + ) + # Edge terminates client TLS; its local Frontend and IAM hops use HTTP. + FRONTEND_SSL_ENABLE=false + IAM_SSL_ENABLE=false +fi + +exec "${YR_BIN}" start --master --block true "${EDGE_ARGS[@]}" \ -e -c 0 -m 8000 -s 4096 -n $HOSTNAME \ -d $DEPLOY_PATH \ --fs_health_check_retry_interval 1 \ @@ -79,7 +101,7 @@ exec "${YR_BIN}" start --master --block true \ --enable_iam_server ${ENABLE_IAM_SERVER:-true} \ --iam_token_expired_time_span 604800 \ --ssl_base_path=/home/yuanrong/.cert/ \ - --frontend_ssl_enable=true \ + --frontend_ssl_enable=${FRONTEND_SSL_ENABLE:-true} \ --frontend_client_auth_type NoClientCert \ --enable_function_token_auth ${ENABLE_FUNCTION_TOKEN_AUTH:-true} \ --enable_inherit_env false \ @@ -103,7 +125,7 @@ exec "${YR_BIN}" start --master --block true \ --traefik_enable_tls=${TRAEFIK_ENABLE_TLS:-false} \ --traefik_forward_timeout_ms=3000 \ --frontend_lease_bypass true \ - --iam_ssl_enable true \ + --iam_ssl_enable ${IAM_SSL_ENABLE:-true} \ --ssl_root_file ca.crt \ --ssl_cert_file module.crt \ --ssl_key_file module.key \ diff --git a/builder/scripts/yr_node_bootstrap.sh b/builder/scripts/yr_node_bootstrap.sh index 4b8b8e4..7ee6ece 100755 --- a/builder/scripts/yr_node_bootstrap.sh +++ b/builder/scripts/yr_node_bootstrap.sh @@ -46,7 +46,10 @@ CHECKPOINT_DIR="/home/akernel/checkpoints" mkdir -p "${CHECKPOINT_DIR}" # Select the legacy etcd registry or the FunctionMaster HTTP provider. -if [ "${TRAEFIK_MODE:-etcd}" = "etcd" ]; then +if [ "${ENABLE_TRAEFIK:-true}" != "true" ]; then + ENABLE_TRAEFIK_REGISTRY=false + ENABLE_TRAEFIK_PROVIDER=false +elif [ "${TRAEFIK_MODE:-etcd}" = "etcd" ]; then ENABLE_TRAEFIK_REGISTRY=${ENABLE_TRAEFIK_REGISTRY:-true} ENABLE_TRAEFIK_PROVIDER=false else @@ -54,6 +57,56 @@ else ENABLE_TRAEFIK_PROVIDER=true fi +NODE_PROXY_ARGS=() +if [ "${ENABLE_NODE_PROXY:-false}" = "true" ]; then + NODE_PROXY_ARGS=( + --enable_node_proxy true + --node_proxy_bind "0.0.0.0:${NODE_PROXY_PORT:-9443}" + --node_proxy_advertise_address "${YR_NODE_IP}:${NODE_PROXY_PORT:-9443}" + --node_proxy_health_bind "0.0.0.0:${NODE_PROXY_HEALTH_PORT:-18443}" + --node_proxy_security_mode network + --node_proxy_allowed_target_cidrs "${NODE_PROXY_ALLOWED_TARGET_CIDRS:?required for Node Proxy}" + --node_proxy_allowed_edge_cidrs "${NODE_PROXY_ALLOWED_EDGE_CIDRS:?required for Node Proxy}" + --data_plane_log_dir "${DATA_PLANE_LOG_DIR:-/var/log/akernel-edge}" + --data_plane_log_stdout true + ) +fi + +run_yuanrong() { + local child_pid="" + local stop_requested=false + local status=0 + stop_yuanrong() { + if [ "$stop_requested" = false ]; then + stop_requested=true + if [ -n "$child_pid" ]; then + kill -TERM "$child_pid" 2>/dev/null || true + fi + fi + } + trap stop_yuanrong TERM INT + "$@" & + child_pid=$! + if [ "$stop_requested" = true ]; then + kill -TERM "$child_pid" 2>/dev/null || true + fi + # A signal interrupts wait; keep the service MainPID alive until the CLI + # has finished its ordered shutdown, including sandbox cleanup. + while true; do + if wait "$child_pid"; then + status=0 + break + else + status=$? + fi + if ! kill -0 "$child_pid" 2>/dev/null; then + break + fi + done + trap - TERM INT + return "$status" +} + if [ "x${AKS_LOCAL_MODE}" == "xtrue" ]; then if [ -z "${LITEBUS_DATA_KEY:-}" ] && [ -r /home/akernel/iam-seed ]; then LITEBUS_DATA_KEY="$(tr -d '[:space:]' < /home/akernel/iam-seed)" @@ -63,7 +116,7 @@ if [ "x${AKS_LOCAL_MODE}" == "xtrue" ]; then echo "LITEBUS_DATA_KEY is required in standalone mode" >&2 exit 1 fi - /usr/bin/yr start --master \ + run_yuanrong /usr/bin/yr start --master "${NODE_PROXY_ARGS[@]}" \ --ip_address "${YR_NODE_IP}" \ --port_policy FIX \ --enable_function_scheduler=false \ @@ -115,7 +168,7 @@ if [ "x${AKS_LOCAL_MODE}" == "xtrue" ]; then --enable_sandbox_router true \ --enable_direct_routing false else - /usr/bin/yr start \ + run_yuanrong /usr/bin/yr start "${NODE_PROXY_ARGS[@]}" \ --ip_address "${YR_NODE_IP}" \ --port_policy FIX \ --ds_node_timeout_s 30 \ diff --git a/builder/systemd_services/yuanrong.service b/builder/systemd_services/yuanrong.service index 4572f5b..8f350ec 100644 --- a/builder/systemd_services/yuanrong.service +++ b/builder/systemd_services/yuanrong.service @@ -4,7 +4,7 @@ Description=yuanrong.service [Service] #Type=simple PIDFile=/run/yuanrong.pid -PassEnvironment=ETCD_PORT ETCD_PEER_PORT ETCD_ADDRESS HOSTNAME AKS_LOCAL_MODE AKERNEL_NODE_IP INSTANCE_IP LITEBUS_DATA_KEY YR_LOG_PATH YR_INSTALLATION_DIR YR_RRT_CONTROL_SOCKET_PATH YR_IMAGE_PROCESS_CONFIG ENABLE_METRICS ENABLE_TRACE TRAEFIK_MODE TRAEFIK_ENABLE_TLS TRAEFIK_HTTP_ENTRYPOINT +PassEnvironment=ENABLE_NODE_PROXY NODE_PROXY_PORT NODE_PROXY_HEALTH_PORT NODE_PROXY_ALLOWED_TARGET_CIDRS NODE_PROXY_ALLOWED_EDGE_CIDRS DATA_PLANE_LOG_DIR ENABLE_TRAEFIK ETCD_PORT ETCD_PEER_PORT ETCD_ADDRESS HOSTNAME AKS_LOCAL_MODE AKERNEL_NODE_IP INSTANCE_IP LITEBUS_DATA_KEY YR_LOG_PATH YR_INSTALLATION_DIR YR_RRT_CONTROL_SOCKET_PATH YR_IMAGE_PROCESS_CONFIG ENABLE_METRICS ENABLE_TRACE TRAEFIK_MODE TRAEFIK_ENABLE_TLS TRAEFIK_HTTP_ENTRYPOINT Environment="CONTAINER_EP=unix:///run/sandboxd/sandboxd.sock" Environment="RUNTIME_HOME_DIR=/home/yuanrong/runtime" Environment="YR_NOSET_CUDA_VISIBLE_DEVICES=1" @@ -12,7 +12,8 @@ ExecStartPre=-/bin/mkdir -p /home/yuanrong/runtime ExecStart=/usr/bin/bash /home/yuanrong/yr_node_bootstrap.sh ExecReload=/bin/kill -15 $MAINPID -KillMode=control-group +# The bootstrap forwards TERM and waits for the CLI's ordered cleanup. +KillMode=mixed Restart=always Delegate=yes UMask=000 diff --git a/builder/tests/test_node_supervisor.py b/builder/tests/test_node_supervisor.py new file mode 100644 index 0000000..4325306 --- /dev/null +++ b/builder/tests/test_node_supervisor.py @@ -0,0 +1,64 @@ +"""Exercise the real bootstrap supervisor with a blocked CLI cleanup.""" +import os +from pathlib import Path +import signal +import subprocess +import tempfile +import time +import unittest + + +class NodeSupervisorTest(unittest.TestCase): + def test_signal_waits_for_cleanup(self): + source = (Path(__file__).resolve().parents[1] / "scripts/yr_node_bootstrap.sh").read_text() + function = source.split("run_yuanrong() {", 1)[1].split('\nif [ "x${AKS_LOCAL_MODE}"', 1)[0] + function = "run_yuanrong() {" + function + for sig in (signal.SIGTERM, signal.SIGINT): + with self.subTest(signal=sig), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + child = root / "cli.sh" + child.write_text('''#!/bin/bash +cleanup() { + echo signal >> "$1/signals" + touch "$1/draining" + while [ ! -f "$1/release" ]; do sleep 0.02; done + touch "$1/cleaned" + exit 7 +} +trap 'cleanup "$1"' TERM +touch "$1/ready" +while :; do sleep 0.02; done +''') + supervisor = root / "supervisor.sh" + supervisor.write_text(function + '\nrun_yuanrong /bin/bash "$1" "$2"\n') + process = subprocess.Popen( + ["/bin/bash", str(supervisor), str(child), directory], start_new_session=True + ) + try: + self.wait_file(root / "ready", process) + process.send_signal(sig) + self.wait_file(root / "draining", process) + process.send_signal(sig) + time.sleep(0.15) + self.assertIsNone(process.poll(), "Bash exited before CLI cleanup") + (root / "release").touch() + self.assertEqual(process.wait(timeout=5), 7) + self.assertTrue((root / "cleaned").exists()) + self.assertEqual((root / "signals").read_text().splitlines(), ["signal"]) + finally: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=5) + + def wait_file(self, path, process): + deadline = time.monotonic() + 5 + while not path.exists(): + self.assertIsNone(process.poll(), "Supervisor exited while waiting for " + path.name) + self.assertLess(time.monotonic(), deadline) + time.sleep(0.01) + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/README.md b/deploy/README.md index 9ba6a48..f427e77 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -339,3 +339,49 @@ deploy/ ├── terraform/ # multi-cloud provisioning (aliyun, huaweicloud, shared) └── scripts/ # deployment and image helper scripts ``` + +## Edge and Node Proxy ingress + +Node shutdown is supervised by systemd. The YuanRong bootstrap forwards the +stop signal to the Go CLI and waits for its ordered cleanup before exiting; +sandboxd stops after YuanRong. The YuanRong service uses `KillMode=mixed` so +the bootstrap controls the initial shutdown of its child processes. + +Images containing the YuanRong data-plane package can run Edge alongside the +Frontend and Node Proxy alongside each node through the Go CLI. Set +`dataPlane.enabled=true` and `traefik.enabled=false` in the core chart. Supply an +existing TLS Secret (`tls.crt` and `tls.key`) as +`dataPlane.edge.tlsSecretName`, and configure the allowed client, Edge pod and +sandbox destination CIDRs explicitly: + +```yaml +dataPlane: + enabled: true + edge: + tlsSecretName: akernel-edge-tls + allowedClientCIDRs: "0.0.0.0/0" + service: + name: akernel-edge + type: LoadBalancer + nodeProxy: + allowedEdgeCIDRs: "192.168.0.0/16" + allowedTargetCIDRs: "10.88.0.0/16" +traefik: + enabled: false +``` + +Replace the CIDRs with those of the deployment. Edge exposes HTTPS/WSS on +service port 443 and HTTP/WS on port 80; API and authenticated direct routes +use TLS. The local Frontend and IAM hops use HTTP with IAM validation enabled. +Node Proxy uses network security mode and only accepts connections from the +configured Edge CIDRs. Node readiness includes Node Proxy; nodes roll one at +a time. Frontend readiness includes Edge route readiness. + +During migration, `dataPlane.edge.service.name` can match the existing ingress +Service name to retain its LoadBalancer identity and public address. Transfer +its annotations, clusterIP and loadBalancerIP as well. The Service then selects +Frontend/Edge pods; the Traefik Deployment and configuration are removed. +Existing SDK API and gateway address settings select TLS or plain WebSocket. + +`make print-env` discovers the Service labeled `app.kubernetes.io/component=edge`. +Set `AKERNEL_GATEWAY_SERVICE` to select a specific gateway Service. diff --git a/deploy/akernel/charts/core/templates/edge/service.yaml b/deploy/akernel/charts/core/templates/edge/service.yaml new file mode 100644 index 0000000..8801ff1 --- /dev/null +++ b/deploy/akernel/charts/core/templates/edge/service.yaml @@ -0,0 +1,36 @@ +{{- if .Values.dataPlane.enabled }} +{{- if not .Values.frontend.enabled }} +{{- fail "dataPlane.enabled requires frontend.enabled" }} +{{- end }} +{{- if .Values.traefik.enabled }} +{{- fail "Disable traefik.enabled when dataPlane.enabled is true" }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.dataPlane.edge.service.name }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: edge + {{- with .Values.dataPlane.edge.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.dataPlane.edge.service.type }} + {{- with .Values.dataPlane.edge.service.clusterIP }} + clusterIP: {{ . }} + {{- end }} + {{- with .Values.dataPlane.edge.service.loadBalancerIP }} + loadBalancerIP: {{ . | quote }} + {{- end }} + ports: + - name: websecure + port: {{ .Values.dataPlane.edge.service.httpsPort }} + targetPort: websecure + - name: web + port: {{ .Values.dataPlane.edge.service.httpPort }} + targetPort: web + selector: + app: akernel-frontend +{{- end }} diff --git a/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml b/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml index 2b02967..a984ecf 100644 --- a/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml +++ b/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml @@ -87,6 +87,24 @@ spec: command: - /usr/local/bin/akernel-entrypoint env: + {{- if .Values.dataPlane.enabled }} + - name: ENABLE_EDGE_FRONTEND + value: "true" + - name: INSTANCE_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: EDGE_TLS_PORT + value: {{ .Values.dataPlane.edge.tlsPort | quote }} + - name: EDGE_PLAIN_PORT + value: {{ .Values.dataPlane.edge.plainPort | quote }} + - name: EDGE_HEALTH_PORT + value: {{ .Values.dataPlane.edge.healthPort | quote }} + - name: EDGE_ALLOWED_CLIENT_CIDRS + value: {{ required "dataPlane.edge.allowedClientCIDRs is required" .Values.dataPlane.edge.allowedClientCIDRs | quote }} + {{- end }} + - name: ENABLE_TRAEFIK + value: {{ and .Values.traefik.enabled (not .Values.dataPlane.enabled) | quote }} - name: AKERNEL_ROLE value: "frontend" - name: LITEBUS_DATA_KEY @@ -129,22 +147,30 @@ spec: - name: ENABLE_TRACE value: {{ ne (.Values.monitoring.tempoEndpoint | default "") "" | ternary "true" "false" | quote }} ports: + {{- if .Values.dataPlane.enabled }} + - name: websecure + containerPort: {{ .Values.dataPlane.edge.tlsPort }} + - name: web + containerPort: {{ .Values.dataPlane.edge.plainPort }} + - name: edge-health + containerPort: {{ .Values.dataPlane.edge.healthPort }} + {{- end }} - name: http containerPort: 8888 livenessProbe: httpGet: path: /healthz port: 8888 - scheme: HTTPS + scheme: {{ .Values.dataPlane.enabled | ternary "HTTP" "HTTPS" }} initialDelaySeconds: 15 periodSeconds: 20 timeoutSeconds: 5 failureThreshold: 3 readinessProbe: httpGet: - path: /healthz - port: 8888 - scheme: HTTPS + path: {{ .Values.dataPlane.enabled | ternary "/readyz" "/healthz" }} + port: {{ .Values.dataPlane.enabled | ternary (.Values.dataPlane.edge.healthPort | int) 8888 }} + scheme: {{ .Values.dataPlane.enabled | ternary "HTTP" "HTTPS" }} initialDelaySeconds: 5 periodSeconds: 10 timeoutSeconds: 3 @@ -152,6 +178,11 @@ spec: resources: {{- toYaml .Values.frontend.resources | nindent 12 }} volumeMounts: + {{- if .Values.dataPlane.enabled }} + - name: edge-tls + mountPath: /etc/akernel-edge-tls + readOnly: true + {{- end }} - name: component-tls mountPath: /home/yuanrong/.cert readOnly: true @@ -161,6 +192,12 @@ spec: readOnly: true {{- end }} volumes: + {{- if .Values.dataPlane.enabled }} + - name: edge-tls + secret: + secretName: {{ .Values.dataPlane.edge.tlsSecretName }} + defaultMode: 0400 + {{- end }} - name: component-tls secret: secretName: {{ include "core.componentTLSSecretName" . }} diff --git a/deploy/akernel/charts/core/templates/master/akernel_master.yaml b/deploy/akernel/charts/core/templates/master/akernel_master.yaml index de37591..0c4023c 100644 --- a/deploy/akernel/charts/core/templates/master/akernel_master.yaml +++ b/deploy/akernel/charts/core/templates/master/akernel_master.yaml @@ -77,6 +77,8 @@ spec: command: - /usr/local/bin/akernel-entrypoint env: + - name: ENABLE_TRAEFIK + value: {{ and .Values.traefik.enabled (not .Values.dataPlane.enabled) | quote }} - name: AKERNEL_ROLE value: "master" {{- if .Values.frontend.enabled }} diff --git a/deploy/akernel/charts/core/templates/node/daemonset.yaml b/deploy/akernel/charts/core/templates/node/daemonset.yaml index 01eb11d..1ce2a77 100644 --- a/deploy/akernel/charts/core/templates/node/daemonset.yaml +++ b/deploy/akernel/charts/core/templates/node/daemonset.yaml @@ -21,11 +21,11 @@ spec: {{- if .Values.kruise.enabled }} rollingUpdate: rollingUpdateType: Standard - maxUnavailable: 200 + maxUnavailable: 1 partition: 0 {{- else }} rollingUpdate: - maxUnavailable: 200 + maxUnavailable: 1 {{- end }} type: RollingUpdate selector: @@ -65,6 +65,15 @@ spec: imagePullPolicy: {{ include "core.image.pullPolicy" (dict "root" . "image" .Values.node.image) }} command: - /usr/local/bin/akernel-entrypoint + {{- if .Values.dataPlane.enabled }} + readinessProbe: + httpGet: + path: /readyz + port: {{ .Values.dataPlane.nodeProxy.healthPort }} + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + {{- end }} lifecycle: postStart: exec: @@ -72,10 +81,28 @@ spec: - bash - -c - | - cp /etc/resolv.conf /etc/resolv_akernel.conf && sed -i 's/127.0.0.1/10.88.0.1/g' /etc/resolv_akernel.conf + sandbox_resolver_ip={{ first (splitList "/" (.Values.node.sandboxIPRange | default "10.88.0.1/16")) | quote }} + awk -v resolver="${sandbox_resolver_ip}" ' + $1 == "nameserver" && $2 ~ /^127\./ { $2 = resolver } + { print } + ' /etc/resolv.conf > /etc/resolv_akernel.conf resources: {{- toYaml .Values.node.resources | nindent 10 }} env: + {{- if .Values.dataPlane.enabled }} + - name: ENABLE_NODE_PROXY + value: "true" + - name: NODE_PROXY_PORT + value: {{ .Values.dataPlane.nodeProxy.port | quote }} + - name: NODE_PROXY_HEALTH_PORT + value: {{ .Values.dataPlane.nodeProxy.healthPort | quote }} + - name: NODE_PROXY_ALLOWED_TARGET_CIDRS + value: {{ required "dataPlane.nodeProxy.allowedTargetCIDRs is required" .Values.dataPlane.nodeProxy.allowedTargetCIDRs | quote }} + - name: NODE_PROXY_ALLOWED_EDGE_CIDRS + value: {{ required "dataPlane.nodeProxy.allowedEdgeCIDRs is required" .Values.dataPlane.nodeProxy.allowedEdgeCIDRs | quote }} + {{- end }} + - name: ENABLE_TRAEFIK + value: {{ and .Values.traefik.enabled (not .Values.dataPlane.enabled) | quote }} - name: AKERNEL_ROLE value: "node" - name: RUNSC_AKERNEL diff --git a/deploy/akernel/charts/core/values.yaml b/deploy/akernel/charts/core/values.yaml index ab638fd..2e59a96 100644 --- a/deploy/akernel/charts/core/values.yaml +++ b/deploy/akernel/charts/core/values.yaml @@ -569,3 +569,27 @@ traefik: grafana: enabled: false url: "http://grafana.akernel-monitor.svc:3000" + +# Edge handles API, direct, tunnel and published-port ingress. It is started +# with Frontend by the Go CLI; Node Proxy runs in each node network namespace. +dataPlane: + enabled: false + edge: + tlsPort: 8443 + plainPort: 8080 + healthPort: 18080 + tlsSecretName: akernel-edge-tls + allowedClientCIDRs: "" + service: + name: akernel-edge + type: LoadBalancer + annotations: {} + clusterIP: "" + loadBalancerIP: "" + httpsPort: 443 + httpPort: 80 + nodeProxy: + port: 9443 + healthPort: 18443 + allowedTargetCIDRs: "" + allowedEdgeCIDRs: "" diff --git a/deploy/scripts/print-sdk-env.sh b/deploy/scripts/print-sdk-env.sh index da184ef..fb6b279 100755 --- a/deploy/scripts/print-sdk-env.sh +++ b/deploy/scripts/print-sdk-env.sh @@ -52,14 +52,20 @@ get_lb_host() { printf '%s' "${host}" } -traefik_host="$(get_lb_host "${core_ns}" traefik)" -[[ -n "${traefik_host}" ]] || die "traefik LoadBalancer address is not ready" +gateway_service="${AKERNEL_GATEWAY_SERVICE:-}" +if [[ -z "${gateway_service}" ]]; then + gateway_service="$(kubectl --kubeconfig "${kubeconfig}" -n "${core_ns}" get svc \ + -l app.kubernetes.io/component=edge -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" +fi +gateway_service="${gateway_service:-traefik}" +gateway_host="$(get_lb_host "${core_ns}" "${gateway_service}")" +[[ -n "${gateway_host}" ]] || die "${gateway_service} LoadBalancer address is not ready" token="$("${AKERNEL_REPO_ROOT}/deploy/scripts/generate-token.py" --env "${env_name}" --write-file "${dir}/token")" sdk_env="${dir}/sdk.env" { - printf 'export AKERNEL_SERVER_ADDRESS=%q\n' "${traefik_host}" + printf 'export AKERNEL_SERVER_ADDRESS=%q\n' "${gateway_host}" printf 'export AKERNEL_TOKEN=%q\n' "${token}" } > "${sdk_env}" chmod 600 "${sdk_env}" From 9f0b2ebf5bb82ff59d7bba30d438d2f3ef940264 Mon Sep 17 00:00:00 2001 From: robbluo Date: Wed, 9 Sep 2026 20:01:49 +0800 Subject: [PATCH 2/2] feat(deploy): replace Traefik with Edge and Node Proxy Make the Edge ingress the default for standalone, Helm and cloud profiles, and remove the Traefik deployment, routing and image configuration. Package the matching Core and data-plane binaries while preserving the RRT and SDK command protocol. Update sandboxd to main for its Start response IP. Serve gateway metadata and optional Grafana routes through Edge, resolve internal SDK ports from the gateway listeners, and update CI and deployment documentation for the single-container standalone topology. Signed-off-by: robbluo --- .dockerignore | 2 +- .github/workflows/ci.yml | 8 +- AGENTS.md | 20 +- assets/akernel-architecture.svg | 2 +- builder/node.Dockerfile | 52 ++++- builder/scripts/edge-config.sh | 48 +++++ builder/scripts/master_entrypoint.sh | 35 +--- builder/scripts/yr_node_bootstrap.sh | 36 +--- builder/systemd_services/yuanrong.service | 2 +- deploy/README.md | 72 +++---- .../charts/core/templates/edge/secret.yaml | 11 + .../charts/core/templates/edge/service.yaml | 3 - .../templates/frontend/akernel_frontend.yaml | 17 +- .../core/templates/master/akernel_master.yaml | 8 - .../charts/core/templates/node/daemonset.yaml | 8 - .../core/templates/traefik/configmap.yaml | 151 -------------- .../core/templates/traefik/deployment.yaml | 147 -------------- .../charts/core/templates/traefik/secret.yaml | 11 - .../core/templates/traefik/service.yaml | 39 ---- deploy/akernel/charts/core/values.yaml | 64 ++---- .../templates/prometheus/configmap.yaml | 8 +- deploy/akernel/charts/monitor/values.yaml | 2 +- deploy/scripts/configure.sh | 19 -- deploy/scripts/print-sdk-env.sh | 13 +- deploy/standalone/README.md | 51 ++--- deploy/standalone/start.sh | 116 ++--------- deploy/standalone/stop.sh | 9 +- deploy/terraform/aliyun/README.md | 22 +- deploy/terraform/aliyun/main.tf | 46 ++--- .../terraform/aliyun/terraform.tfvars.example | 27 +-- .../terraform/aliyun/values-akernel.yaml.tmpl | 59 ++---- deploy/terraform/aliyun/variables.tf | 190 ++++++++---------- deploy/terraform/huaweicloud/README.md | 10 +- deploy/terraform/huaweicloud/main.tf | 38 ++-- .../huaweicloud/terraform.tfvars.example | 18 +- .../huaweicloud/values-akernel.yaml.tmpl | 60 ++---- deploy/terraform/huaweicloud/variables.tf | 190 +++++++++--------- sdk/python/README.md | 4 +- sdk/python/akernel_sdk/_addresses.py | 2 +- sdk/python/akernel_sdk/sandbox.py | 27 ++- .../tests/unit/test_gateway_metadata.py | 53 +++++ src/sandboxd | 2 +- 42 files changed, 606 insertions(+), 1096 deletions(-) create mode 100644 builder/scripts/edge-config.sh create mode 100644 deploy/akernel/charts/core/templates/edge/secret.yaml delete mode 100644 deploy/akernel/charts/core/templates/traefik/configmap.yaml delete mode 100644 deploy/akernel/charts/core/templates/traefik/deployment.yaml delete mode 100644 deploy/akernel/charts/core/templates/traefik/secret.yaml delete mode 100644 deploy/akernel/charts/core/templates/traefik/service.yaml create mode 100644 sdk/python/tests/unit/test_gateway_metadata.py diff --git a/.dockerignore b/.dockerignore index 9550d54..58e47ae 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,6 @@ # Git metadata is not part of the image build. Component versions come from # their source manifests, and exact revisions remain traceable through the -# parent repository's submodule gitlinks. +# parent repository's submodule gitlinks or checksum-pinned source archives. .git **/.git diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5de3225..e9687fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -142,7 +142,7 @@ jobs: run: | gateway_ip="$(docker inspect \ --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' \ - akernel-traefik)" + akernel-node)" test -n "${gateway_ip}" token="$(cat deploy/standalone/data/token)" export AKERNEL_TOKEN="${token}" @@ -170,7 +170,7 @@ jobs: run: | gateway_ip="$(docker inspect \ --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' \ - akernel-traefik)" + akernel-node)" test -n "${gateway_ip}" token="$(cat deploy/standalone/data/token)" export AKERNEL_TOKEN="${token}" @@ -186,7 +186,7 @@ jobs: if: failure() run: | docker ps -a - for container in akernel-node akernel-traefik; do + for container in akernel-node; do if docker container inspect "${container}" >/dev/null 2>&1; then echo "=== ${container}: docker logs ===" docker logs --tail 500 "${container}" 2>&1 || true @@ -204,5 +204,5 @@ jobs: if [[ -x ./deploy/standalone/stop.sh ]]; then ./deploy/standalone/stop.sh else - docker rm -f akernel-traefik akernel-node >/dev/null 2>&1 || true + docker rm -f akernel-node >/dev/null 2>&1 || true fi diff --git a/AGENTS.md b/AGENTS.md index 9fe10a4..69b972d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -123,6 +123,14 @@ runtimes and `openyuanrong_sdk`. `builder/node.Dockerfile` then compiles the node components and produces the AKernel all-in-one image using the selected runtime image and its matching service configuration. +The image also builds Edge, Node Proxy and the forwarding helper from a +checksum-pinned YuanRong source archive with its Cargo lockfile; their binaries +are installed under `data_plane/bin` for the Go CLI. The default Core wheel +and data-plane source are pinned to the matching YuanRong `2b54c26885c6` +build so the Core package includes Node Proxy address +registration and Edge process scripts. The RRT binary and sandbox SDK retain +their matching `0.10.2rc2` command protocol. + The control-plane and RRT release version is independent of the optional actor-based `openyuanrong_sdk` installed in the Python runtime profile. This actor backend is deprecated and retained only for compatibility with existing @@ -175,9 +183,9 @@ source revisions are traceable through the AKernel commit's submodule gitlinks. Use [`deploy/README.md`](./deploy/README.md) as the deployment entry point. AKernel supports standalone, existing Kubernetes clusters via Helm, and -Terraform-based cloud provisioning. The core chart can use Edge and Node Proxy -through `dataPlane.enabled` with an image containing the data-plane binaries; -configure its TLS Secret and allowed CIDRs and disable `traefik.enabled`. +Terraform-based cloud provisioning. The core chart enables Edge and Node Proxy +by default with an image containing the data-plane binaries and Go CLI support; +configure its TLS Secret and allowed CIDRs. See `deploy/README.md` for ingress migration and SDK endpoint configuration. Aliyun's aggregate Pod PID budget is configurable independently of the @@ -383,13 +391,13 @@ export AKERNEL_SERVER_ADDRESS="" export AKERNEL_TOKEN="" ``` -When the public Traefik dual-entrypoint mode is enabled, a host/IP-only +With the default Edge ingress, a host/IP-only `AKERNEL_SERVER_ADDRESS` uses HTTPS/WSS on 443 for the frontend API and exec websocket, and HTTP on 80 for sandbox port URLs. For standalone deployments, -use the Traefik container IP printed by `deploy/standalone/start.sh`: +use the AKernel container IP printed by `deploy/standalone/start.sh`: ```bash -export AKERNEL_SERVER_ADDRESS= +export AKERNEL_SERVER_ADDRESS= ``` No separate `AKERNEL_GATEWAY_ADDRESS` is required for the default standalone diff --git a/assets/akernel-architecture.svg b/assets/akernel-architecture.svg index dd2d904..400bb2e 100644 --- a/assets/akernel-architecture.svg +++ b/assets/akernel-architecture.svg @@ -1,3 +1,3 @@ SDK/CLItraefikfrontend & schedulerlocal gatewaylocal gatewaysandboxdsandboxdsandboxsandboxgVisorsandboxsandboxgVisorOTel CollectorOTel CollectorGrafanaprometheuslokitempoOCI RegistryObject StoragePrivate K8s / Cloud VendorBrowser \ No newline at end of file + @font-face { font-family: "Lilita One"; src: url(data:font/woff2;base64,d09GMgABAAAAAAa4AA4AAAAACxQAAAZiAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGhYbgQAcGAZgAIEUEQgKi0CJQgsqAAE2AiQDUAQgBYMkByAb+QijopRuRsn+eRgbsx7b2ZmN2i1VDxbGuVyjXbC0oyZh2i53OblHQ/yu//k8eFk/yVDVqCnlLYS3NwurCwCHFl6+A6GJ0W3NYX7jvB0Qy2kgiyTRTn5uLtFtzw8IFfG3HQi5/A8H+PtL09RKmVgfT4QiHpFW/qbSvk1p1Bo1b2AJUAQzoUK1WjmwowZy40cB06Nb1CsHKc2Ec9UgAAgQVJIgQEDurgfwoGWPqCtuawAHDAD4JYja8pY+hlYSDJAamYMkqNkphLKnRIACEyywwQDn/7u4MZlHFgETbA6c/5/3yG/w5yTMdDR/HyX69ClYAiSx/CZiAuSeRgPG5h8k7nL1C0gEugTWSv6paQhG63izZwFQc9yBpcSvDmRoJOCTqYwCBYKLGdC9ZjUJCgLQnHgLEydZpmLlqtVp1DEA1DJRg5bfGZZddtpBu2yz0XLLLLbQAvMiEGgRIP6XGBwFlrVmJcRFIIMQrfFYoECEiNQppL1tZe9CJuFys4xlHDnNUXH1uJw4zOYwMs0NNaJeTxJPUx2TMDuVk87SXEqe0VkOqDsYFIpvk3f6I7pLuVbUdZ81aA05x3Re//s6i+weIVvja7lLniGXPvRvE8yFuttg0y0RTQJklqcl1ST05OgKLloWsKi/qovXyeOgtSj70jf0MaC7Mgw95pqucJhCh+jNmzMxLH9qg72mn7eJip9aCfDHbG69R72bdsPjQJHWipaQO82qJCseTxB+IoMvmY5J2UB+TNK/MDrYanRhx2jQrZVy1T/+PPo5RaDjoTm18XYv0Zyhj5BXAPwx3YZVJAkQOVkZ2gEMTon55fRI5CxXUfq2vKReHuMDqZpGPkkIt2sylv+M0LgavqXIIT5g/VOKHGM3+KgvgqyrWUVtAXECuJKwy67SrXw4Lb38toii/dmEovnHxQRg5Br6fuyvWKQuUB6pGUQt6NQbyHUJF+b8j/S7Ajl5M7Qk9ExlL+lrStmxmm/B/Z4CucbNLkf7LGWZBd8cRpdKxva+Ht5xcEBsmG//oNpTXWWY+Xece6G3k8bUndR4hps61j7Qfyt3kziEW+lh5pEG60j3Chnz1Bupb2TKyugUb4Oxu6UCgSpaYdXpkDp05c60dQeH4KdQXNgi5y2QnpCi+//61mEVJLHW56Q+6Xj9mqW8+buk+aYEI4k8xUhT4w1TB3e3DQhIet7plLxZEMuJFc8Xmv1OJmzBW8fM5Kgb4zO2J2/P79BksZkbpgWVpXNG8wJQW7MxiDPyeEp+3nxTqVWkaZjWU/C66WZxTGll1JLDXjylyNiSe1y/w+JdgtoZNlvFSapIFktsZtSndrjvZ29iM25sWX/7qO+9wwtUW+lZ3uVuf6wkq7joVz45NmL4+8GTTtSVxyzeFaFl0ZHvMbDHb4KBrY28X9/Y0wSPmZzt7Nfs7Sy/mDWRd088YnMrksBeKS5OJ9n4y2RHl5KccKWyVy003qeK3zzp6bqEglp7wWqBiasZzzkMRv996Rt1pXfUF/HHuOIyYEfzDrDBzsslWPEqiaWmLthhm1l0pLOF714rY1mSG0MktPRHyOZkZyetntirz7UttjZhRpWxUt82qmW5q0ouCO4KK/4akFxdUhvV6mSrXgsZU5BeSnFL3BQRslK+R7ZipSSYnyEdK42C1Vv+Gr65K214JYS3ihetNA0xuehnGvYNRyOL8wY0gfnB6iAzKs0j0XpAsdXYF9UPBsIiu6kJgP9nFEDJ4Lqf1ZIH8EGQo9tHuW+Zksc47K8nHP55DGnK9+tqyQGwlkmJX1j+qPwcAgCBIeUBH9cNi/2/cvjUCwBOO8SfIlPHM6d2vjkADkgAjYOaMxIHqHcA6XwPBILZFY4D2AOQAQDpC1Uu0FYH13qcY9PwSSZEpAeU98yXSeFwkteeFRQRUN5YijvE6Jogj4Z1XLBAAFBIwAIAEAhGgI1cFAgG91D3eK4moNCjJiFCnZpCMPtWBqxmQknmZDlvBcSrVqdam2JJGpTLVK5Fq2qNGtBcOHLmzFWsdsUaJIxCm0YMKu8NNvJFb4PRunks5cqZS/j9qcpValenWAvy58M0atKtRbVKVdrQrJWy2cBLX6ErrZvR7hoQBY856zXTcVCqioKjEjfVw4a9jjoPJ1eZLoZan93coVwZWuTGQEvUAvWg/jTTSydNUQj0/b+VKwAAAA==); }SDK/CLIedgefrontend & schedulerlocal gatewaylocal gatewaysandboxdsandboxdsandboxsandboxgVisorsandboxsandboxgVisorOTel CollectorOTel CollectorGrafanaprometheuslokitempoOCI RegistryObject StoragePrivate K8s / Cloud VendorBrowser \ No newline at end of file diff --git a/builder/node.Dockerfile b/builder/node.Dockerfile index 19669be..be84f62 100644 --- a/builder/node.Dockerfile +++ b/builder/node.Dockerfile @@ -10,12 +10,17 @@ ARG AKERNEL_ENABLE_RUNC=false ARG AKERNEL_ENABLE_FIRECRACKER=true ARG SANDBOXD_BUILD_IMAGE=golang:1.25.5-bookworm ARG DISTILL_FS_BUILD_IMAGE=rust:1.85.0-bookworm -ARG OPEN_YR_VERSION=0.10.2rc2 +ARG DATA_PLANE_BUILD_IMAGE=rust:1.95.0-bookworm +ARG DATA_PLANE_SOURCE_URL=https://codeload.github.com/openYuanrong-mirror/yuanrong/tar.gz/2b54c26885c67535e7b5b80608812f0d5ed33244 +ARG DATA_PLANE_SOURCE_SHA256=c57b633b3fe50802ec9fcd70cb1da24ac80bea54984ed8d5d2cff15e7781dcfb +ARG OPEN_YR_VERSION=0.7.0+2b54c26885c6 ARG OPEN_YR_CORE_WHEEL_URL= ARG OPEN_YR_CORE_WHEEL_SHA256= -ARG OPEN_YR_RELEASE_BASE_URL=https://openyuanrong.obs.cn-southwest-2.myhuaweicloud.com/release -ARG OPEN_YR_CORE_AMD64_SHA256=8cdefba9a415a7a35b6f39bf847e7fb933ad6ca55b27b7dec2f377ece47198c4 -ARG OPEN_YR_CORE_ARM64_SHA256=2e9d2d18922b87721fcc3e92fa959cdd2ce026a51b18512c7c71ab24ac6a6eaa +# Core includes the matching process scripts and FunctionProxy registration. +ARG OPEN_YR_CORE_AMD64_URL=https://openyuanrong.obs.cn-southwest-2.myhuaweicloud.com/daily/20260909023812/linux/amd64/openyuanrong_core-0.7.0%2B2b54c26885c6-py3-none-manylinux_2_31_x86_64.whl +ARG OPEN_YR_CORE_ARM64_URL=https://openyuanrong.obs.cn-southwest-2.myhuaweicloud.com/daily/20260909024016/linux/arm64/openyuanrong_core-0.7.0%2B2b54c26885c6-py3-none-manylinux_2_31_aarch64.whl +ARG OPEN_YR_CORE_AMD64_SHA256=b459e916e75d5ef6c59988ad7b4108fd15a19c8fd102f2ca7ab98b5e9fc7a09c +ARG OPEN_YR_CORE_ARM64_SHA256=0678dc164e371471665fa2468967c456055c5cbd2f24bb130384189f75fc7a2a ARG GVISOR_DOWNLOAD_IMAGE=ubuntu:24.04 ARG GVISOR_RELEASE ARG GVISOR_AMD64_URL @@ -38,6 +43,29 @@ ARG OTELCOL_CONTRIB_URL=https://github.com/open-telemetry/opentelemetry-collecto ARG AKERNEL_VERSION=unknown ARG AKERNEL_REVISION=unknown +# The data plane is packaged independently from the Core wheel. Build the +# checksum-pinned source with its lockfile and include it in every node image. +FROM ${DATA_PLANE_BUILD_IMAGE} AS data-plane-builder +ARG DATA_PLANE_SOURCE_URL +ARG DATA_PLANE_SOURCE_SHA256 +RUN apt-get update && \ + apt-get install -y --no-install-recommends ca-certificates curl protobuf-compiler && \ + rm -rf /var/lib/apt/lists/* +WORKDIR /src/data-plane-gateway +RUN set -eux; \ + curl -fSL --retry 5 --retry-delay 2 \ + "${DATA_PLANE_SOURCE_URL}" -o /tmp/data-plane-source.tar.gz; \ + echo "${DATA_PLANE_SOURCE_SHA256} /tmp/data-plane-source.tar.gz" | sha256sum -c -; \ + tar -xzf /tmp/data-plane-source.tar.gz --strip-components=2 \ + --wildcards '*/data-plane-gateway/*'; \ + rm /tmp/data-plane-source.tar.gz; \ + cargo build --locked --release --all-features --bins; \ + mkdir -p /output/bin; \ + for binary in yr-edge-frontend yr-node-proxy yr-data-plane-forward; do \ + install -m 0755 "target/release/${binary}" "/output/bin/${binary}"; \ + strip "/output/bin/${binary}"; \ + done + FROM ${GVISOR_DOWNLOAD_IMAGE} AS gvisor-runtime ARG GVISOR_RELEASE ARG GVISOR_AMD64_URL @@ -223,7 +251,8 @@ ARG AKERNEL_REVISION ARG OPEN_YR_VERSION ARG OPEN_YR_CORE_WHEEL_URL ARG OPEN_YR_CORE_WHEEL_SHA256 -ARG OPEN_YR_RELEASE_BASE_URL +ARG OPEN_YR_CORE_AMD64_URL +ARG OPEN_YR_CORE_ARM64_URL ARG OPEN_YR_CORE_AMD64_SHA256 ARG OPEN_YR_CORE_ARM64_SHA256 ARG GVISOR_RELEASE @@ -299,21 +328,21 @@ ENV YR_INSTALLATION_DIR=/home/yuanrong # Install the complete, language-runtime-free openYuanRong control plane from # its checksum-pinned core wheel. A URL and checksum pair may override the -# release asset when validating an unreleased daily build. +# selected artifact when validating another build. RUN set -eux; \ case "${TARGETARCH:-}" in \ - amd64) wheel_arch=x86_64; wheel_platform=amd64; release_sha="${OPEN_YR_CORE_AMD64_SHA256}" ;; \ - arm64) wheel_arch=aarch64; wheel_platform=arm64; release_sha="${OPEN_YR_CORE_ARM64_SHA256}" ;; \ + amd64) wheel_arch=x86_64; default_core_url="${OPEN_YR_CORE_AMD64_URL}"; release_sha="${OPEN_YR_CORE_AMD64_SHA256}" ;; \ + arm64) wheel_arch=aarch64; default_core_url="${OPEN_YR_CORE_ARM64_URL}"; release_sha="${OPEN_YR_CORE_ARM64_SHA256}" ;; \ "") \ case "$(uname -m)" in \ - x86_64) wheel_arch=x86_64; wheel_platform=amd64; release_sha="${OPEN_YR_CORE_AMD64_SHA256}" ;; \ - aarch64) wheel_arch=aarch64; wheel_platform=arm64; release_sha="${OPEN_YR_CORE_ARM64_SHA256}" ;; \ + x86_64) wheel_arch=x86_64; default_core_url="${OPEN_YR_CORE_AMD64_URL}"; release_sha="${OPEN_YR_CORE_AMD64_SHA256}" ;; \ + aarch64) wheel_arch=aarch64; default_core_url="${OPEN_YR_CORE_ARM64_URL}"; release_sha="${OPEN_YR_CORE_ARM64_SHA256}" ;; \ *) echo "unsupported openYuanRong target architecture: $(uname -m)" >&2; exit 1 ;; \ esac ;; \ *) echo "unsupported openYuanRong target architecture: ${TARGETARCH}" >&2; exit 1 ;; \ esac; \ wheel_name="openyuanrong_core-${OPEN_YR_VERSION}-py3-none-manylinux_2_31_${wheel_arch}.whl"; \ - wheel_url="${OPEN_YR_RELEASE_BASE_URL}/${OPEN_YR_VERSION}/linux/${wheel_platform}/${wheel_name}"; \ + wheel_url="${default_core_url}"; \ wheel_sha="${release_sha}"; \ if [ -n "${OPEN_YR_CORE_WHEEL_URL}" ]; then \ test -n "${OPEN_YR_CORE_WHEEL_SHA256}"; \ @@ -343,6 +372,7 @@ RUN set -eux; \ COPY --from=runtime-image /yr-runtime-rootfs.img ${YR_INSTALLATION_DIR}/yr-runtime-rootfs.img +COPY --from=data-plane-builder /output/ ${YR_INSTALLATION_DIR}/data_plane/ COPY --from=gvisor-runtime /gvisor/runsc /usr/local/bin/runsc COPY --from=sandboxd-builder /src/sandboxd/output/sandboxd /usr/local/bin/sandboxd COPY --from=sandboxd-builder /src/sandboxd/output/sbox /usr/local/bin/sbox diff --git a/builder/scripts/edge-config.sh b/builder/scripts/edge-config.sh new file mode 100644 index 0000000..3866ae5 --- /dev/null +++ b/builder/scripts/edge-config.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Copyright (c) 2026 Ant Group Corporation. +# SPDX-License-Identifier: Apache-2.0 + +configure_edge() { + EDGE_ARGS=() + if [ "${ENABLE_EDGE_FRONTEND:-false}" != true ]; then + return + fi + export EDGE_ADVERTISE_IP="${INSTANCE_IP:-${YR_NODE_IP:-127.0.0.1}}" + export YR_DATA_PLANE_EDGE_FRONTEND_PROXY_ROUTES_FILE=/run/akernel/edge-proxy-routes.json + mkdir -p /run/akernel/edge-http || return 1 + python3 - <<'PY' || return 1 +import json +import os +from pathlib import Path + +Path('/run/akernel/edge-http/internal-stats').write_text(json.dumps({ + 'pod_ip': os.environ['EDGE_ADVERTISE_IP'], + 'http_port': int(os.environ.get('EDGE_PLAIN_PORT', '80')), + 'https_port': int(os.environ.get('EDGE_TLS_PORT', '443')), +})) +routes = [{'name': 'internal-stats', 'path_prefix': '/internal-stats', + 'upstream': 'http://127.0.0.1:18081', 'strip_prefix': False}] +if os.environ.get('EDGE_GRAFANA_URL'): + routes.append({'name': 'grafana', 'path_prefix': '/grafana', + 'upstream': os.environ['EDGE_GRAFANA_URL'], 'strip_prefix': False}) +Path(os.environ['YR_DATA_PLANE_EDGE_FRONTEND_PROXY_ROUTES_FILE']).write_text(json.dumps(routes)) +PY + python3 -m http.server 18081 --bind 127.0.0.1 --directory /run/akernel/edge-http & + EDGE_ARGS=( + --enable_edge_frontend true + --edge_frontend_tls_bind "0.0.0.0:${EDGE_TLS_PORT:-443}" + --edge_frontend_plain_bind "0.0.0.0:${EDGE_PLAIN_PORT:-80}" + --edge_frontend_health_bind "0.0.0.0:${EDGE_HEALTH_PORT:-18080}" + --edge_frontend_tls_cert "${EDGE_TLS_CERT:-/home/yuanrong/.cert/module.crt}" + --edge_frontend_tls_key "${EDGE_TLS_KEY:-/home/yuanrong/.cert/module.key}" + --edge_frontend_control_plane_address "${EDGE_ADVERTISE_IP}:8888" + --edge_frontend_iam_address 127.0.0.1:31113 + --edge_frontend_validate_iam true + --edge_frontend_allowed_client_cidrs "${EDGE_ALLOWED_CLIENT_CIDRS:-0.0.0.0/0}" + --data_plane_log_dir "${DATA_PLANE_LOG_DIR:-/var/log/akernel-edge}" + --data_plane_log_stdout true + --edge_frontend_access_log_enabled true + ) + FRONTEND_SSL_ENABLE=false + IAM_SSL_ENABLE=false +} diff --git a/builder/scripts/master_entrypoint.sh b/builder/scripts/master_entrypoint.sh index eed7915..104d2bc 100644 --- a/builder/scripts/master_entrypoint.sh +++ b/builder/scripts/master_entrypoint.sh @@ -50,13 +50,6 @@ else echo "otelcol watchdog skipped" fi -# Set enable_traefik_provider based on TRAEFIK_MODE -if [ "${ENABLE_TRAEFIK:-true}" = "true" ] && [ "${TRAEFIK_MODE:-etcd}" = "http" ]; then - ENABLE_TRAEFIK_PROVIDER=true -else - ENABLE_TRAEFIK_PROVIDER=false -fi - if [ -z "${LITEBUS_DATA_KEY:-}" ]; then echo "LITEBUS_DATA_KEY is required for akernel master/frontend" >&2 exit 1 @@ -68,27 +61,8 @@ if [ ! -x "${YR_BIN}" ]; then exit 1 fi -EDGE_ARGS=() -if [ "${ENABLE_EDGE_FRONTEND:-false}" = "true" ]; then - EDGE_ARGS=( - --enable_edge_frontend true - --edge_frontend_tls_bind "0.0.0.0:${EDGE_TLS_PORT:-8443}" - --edge_frontend_plain_bind "0.0.0.0:${EDGE_PLAIN_PORT:-8080}" - --edge_frontend_health_bind "0.0.0.0:${EDGE_HEALTH_PORT:-18080}" - --edge_frontend_tls_cert "${EDGE_TLS_CERT:-/etc/akernel-edge-tls/tls.crt}" - --edge_frontend_tls_key "${EDGE_TLS_KEY:-/etc/akernel-edge-tls/tls.key}" - --edge_frontend_control_plane_address "${INSTANCE_IP:?required for Edge}:8888" - --edge_frontend_iam_address 127.0.0.1:31113 - --edge_frontend_validate_iam true - --edge_frontend_allowed_client_cidrs "${EDGE_ALLOWED_CLIENT_CIDRS:?required for Edge}" - --data_plane_log_dir "${DATA_PLANE_LOG_DIR:-/var/log/akernel-edge}" - --data_plane_log_stdout true - --edge_frontend_access_log_enabled true - ) - # Edge terminates client TLS; its local Frontend and IAM hops use HTTP. - FRONTEND_SSL_ENABLE=false - IAM_SSL_ENABLE=false -fi +. /root/edge-config.sh +configure_edge || exit 1 exec "${YR_BIN}" start --master --block true "${EDGE_ARGS[@]}" \ -e -c 0 -m 8000 -s 4096 -n $HOSTNAME \ @@ -120,10 +94,7 @@ exec "${YR_BIN}" start --master --block true "${EDGE_ARGS[@]}" \ --ds_rpc_thread_num 128 \ --function_proxy_merge_process_enable true \ --force_low_reliability_instance true \ - --enable_traefik_provider=${ENABLE_TRAEFIK_PROVIDER} \ - --traefik_http_entry_point=${TRAEFIK_HTTP_ENTRYPOINT:-websecure} \ - --traefik_enable_tls=${TRAEFIK_ENABLE_TLS:-false} \ - --traefik_forward_timeout_ms=3000 \ + --enable_traefik_provider=false \ --frontend_lease_bypass true \ --iam_ssl_enable ${IAM_SSL_ENABLE:-true} \ --ssl_root_file ca.crt \ diff --git a/builder/scripts/yr_node_bootstrap.sh b/builder/scripts/yr_node_bootstrap.sh index 7ee6ece..15fa567 100755 --- a/builder/scripts/yr_node_bootstrap.sh +++ b/builder/scripts/yr_node_bootstrap.sh @@ -45,17 +45,8 @@ echo "Using ${YR_NODE_IP} as the YuanRong node address" CHECKPOINT_DIR="/home/akernel/checkpoints" mkdir -p "${CHECKPOINT_DIR}" -# Select the legacy etcd registry or the FunctionMaster HTTP provider. -if [ "${ENABLE_TRAEFIK:-true}" != "true" ]; then - ENABLE_TRAEFIK_REGISTRY=false - ENABLE_TRAEFIK_PROVIDER=false -elif [ "${TRAEFIK_MODE:-etcd}" = "etcd" ]; then - ENABLE_TRAEFIK_REGISTRY=${ENABLE_TRAEFIK_REGISTRY:-true} - ENABLE_TRAEFIK_PROVIDER=false -else - ENABLE_TRAEFIK_REGISTRY=false - ENABLE_TRAEFIK_PROVIDER=true -fi +. /root/edge-config.sh +configure_edge || exit 1 NODE_PROXY_ARGS=() if [ "${ENABLE_NODE_PROXY:-false}" = "true" ]; then @@ -116,7 +107,7 @@ if [ "x${AKS_LOCAL_MODE}" == "xtrue" ]; then echo "LITEBUS_DATA_KEY is required in standalone mode" >&2 exit 1 fi - run_yuanrong /usr/bin/yr start --master "${NODE_PROXY_ARGS[@]}" \ + run_yuanrong /usr/bin/yr start --master "${NODE_PROXY_ARGS[@]}" "${EDGE_ARGS[@]}" \ --ip_address "${YR_NODE_IP}" \ --port_policy FIX \ --enable_function_scheduler=false \ @@ -125,7 +116,7 @@ if [ "x${AKS_LOCAL_MODE}" == "xtrue" ]; then --enable_iam_server=true \ --iam_token_expired_time_span 604800 \ --ssl_base_path=/home/yuanrong/.cert/ \ - --frontend_ssl_enable=true \ + --frontend_ssl_enable="${FRONTEND_SSL_ENABLE:-true}" \ --frontend_client_auth_type NoClientCert \ --enable_function_token_auth true \ --ds_node_timeout_s 30 \ @@ -140,13 +131,8 @@ if [ "x${AKS_LOCAL_MODE}" == "xtrue" ]; then --npu_collection_mode off \ --enable_distributed_master false \ --metrics_collector_type external \ - --enable_traefik_registry=${ENABLE_TRAEFIK_REGISTRY} \ - --enable_traefik_provider=${ENABLE_TRAEFIK_PROVIDER} \ - --traefik_enable_tls=${TRAEFIK_ENABLE_TLS:-false} \ - --traefik_etcd_prefix=traefik \ - --traefik_lease_ttl=300000 \ - --traefik_http_entrypoint=${TRAEFIK_HTTP_ENTRYPOINT:-websecure} \ - --traefik_http_entry_point=${TRAEFIK_HTTP_ENTRYPOINT:-websecure} \ + --enable_traefik_registry=false \ + --enable_traefik_provider=false \ --enable_metrics ${ENABLE_METRICS} \ --metrics_config_file "/home/yuanrong/metrics/metrics_config.json" \ --enable_trace ${ENABLE_TRACE} \ @@ -155,7 +141,7 @@ if [ "x${AKS_LOCAL_MODE}" == "xtrue" ]; then --function_proxy_merge_process_enable true \ --fc_agent_mgr_retry_times 30 \ --fc_agent_mgr_retry_cycle 60000 \ - --iam_ssl_enable true \ + --iam_ssl_enable "${IAM_SSL_ENABLE:-true}" \ --ssl_root_file ca.crt \ --ssl_cert_file module.crt \ --ssl_key_file module.key \ @@ -168,7 +154,7 @@ if [ "x${AKS_LOCAL_MODE}" == "xtrue" ]; then --enable_sandbox_router true \ --enable_direct_routing false else - run_yuanrong /usr/bin/yr start "${NODE_PROXY_ARGS[@]}" \ + run_yuanrong /usr/bin/yr start "${NODE_PROXY_ARGS[@]}" "${EDGE_ARGS[@]}" \ --ip_address "${YR_NODE_IP}" \ --port_policy FIX \ --ds_node_timeout_s 30 \ @@ -189,11 +175,7 @@ else --enable_trace ${ENABLE_TRACE} \ --trace_config "$(cat /home/yuanrong/trace/trace_config.json)" \ -n ${HOSTNAME} \ - --enable_traefik_registry=${ENABLE_TRAEFIK_REGISTRY} \ - --traefik_enable_tls=${TRAEFIK_ENABLE_TLS:-false} \ - --traefik_etcd_prefix=traefik \ - --traefik_lease_ttl=300000 \ - --traefik_http_entrypoint=${TRAEFIK_HTTP_ENTRYPOINT:-websecure} \ + --enable_traefik_registry=false \ --log_root "${YR_LOG_PATH}" \ --fc_agent_mgr_retry_times 30 \ --fc_agent_mgr_retry_cycle 60000 \ diff --git a/builder/systemd_services/yuanrong.service b/builder/systemd_services/yuanrong.service index 8f350ec..6b9ef36 100644 --- a/builder/systemd_services/yuanrong.service +++ b/builder/systemd_services/yuanrong.service @@ -4,7 +4,7 @@ Description=yuanrong.service [Service] #Type=simple PIDFile=/run/yuanrong.pid -PassEnvironment=ENABLE_NODE_PROXY NODE_PROXY_PORT NODE_PROXY_HEALTH_PORT NODE_PROXY_ALLOWED_TARGET_CIDRS NODE_PROXY_ALLOWED_EDGE_CIDRS DATA_PLANE_LOG_DIR ENABLE_TRAEFIK ETCD_PORT ETCD_PEER_PORT ETCD_ADDRESS HOSTNAME AKS_LOCAL_MODE AKERNEL_NODE_IP INSTANCE_IP LITEBUS_DATA_KEY YR_LOG_PATH YR_INSTALLATION_DIR YR_RRT_CONTROL_SOCKET_PATH YR_IMAGE_PROCESS_CONFIG ENABLE_METRICS ENABLE_TRACE TRAEFIK_MODE TRAEFIK_ENABLE_TLS TRAEFIK_HTTP_ENTRYPOINT +PassEnvironment=ENABLE_NODE_PROXY NODE_PROXY_PORT NODE_PROXY_HEALTH_PORT NODE_PROXY_ALLOWED_TARGET_CIDRS NODE_PROXY_ALLOWED_EDGE_CIDRS DATA_PLANE_LOG_DIR ETCD_PORT ETCD_PEER_PORT ETCD_ADDRESS HOSTNAME AKS_LOCAL_MODE AKERNEL_NODE_IP INSTANCE_IP LITEBUS_DATA_KEY YR_LOG_PATH YR_INSTALLATION_DIR YR_RRT_CONTROL_SOCKET_PATH YR_IMAGE_PROCESS_CONFIG ENABLE_METRICS ENABLE_TRACE ENABLE_EDGE_FRONTEND EDGE_TLS_PORT EDGE_PLAIN_PORT EDGE_HEALTH_PORT EDGE_TLS_CERT EDGE_TLS_KEY EDGE_ALLOWED_CLIENT_CIDRS EDGE_GRAFANA_URL Environment="CONTAINER_EP=unix:///run/sandboxd/sandboxd.sock" Environment="RUNTIME_HOME_DIR=/home/yuanrong/runtime" Environment="YR_NOSET_CUDA_VISIBLE_DEVICES=1" diff --git a/deploy/README.md b/deploy/README.md index f427e77..2c5658d 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -179,7 +179,7 @@ start/stop instructions. The umbrella chart in [`akernel/`](./akernel/) bundles two subcharts: - **core** — scheduler + node-side components (etcd, master, frontend, node - DaemonSet, and Traefik) + DaemonSet, and Edge ingress) - **monitor** — observability stack (Prometheus, Grafana, Loki, Tempo) ```bash @@ -213,40 +213,18 @@ image: Each component can still override `master.image`, `frontend.image`, or `node.image` when a split-image deployment is required. -### Public Traefik entrypoints +### Public Edge entrypoints -For cloud deployments, use Traefik with two public entrypoints: - -```yaml -traefik: - enabled: true - enableWebEntrypoint: true - ports: - websecure: 443 - web: 80 -``` - -The `websecure` entrypoint serves the AKernel frontend API and exec websocket -over HTTPS/WSS. The `web` entrypoint serves function port-forwarding traffic -over plain HTTP/WS. With this layout the Python SDK only needs the LoadBalancer -host or IP: +The core chart starts Edge alongside Frontend and Node Proxy alongside each +node by default. Edge serves API and exec WebSocket traffic over HTTPS/WSS on +443, and published sandbox ports and reverse tunnels over HTTP/WS on 80. ```bash -export AKERNEL_SERVER_ADDRESS= +export AKERNEL_SERVER_ADDRESS= ``` -Do not set `traefik.tls.enabled` just to make port 443 work. The frontend -router is already configured as a TLS router; `traefik.tls.enabled` only mounts -a custom default certificate Secret. When it is `false`, Traefik uses its -default certificate. - -The legacy single-entrypoint mode is still available by setting -`traefik.enableWebEntrypoint=false`. In that mode API, exec, and function -traffic share `traefik.ports.tcp`, so SDK clients should use an explicit port: - -```bash -export AKERNEL_SERVER_ADDRESS=: -``` +See [Edge and Node Proxy ingress](#edge-and-node-proxy-ingress) for certificate, +network allowlist, and existing LoadBalancer migration settings. ### IAM token signing seed @@ -274,8 +252,8 @@ helm template akernel ./akernel \ The all-in-one image does not contain a TLS private key. The core chart creates one deployment-specific Secret and mounts the same certificate into master and frontend Pods. This certificate protects the openYuanrong frontend and IAM -service connections; it is separate from the certificate served by Traefik's -public `websecure` entrypoint. +service connections and is also the default Edge ingress certificate. Set +`dataPlane.edge.tlsSecretName` to use a dedicated ingress certificate. Regular `helm install` and `helm upgrade` reuse the existing Secret. For a render-and-apply workflow, create it once before rendering so a new certificate @@ -314,7 +292,7 @@ Per-vendor details are in [`terraform/huaweicloud/README.md`](./terraform/huaweicloud/README.md). The Alibaba Cloud Terraform defaults follow the recommended public layout: -frontend enabled, Traefik `websecure:443` plus `web:80`, and Grafana exposed +frontend enabled, Edge HTTPS 443 plus HTTP 80, and Grafana exposed through its own LoadBalancer when `install_monitor=true`. Set `install_dragonfly=true` to install the pinned official Dragonfly chart and inject its seed-client proxy into the node runtime configuration. @@ -326,7 +304,7 @@ loop-backed filestore. See the Aliyun guide for capacity, opt-out, and node replacement details. Only the AKernel all-in-one image is pushed to the registry selected by -`make config`. etcd, Traefik, Grafana, Prometheus, Loki, Tempo, and BusyBox use +`make config`. etcd, Grafana, Prometheus, Loki, Tempo, and BusyBox use their pinned official public images by default. Set the per-component image overrides when a private cluster requires mirrored third-party images. @@ -347,12 +325,14 @@ stop signal to the Go CLI and waits for its ordered cleanup before exiting; sandboxd stops after YuanRong. The YuanRong service uses `KillMode=mixed` so the bootstrap controls the initial shutdown of its child processes. -Images containing the YuanRong data-plane package can run Edge alongside the -Frontend and Node Proxy alongside each node through the Go CLI. Set -`dataPlane.enabled=true` and `traefik.enabled=false` in the core chart. Supply an -existing TLS Secret (`tls.crt` and `tls.key`) as -`dataPlane.edge.tlsSecretName`, and configure the allowed client, Edge pod and -sandbox destination CIDRs explicitly: +The image must contain the YuanRong Edge and Node Proxy binaries and Go CLI +data-plane deployment support. The builder pins Core and the data-plane source to the matching +`2b54c26885c6` build, including `config.sh`/`deploy.sh` and FunctionProxy +address registration. RRT and the sandbox SDK use `0.10.2rc2`. Both data-plane +components are enabled by default. Edge uses the +component TLS certificate unless an existing Secret (`tls.crt` and `tls.key`) +is selected with `dataPlane.edge.tlsSecretName`. Configure the allowed client, +Edge Pod and sandbox destination CIDRs for the deployment: ```yaml dataPlane: @@ -366,11 +346,10 @@ dataPlane: nodeProxy: allowedEdgeCIDRs: "192.168.0.0/16" allowedTargetCIDRs: "10.88.0.0/16" -traefik: - enabled: false ``` -Replace the CIDRs with those of the deployment. Edge exposes HTTPS/WSS on +Replace the CIDRs with those of the deployment. Node Proxy defaults allow +RFC1918 private networks; narrow them to the actual Pod and sandbox ranges. Edge exposes HTTPS/WSS on service port 443 and HTTP/WS on port 80; API and authenticated direct routes use TLS. The local Frontend and IAM hops use HTTP with IAM validation enabled. Node Proxy uses network security mode and only accepts connections from the @@ -382,6 +361,13 @@ Service name to retain its LoadBalancer identity and public address. Transfer its annotations, clusterIP and loadBalancerIP as well. The Service then selects Frontend/Edge pods; the Traefik Deployment and configuration are removed. Existing SDK API and gateway address settings select TLS or plain WebSocket. +The `/internal-stats` endpoint reports the Edge Pod address and listener ports +for SDK `internal=True` URLs. Set `dataPlane.edge.grafanaURL` to the Grafana +HTTP service URL to expose its configured `/grafana` subpath through Edge. + +Cloud profiles use `edge_*` and `node_proxy_allowed_*` Terraform variables. +Migrate existing ingress Service annotations, name and IP to the corresponding +`edge_service_*` variables before planning an upgrade. `make print-env` discovers the Service labeled `app.kubernetes.io/component=edge`. Set `AKERNEL_GATEWAY_SERVICE` to select a specific gateway Service. diff --git a/deploy/akernel/charts/core/templates/edge/secret.yaml b/deploy/akernel/charts/core/templates/edge/secret.yaml new file mode 100644 index 0000000..94141b3 --- /dev/null +++ b/deploy/akernel/charts/core/templates/edge/secret.yaml @@ -0,0 +1,11 @@ +{{- if and .Values.dataPlane.enabled .Values.dataPlane.edge.tls.createSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ required "dataPlane.edge.tlsSecretName is required when creating its Secret" .Values.dataPlane.edge.tlsSecretName }} + namespace: {{ .Release.Namespace }} +type: kubernetes.io/tls +data: + tls.crt: {{ required "dataPlane.edge.tls.cert is required" .Values.dataPlane.edge.tls.cert | b64enc | quote }} + tls.key: {{ required "dataPlane.edge.tls.key is required" .Values.dataPlane.edge.tls.key | b64enc | quote }} +{{- end }} diff --git a/deploy/akernel/charts/core/templates/edge/service.yaml b/deploy/akernel/charts/core/templates/edge/service.yaml index 8801ff1..449d259 100644 --- a/deploy/akernel/charts/core/templates/edge/service.yaml +++ b/deploy/akernel/charts/core/templates/edge/service.yaml @@ -2,9 +2,6 @@ {{- if not .Values.frontend.enabled }} {{- fail "dataPlane.enabled requires frontend.enabled" }} {{- end }} -{{- if .Values.traefik.enabled }} -{{- fail "Disable traefik.enabled when dataPlane.enabled is true" }} -{{- end }} apiVersion: v1 kind: Service metadata: diff --git a/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml b/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml index a984ecf..9aa2ca8 100644 --- a/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml +++ b/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml @@ -90,10 +90,16 @@ spec: {{- if .Values.dataPlane.enabled }} - name: ENABLE_EDGE_FRONTEND value: "true" + - name: EDGE_GRAFANA_URL + value: {{ .Values.dataPlane.edge.grafanaURL | quote }} - name: INSTANCE_IP valueFrom: fieldRef: fieldPath: status.podIP + - name: EDGE_TLS_CERT + value: /etc/akernel-edge-tls/tls.crt + - name: EDGE_TLS_KEY + value: /etc/akernel-edge-tls/tls.key - name: EDGE_TLS_PORT value: {{ .Values.dataPlane.edge.tlsPort | quote }} - name: EDGE_PLAIN_PORT @@ -103,8 +109,6 @@ spec: - name: EDGE_ALLOWED_CLIENT_CIDRS value: {{ required "dataPlane.edge.allowedClientCIDRs is required" .Values.dataPlane.edge.allowedClientCIDRs | quote }} {{- end }} - - name: ENABLE_TRAEFIK - value: {{ and .Values.traefik.enabled (not .Values.dataPlane.enabled) | quote }} - name: AKERNEL_ROLE value: "frontend" - name: LITEBUS_DATA_KEY @@ -195,7 +199,14 @@ spec: {{- if .Values.dataPlane.enabled }} - name: edge-tls secret: - secretName: {{ .Values.dataPlane.edge.tlsSecretName }} + secretName: {{ .Values.dataPlane.edge.tlsSecretName | default (include "core.componentTLSSecretName" .) }} + {{- if not .Values.dataPlane.edge.tlsSecretName }} + items: + - key: module.crt + path: tls.crt + - key: module.key + path: tls.key + {{- end }} defaultMode: 0400 {{- end }} - name: component-tls diff --git a/deploy/akernel/charts/core/templates/master/akernel_master.yaml b/deploy/akernel/charts/core/templates/master/akernel_master.yaml index 0c4023c..12532ef 100644 --- a/deploy/akernel/charts/core/templates/master/akernel_master.yaml +++ b/deploy/akernel/charts/core/templates/master/akernel_master.yaml @@ -77,8 +77,6 @@ spec: command: - /usr/local/bin/akernel-entrypoint env: - - name: ENABLE_TRAEFIK - value: {{ and .Values.traefik.enabled (not .Values.dataPlane.enabled) | quote }} - name: AKERNEL_ROLE value: "master" {{- if .Values.frontend.enabled }} @@ -114,12 +112,6 @@ spec: value: {{ ne (.Values.monitoring.prometheusEndpoint | default "") "" | ternary "true" "false" | quote }} - name: ENABLE_TRACE value: {{ ne (.Values.monitoring.tempoEndpoint | default "") "" | ternary "true" "false" | quote }} - - name: TRAEFIK_MODE - value: {{ .Values.traefik.mode | default "http" | quote }} - - name: TRAEFIK_ENABLE_TLS - value: {{ .Values.traefik.enableWebEntrypoint | default false | ternary false (.Values.traefik.enableTLS | default false) | quote }} - - name: TRAEFIK_HTTP_ENTRYPOINT - value: {{ .Values.traefik.enableWebEntrypoint | default false | ternary "web" "websecure" | quote }} ports: - name: http containerPort: 8888 diff --git a/deploy/akernel/charts/core/templates/node/daemonset.yaml b/deploy/akernel/charts/core/templates/node/daemonset.yaml index 1ce2a77..187f72d 100644 --- a/deploy/akernel/charts/core/templates/node/daemonset.yaml +++ b/deploy/akernel/charts/core/templates/node/daemonset.yaml @@ -101,8 +101,6 @@ spec: - name: NODE_PROXY_ALLOWED_EDGE_CIDRS value: {{ required "dataPlane.nodeProxy.allowedEdgeCIDRs is required" .Values.dataPlane.nodeProxy.allowedEdgeCIDRs | quote }} {{- end }} - - name: ENABLE_TRAEFIK - value: {{ and .Values.traefik.enabled (not .Values.dataPlane.enabled) | quote }} - name: AKERNEL_ROLE value: "node" - name: RUNSC_AKERNEL @@ -153,12 +151,6 @@ spec: value: {{ ne (.Values.monitoring.prometheusEndpoint | default "") "" | ternary "true" "false" | quote }} - name: ENABLE_TRACE value: {{ ne (.Values.monitoring.tempoEndpoint | default "") "" | ternary "true" "false" | quote }} - - name: TRAEFIK_MODE - value: {{ .Values.traefik.mode | default "http" | quote }} - - name: TRAEFIK_ENABLE_TLS - value: {{ .Values.traefik.enableWebEntrypoint | default false | ternary false (.Values.traefik.enableTLS | default false) | quote }} - - name: TRAEFIK_HTTP_ENTRYPOINT - value: {{ .Values.traefik.enableWebEntrypoint | default false | ternary "web" "websecure" | quote }} securityContext: privileged: true volumeMounts: diff --git a/deploy/akernel/charts/core/templates/traefik/configmap.yaml b/deploy/akernel/charts/core/templates/traefik/configmap.yaml deleted file mode 100644 index 585259d..0000000 --- a/deploy/akernel/charts/core/templates/traefik/configmap.yaml +++ /dev/null @@ -1,151 +0,0 @@ -{{- if .Values.traefik.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: traefik-static - namespace: {{ .Release.Namespace }} -data: - traefik.yml: | - entryPoints: - {{- if .Values.traefik.enableWebEntrypoint }} - websecure: - address: ":{{ .Values.traefik.ports.websecure }}" - web: - address: ":{{ .Values.traefik.ports.web }}" - {{- else }} - websecure: - address: ":{{ .Values.traefik.ports.tcp }}" - {{- end }} - traefik: - address: ":{{ .Values.traefik.ports.dashboard }}" - - {{- $masterEtcd := .Values.master.etcd | default dict }} - {{- $etcdHost := get $masterEtcd "host" | default (printf "akernel-etcd.%s.svc.cluster.local" .Release.Namespace) }} - {{- $etcdPort := get $masterEtcd "port" | default "2379" }} - {{- $masterHost := .Values.master.host | default (printf "akernel-master.%s.svc.cluster.local" .Release.Namespace) }} - {{- $masterPort := .Values.master.port | default "22770" }} - providers: - file: - directory: /etc/traefik/dynamic - watch: true - {{- if eq (.Values.traefik.mode | default "http") "http" }} - http: - endpoint: "http://{{ $masterHost }}:{{ $masterPort }}/global-scheduler/traefik/config" - pollInterval: "1s" - {{- else }} - etcd: - endpoints: - - "{{ $etcdHost }}:{{ $etcdPort }}" - rootKey: "traefik" - {{- end }} - - ping: {} - - api: - dashboard: true - insecure: true - - log: - level: INFO - filePath: /var/logs/traefik.log - - accessLog: - format: json - fields: - names: - # Exec places authentication data in the websocket query string. - # Keep status and timing fields, but never persist the request path. - RequestPath: drop ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: traefik-dynamic - namespace: {{ .Release.Namespace }} -data: - config.yml: | - {{- if .Values.traefik.tls.enabled }} - tls: - stores: - default: - defaultCertificate: - certFile: /openyuanrong/cert/module.crt - keyFile: /openyuanrong/cert/module.key - certificates: - - certFile: /openyuanrong/cert/module.crt - keyFile: /openyuanrong/cert/module.key - {{- end }} - - http: - routers: - frontend-terminal: - entryPoints: - - websecure - rule: "PathPrefix(`/terminal`) || PathPrefix(`/api/instances`) || PathPrefix(`/api/jobs`) || PathPrefix(`/functions`) || PathPrefix(`/api-docs`) || PathPrefix(`/admin/v1/functions`) || PathPrefix(`/serverless/v1/functions`) || PathPrefix(`/serverless/v1/stream`) || PathPrefix(`/serverless/v1/componentshealth`) || PathPrefix(`/serverless/v1/posix`) || PathPrefix(`/serverless/v2`) || PathPrefix(`/frontend/v1/instance`) || PathPrefix(`/datasystem/v1`) || PathPrefix(`/app/v1`) || PathPrefix(`/client/v1/lease`) || PathPrefix(`/invocations`) || PathPrefix(`/global-scheduler`) || Path(`/healthz`)" - service: {{ if .Values.frontend.enabled }}akernel-frontend{{ else }}akernel-master{{ end }} - tls: {} - sandbox-router: - entryPoints: - - websecure - rule: "PathPrefix(`/api/sandbox`) || PathPrefix(`/direct/`) || Path(`/direct`)" - priority: 100 - service: {{ if .Values.frontend.enabled }}akernel-frontend{{ else }}akernel-master{{ end }} - tls: {} - {{- if .Values.traefik.grafana.enabled }} - grafana: - entryPoints: - - websecure - rule: "PathPrefix(`/grafana`)" - service: grafana - tls: {} - {{- end }} - {{- if .Values.traefik.internalStats.enabled }} - internal-stats: - entryPoints: - - websecure - rule: "Path(`/internal-stats`)" - service: internal-stats - middlewares: - - internal-stats-strip - tls: {} - {{- end }} - - services: - {{ if .Values.frontend.enabled }}akernel-frontend{{ else }}akernel-master{{ end }}: - loadBalancer: - serversTransport: frontend-transport - servers: - - url: "https://{{ if .Values.frontend.enabled }}akernel-frontend{{ else }}akernel-master{{ end }}:8888" - {{- if .Values.traefik.grafana.enabled }} - grafana: - loadBalancer: - servers: - - url: "{{ .Values.traefik.grafana.url }}" - {{- end }} - {{- if .Values.traefik.internalStats.enabled }} - internal-stats: - loadBalancer: - servers: - - url: "http://localhost:{{ .Values.traefik.internalStats.port }}" - {{- end }} - - {{- if .Values.traefik.internalStats.enabled }} - middlewares: - internal-stats-strip: - stripPrefix: - prefixes: - - "/internal-stats" - {{- end }} - - serversTransports: - frontend-transport: - insecureSkipVerify: true - {{- if .Values.traefik.frontendTransport.disableHTTP2 }} - # Force HTTP/1.1 to the frontend backend. With an HTTP/2 backend, - # Traefik multiplexes all requests over a single connection; the 1s - # full-config reloads (390+ churning per-sandbox routes) drain and - # rebuild that connection, stalling in-flight requests for 7-10s. - # An HTTP/1.1 connection pool is unaffected. - disableHTTP2: true - {{- end }} -{{- end }} diff --git a/deploy/akernel/charts/core/templates/traefik/deployment.yaml b/deploy/akernel/charts/core/templates/traefik/deployment.yaml deleted file mode 100644 index 0c7373a..0000000 --- a/deploy/akernel/charts/core/templates/traefik/deployment.yaml +++ /dev/null @@ -1,147 +0,0 @@ -{{- if .Values.traefik.enabled }} -{{- if .Values.kruise.enabled }} -apiVersion: apps.kruise.io/v1alpha1 -kind: CloneSet -{{- else }} -apiVersion: apps/v1 -kind: Deployment -{{- end }} -metadata: - name: traefik - namespace: {{ .Release.Namespace }} - labels: - app: traefik -spec: - {{- if .Values.kruise.enabled }} - updateStrategy: - type: InPlaceIfPossible - {{- else }} - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - maxSurge: 1 - {{- end }} - replicas: {{ .Values.traefik.replicas | default 1 }} - selector: - matchLabels: - app: traefik - template: - metadata: - {{- with .Values.traefik.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - app: traefik - {{- with .Values.traefik.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - {{- if .Values.kruise.enabled }} - readinessGates: - - conditionType: InPlaceUpdateReady - {{- end }} - {{- if .Values.traefik.affinity }} - affinity: - {{- toYaml .Values.traefik.affinity | nindent 8 }} - {{- else if gt (int (.Values.traefik.replicas | default 1)) 1 }} - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - traefik - topologyKey: kubernetes.io/hostname - {{- end }} - {{- with .Values.traefik.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: traefik - image: "{{ .Values.traefik.image.repository }}:{{ .Values.traefik.image.tag }}" - imagePullPolicy: {{ .Values.traefik.image.pullPolicy }} - args: - - --configFile=/etc/traefik/traefik.yml - ports: - {{- if .Values.traefik.enableWebEntrypoint }} - - name: websecure - containerPort: {{ .Values.traefik.ports.websecure }} - - name: web - containerPort: {{ .Values.traefik.ports.web }} - {{- else }} - - name: tcp - containerPort: {{ .Values.traefik.ports.tcp }} - {{- end }} - - name: dashboard - containerPort: {{ .Values.traefik.ports.dashboard }} - livenessProbe: - httpGet: - path: /ping - port: dashboard - initialDelaySeconds: 10 - periodSeconds: 15 - timeoutSeconds: 3 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /ping - port: dashboard - initialDelaySeconds: 5 - periodSeconds: 10 - timeoutSeconds: 3 - failureThreshold: 3 - volumeMounts: - - name: static-config - mountPath: /etc/traefik/traefik.yml - subPath: traefik.yml - readOnly: true - - name: dynamic-config - mountPath: /etc/traefik/dynamic - readOnly: true - - name: logs - mountPath: /var/logs - {{- if .Values.traefik.tls.enabled }} - - name: tls-cert - mountPath: /openyuanrong/cert - readOnly: true - {{- end }} - {{- if .Values.traefik.internalStats.enabled }} - - name: internal-stats - image: "{{ .Values.traefik.internalStats.image }}" - command: ['sh', '-c', 'mkdir -p /var/www && echo "{\"pod_ip\":\"$POD_IP\"}" > /var/www/index.html && httpd -f -p {{ .Values.traefik.internalStats.port }} -h /var/www'] - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - ports: - - name: internal-stats - containerPort: {{ .Values.traefik.internalStats.port }} - {{- end }} - volumes: - - name: static-config - configMap: - name: traefik-static - - name: dynamic-config - configMap: - name: traefik-dynamic - - name: logs - emptyDir: {} - {{- if .Values.traefik.tls.enabled }} - - name: tls-cert - secret: - secretName: {{ .Values.traefik.tls.secretName | default "traefik-tls" }} - optional: false - {{- end }} -{{- end }} diff --git a/deploy/akernel/charts/core/templates/traefik/secret.yaml b/deploy/akernel/charts/core/templates/traefik/secret.yaml deleted file mode 100644 index c64c39d..0000000 --- a/deploy/akernel/charts/core/templates/traefik/secret.yaml +++ /dev/null @@ -1,11 +0,0 @@ -{{- if and .Values.traefik.enabled .Values.traefik.tls.enabled .Values.traefik.tls.createSecret }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ .Values.traefik.tls.secretName | default "traefik-tls" }} - namespace: {{ .Release.Namespace }} -type: Opaque -data: - module.crt: {{ .Values.traefik.tls.moduleCrt | default "" | b64enc | quote }} - module.key: {{ .Values.traefik.tls.moduleKey | default "" | b64enc | quote }} -{{- end }} diff --git a/deploy/akernel/charts/core/templates/traefik/service.yaml b/deploy/akernel/charts/core/templates/traefik/service.yaml deleted file mode 100644 index 5696fcb..0000000 --- a/deploy/akernel/charts/core/templates/traefik/service.yaml +++ /dev/null @@ -1,39 +0,0 @@ -{{- if .Values.traefik.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: traefik - namespace: {{ .Release.Namespace }} - {{- with .Values.traefik.service.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - type: {{ .Values.traefik.service.type }} - {{- with .Values.traefik.service.clusterIP }} - clusterIP: {{ . }} - {{- end }} - {{- if and (eq .Values.traefik.service.type "LoadBalancer") .Values.traefik.service.loadBalancerIP }} - loadBalancerIP: {{ .Values.traefik.service.loadBalancerIP | quote }} - {{- end }} - ports: - {{- if .Values.traefik.enableWebEntrypoint }} - - name: websecure - port: {{ .Values.traefik.ports.websecure }} - targetPort: websecure - - name: web - port: {{ .Values.traefik.ports.web }} - targetPort: web - {{- else }} - - name: tcp - port: {{ .Values.traefik.ports.tcp }} - targetPort: tcp - {{- end }} - {{- if ne .Values.traefik.service.type "LoadBalancer" }} - - name: dashboard - port: {{ .Values.traefik.ports.dashboard }} - targetPort: dashboard - {{- end }} - selector: - app: traefik -{{- end }} diff --git a/deploy/akernel/charts/core/values.yaml b/deploy/akernel/charts/core/values.yaml index 2e59a96..bafacaa 100644 --- a/deploy/akernel/charts/core/values.yaml +++ b/deploy/akernel/charts/core/values.yaml @@ -525,61 +525,21 @@ node: provider="kubernetes" sock_path="/var/run/resource.sock" -traefik: - enabled: false - mode: "http" - enableTLS: false - # When enabled, creates a separate "web" entrypoint (no TLS) on its own port - # for port forwarding traffic, while frontend API stays on "websecure" with TLS. - enableWebEntrypoint: true - # Transport options for the frontend (akernel-master) backend. - frontendTransport: - # Force HTTP/1.1 to the frontend backend. The HTTP provider reloads the - # full config every 1s, and the per-sandbox routes churn constantly; each - # reload drains Traefik's backend connections. An HTTP/2 backend uses a - # single multiplexed connection, so a reload stalls all in-flight requests - # for 7-10s. An HTTP/1.1 connection pool avoids this. Keep true unless the - # frontend backend requires HTTP/2 (e.g. gRPC over :8888). - disableHTTP2: true - replicas: 1 - image: - repository: traefik - tag: v3.6.8 - pullPolicy: IfNotPresent - service: - type: LoadBalancer - clusterIP: "" - annotations: {} - loadBalancerIP: "" - ports: - tcp: 8888 - websecure: 443 - web: 80 - dashboard: 9990 - tls: - enabled: false - createSecret: false - secretName: "traefik-tls" - moduleCrt: "" - moduleKey: "" - internalStats: - enabled: false - image: busybox:1.37.0-musl - port: 8079 - grafana: - enabled: false - url: "http://grafana.akernel-monitor.svc:3000" - # Edge handles API, direct, tunnel and published-port ingress. It is started # with Frontend by the Go CLI; Node Proxy runs in each node network namespace. dataPlane: - enabled: false + enabled: true edge: - tlsPort: 8443 - plainPort: 8080 + tlsPort: 443 + plainPort: 80 healthPort: 18080 - tlsSecretName: akernel-edge-tls - allowedClientCIDRs: "" + tlsSecretName: "" + tls: + createSecret: false + cert: "" + key: "" + grafanaURL: "" + allowedClientCIDRs: "0.0.0.0/0" service: name: akernel-edge type: LoadBalancer @@ -591,5 +551,5 @@ dataPlane: nodeProxy: port: 9443 healthPort: 18443 - allowedTargetCIDRs: "" - allowedEdgeCIDRs: "" + allowedTargetCIDRs: "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" + allowedEdgeCIDRs: "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" diff --git a/deploy/akernel/charts/monitor/templates/prometheus/configmap.yaml b/deploy/akernel/charts/monitor/templates/prometheus/configmap.yaml index 82a80b4..2cb1ea2 100644 --- a/deploy/akernel/charts/monitor/templates/prometheus/configmap.yaml +++ b/deploy/akernel/charts/monitor/templates/prometheus/configmap.yaml @@ -66,7 +66,7 @@ data: regex: Running - source_labels: [__meta_kubernetes_pod_container_name] action: keep - regex: {{ $controlPlanePods.targetContainerRegex | default "akernel-master|akernel-frontend|etcd|traefik|manager|scheduler|seed-client|mysql|redis" | quote }} + regex: {{ $controlPlanePods.targetContainerRegex | default "akernel-master|akernel-frontend|etcd|manager|scheduler|seed-client|mysql|redis" | quote }} - source_labels: [__meta_kubernetes_pod_container_port_name, __meta_kubernetes_pod_container_port_number] action: keep regex: {{ printf "(%s);.*|;(%s)" ($controlPlanePods.targetPortNameRegex | default "http|client|dashboard|mysql|redis") ($controlPlanePods.targetPortNumberRegex | default "8000|4002|3306|6379") | quote }} @@ -103,7 +103,7 @@ data: target_label: component replacement: frontend - source_labels: [__meta_kubernetes_pod_label_app] - regex: (etcd|traefik) + regex: (etcd) action: replace target_label: component replacement: $1 @@ -168,7 +168,7 @@ data: regex: Running - source_labels: [__meta_kubernetes_pod_container_name] action: keep - regex: {{ $controlPlanePods.targetContainerRegex | default "akernel-master|akernel-frontend|etcd|traefik|manager|scheduler|seed-client|mysql|redis" | quote }} + regex: {{ $controlPlanePods.targetContainerRegex | default "akernel-master|akernel-frontend|etcd|manager|scheduler|seed-client|mysql|redis" | quote }} - source_labels: [__meta_kubernetes_pod_container_port_name, __meta_kubernetes_pod_container_port_number] action: keep regex: {{ printf "(%s);.*|;(%s)" ($controlPlanePods.targetPortNameRegex | default "http|client|dashboard|mysql|redis") ($controlPlanePods.targetPortNumberRegex | default "8000|4002|3306|6379") | quote }} @@ -205,7 +205,7 @@ data: target_label: component replacement: frontend - source_labels: [__meta_kubernetes_pod_label_app] - regex: (etcd|traefik) + regex: (etcd) action: replace target_label: component replacement: $1 diff --git a/deploy/akernel/charts/monitor/values.yaml b/deploy/akernel/charts/monitor/values.yaml index ebca533..1519df0 100644 --- a/deploy/akernel/charts/monitor/values.yaml +++ b/deploy/akernel/charts/monitor/values.yaml @@ -200,7 +200,7 @@ prometheusServer: namespaces: - "akernel" labelSelector: "" - targetContainerRegex: "akernel-master|akernel-frontend|etcd|traefik|manager|scheduler|seed-client|mysql|redis" + targetContainerRegex: "akernel-master|akernel-frontend|etcd|manager|scheduler|seed-client|mysql|redis" targetPortNameRegex: "http|client|dashboard|mysql|redis" targetPortNumberRegex: "8000|4002|3306|6379" kubelet: diff --git a/deploy/scripts/configure.sh b/deploy/scripts/configure.sh index 69f1915..2b8c8b4 100755 --- a/deploy/scripts/configure.sh +++ b/deploy/scripts/configure.sh @@ -332,16 +332,6 @@ frontend_replicas = 1 frontend_cpu = "1" frontend_memory = "2Gi" -traefik_enabled = true -install_traefik = true -traefik_service_type = "LoadBalancer" -traefik_enable_web_entrypoint = true -traefik_websecure_port = 443 -traefik_web_port = 80 -traefik_tls_enabled = false -traefik_tls_create_secret = false -traefik_internal_stats_enabled = true - install_prereqs = false install_monitor = ${install_monitor} @@ -387,15 +377,6 @@ frontend_replicas = 1 frontend_cpu = "1" frontend_memory = "2Gi" -install_traefik = true -traefik_public_access = true -traefik_enable_web_entrypoint = true -traefik_websecure_port = 443 -traefik_web_port = 80 -traefik_tls_enabled = false -traefik_tls_create_secret = false -traefik_internal_stats_enabled = true - install_prereqs = false install_monitor = ${install_monitor} diff --git a/deploy/scripts/print-sdk-env.sh b/deploy/scripts/print-sdk-env.sh index fb6b279..5e0307e 100755 --- a/deploy/scripts/print-sdk-env.sh +++ b/deploy/scripts/print-sdk-env.sh @@ -57,15 +57,24 @@ if [[ -z "${gateway_service}" ]]; then gateway_service="$(kubectl --kubeconfig "${kubeconfig}" -n "${core_ns}" get svc \ -l app.kubernetes.io/component=edge -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" fi -gateway_service="${gateway_service:-traefik}" +[[ -n "${gateway_service}" ]] || die "Edge gateway Service not found; set AKERNEL_GATEWAY_SERVICE" gateway_host="$(get_lb_host "${core_ns}" "${gateway_service}")" [[ -n "${gateway_host}" ]] || die "${gateway_service} LoadBalancer address is not ready" +https_port="$(kubectl --kubeconfig "${kubeconfig}" -n "${core_ns}" get svc "${gateway_service}" -o 'jsonpath={.spec.ports[?(@.name=="websecure")].port}')" +http_port="$(kubectl --kubeconfig "${kubeconfig}" -n "${core_ns}" get svc "${gateway_service}" -o 'jsonpath={.spec.ports[?(@.name=="web")].port}')" +[[ -n "${https_port}" && -n "${http_port}" ]] || die "Edge Service must expose websecure and web ports" + token="$("${AKERNEL_REPO_ROOT}/deploy/scripts/generate-token.py" --env "${env_name}" --write-file "${dir}/token")" sdk_env="${dir}/sdk.env" { - printf 'export AKERNEL_SERVER_ADDRESS=%q\n' "${gateway_host}" + if [[ "${https_port}" == 443 && "${http_port}" == 80 ]]; then + printf 'export AKERNEL_SERVER_ADDRESS=%q\n' "${gateway_host}" + else + printf 'export AKERNEL_SERVER_ADDRESS=%q\n' "https://${gateway_host}:${https_port}" + printf 'export AKERNEL_GATEWAY_ADDRESS=%q\n' "http://${gateway_host}:${http_port}" + fi printf 'export AKERNEL_TOKEN=%q\n' "${token}" } > "${sdk_env}" chmod 600 "${sdk_env}" diff --git a/deploy/standalone/README.md b/deploy/standalone/README.md index b24912b..f0e54bd 100644 --- a/deploy/standalone/README.md +++ b/deploy/standalone/README.md @@ -2,15 +2,10 @@ This directory contains scripts and configurations for running AKernel in standalone mode using Docker or Pouch, without Kubernetes. The deployment uses -two containers on the default container bridge: - -- `akernel-node` runs the AKernel all-in-one image. -- `akernel-traefik` runs the official Traefik image as the external gateway. - -Keeping the gateway in a separate network namespace allows sandboxd's normal -`PREROUTING` rules to handle gateway traffic. The all-in-one frontend sends -traffic from the node network namespace, so the standalone sandboxd config -also enables its local-output DNAT support. +one privileged `akernel-node` container. The Go CLI starts Edge and Node Proxy +alongside the control plane, while systemd supervises YuanRong and sandboxd. +The standalone sandboxd config enables local-output DNAT for traffic from the +shared node network namespace. The default runtime is gVisor `runsc`. The bundled image also contains Kata Containers and Firecracker. Both `Sandbox(runtime="kata")` and @@ -108,8 +103,8 @@ initialize ACLs while pre-ACL sandboxes remain in its store. ``` deploy/standalone/ ├── README.md # This file -├── start.sh # Start AKernel and Traefik containers -├── stop.sh # Stop AKernel and Traefik containers +├── start.sh # Start the AKernel container +├── stop.sh # Stop the AKernel container └── config/ # Configuration files ├── config.json # OCI runtime configuration ├── oss_auths.json # OSS authentication (edit as needed) @@ -154,6 +149,10 @@ Edit `config/oss.json` and `config/registry.json` to point to your actual OSS an ### 3. Start AKernel +When upgrading an existing standalone installation, run `stop.sh` first. It +also removes the gateway container from earlier releases. The new deployment +uses the AKernel container IP for SDK access. + ```bash cd deploy/standalone ./start.sh @@ -165,16 +164,13 @@ This will: - Use `akerneldev/all-in-one:latest` if `IMAGE` is not set, reusing a local copy when present and otherwise pulling it from Docker Hub - Start the privileged AKernel all-in-one container -- Start an independent Traefik container for the HTTPS API and HTTP sandbox - port-forwarding gateway -- Configure Traefik to poll FunctionMaster's HTTP provider for per-sandbox - tunnel routes, including custom tunnel ports +- Start Edge and Node Proxy through the Go CLI inside the all-in-one container - Generate a deployment-specific IAM signing seed and a 24-hour SDK token - Generate a sandboxd config using `AKERNEL_NAT_BACKEND` (`iptables` by default) -- Print the Traefik container IP to use as `AKERNEL_SERVER_ADDRESS` +- Print the AKernel container IP to use as `AKERNEL_SERVER_ADDRESS` -No host ports are published. On Linux, the host accesses Traefik directly +No host ports are published. On Linux, the host accesses Edge directly through its Docker bridge IP. ### 4. Check Status @@ -183,8 +179,8 @@ through its Docker bridge IP. # View AKernel logs sudo docker logs -f akernel-node -# View gateway logs -sudo docker logs -f akernel-traefik +# View data-plane logs +sudo docker exec akernel-node ls /var/log/akernel-edge # Enter the container sudo docker exec -it akernel-node bash @@ -205,20 +201,20 @@ sudo docker exec akernel-node systemctl status ### SDK Connection -Traefik listens on port 443 for the AKernel API and port 80 for sandbox port -forwarding. These ports are not published on the host. Use the Traefik +Edge listens on port 443 for the AKernel API and port 80 for sandbox port +forwarding. These ports are not published on the host. Use the AKernel container IP printed by `start.sh`, or retrieve it later: ```bash -TRAEFIK_IP=$(docker inspect \ +NODE_IP=$(docker inspect \ --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' \ - akernel-traefik) + akernel-node) ``` Set the SDK environment: ```bash -export AKERNEL_SERVER_ADDRESS="${TRAEFIK_IP}" +export AKERNEL_SERVER_ADDRESS="${NODE_IP}" export AKERNEL_TOKEN="$(cat data/token)" ``` @@ -240,13 +236,6 @@ variable to test another registry, tag, or locally built image: IMAGE=":" ./start.sh ``` -The gateway defaults to `traefik:v3.6.8`. Override it independently when -needed: - -```bash -TRAEFIK_IMAGE="traefik:v3.6.8" ./start.sh -``` - ### Data Directory Location By default, data is stored in `./data`. To change this, edit `start.sh`: diff --git a/deploy/standalone/start.sh b/deploy/standalone/start.sh index 8faed46..da1e12c 100755 --- a/deploy/standalone/start.sh +++ b/deploy/standalone/start.sh @@ -11,13 +11,10 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" CONFIG_DIR="${SCRIPT_DIR}/config" DATA_DIR="${SCRIPT_DIR}/data" -FRONTEND_PORT="8888" ETCD_PORT="${ETCD_PORT:-2379}" ETCD_PEER_PORT="${ETCD_PEER_PORT:-2378}" -NODE_CONTAINER_NAME="akernel-node" -TRAEFIK_CONTAINER_NAME="akernel-traefik" +NODE_CONTAINER_NAME="${NODE_CONTAINER_NAME:-akernel-node}" IMAGE="${IMAGE:-akerneldev/all-in-one:latest}" -TRAEFIK_IMAGE="${TRAEFIK_IMAGE:-traefik:v3.6.8}" IAM_SEED_FILE="${DATA_DIR}/iam-seed" TOKEN_FILE="${DATA_DIR}/token" SANDBOXD_CONFIG_FILE="${DATA_DIR}/sandboxd/config.toml" @@ -162,7 +159,7 @@ configure_auth() { # Stop and remove existing container cleanup_existing() { local container - for container in "${NODE_CONTAINER_NAME}" "${TRAEFIK_CONTAINER_NAME}"; do + for container in "${NODE_CONTAINER_NAME}"; do if "${DOCKER_PREFIX[@]}" ${DOCKER_CMD} container inspect "${container}" &> /dev/null; then log_warn "Existing container '${container}' found; run stop.sh first" exit 1 @@ -349,12 +346,9 @@ prepare_host_network_modules() { log_info "Loaded host filter, bridge, conntrack, and ipset modules for the iptables ACL backend" } -# Start the AKernel all-in-one container. Traefik runs separately so traffic -# from the gateway enters this network namespace through PREROUTING. +# Start the all-in-one container with Edge and Node Proxy supervised by Go CLI. start_node_container() { log_info "Starting container: ${NODE_CONTAINER_NAME}" - # FunctionMaster's HTTP provider publishes the per-sandbox routes required - # by reverse tunnels; the legacy etcd mode cannot publish those routes. "${DOCKER_PREFIX[@]}" ${DOCKER_CMD} run -d \ --name "${NODE_CONTAINER_NAME}" \ @@ -364,9 +358,11 @@ start_node_container() { -e AKS_LOCAL_MODE="true" \ -e YR_RRT_CONTROL_SOCKET_PATH="/run/akernel" \ -e YR_IMAGE_PROCESS_CONFIG="${YR_IMAGE_PROCESS_CONFIG}" \ - -e TRAEFIK_MODE="http" \ - -e TRAEFIK_HTTP_ENTRYPOINT="web" \ - -e TRAEFIK_ENABLE_TLS="false" \ + -e ENABLE_EDGE_FRONTEND=true \ + -e EDGE_GRAFANA_URL="${EDGE_GRAFANA_URL:-}" \ + -e ENABLE_NODE_PROXY=true \ + -e NODE_PROXY_ALLOWED_EDGE_CIDRS="${NODE_PROXY_ALLOWED_EDGE_CIDRS:-127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16}" \ + -e NODE_PROXY_ALLOWED_TARGET_CIDRS="${NODE_PROXY_ALLOWED_TARGET_CIDRS:-10.0.0.0/8,172.16.0.0/12,192.168.0.0/16}" \ -e ETCD_PORT="${ETCD_PORT}" \ -e ETCD_PEER_PORT="${ETCD_PEER_PORT}" \ -e NODE_NAME="$(hostname)" \ @@ -415,86 +411,28 @@ container_ip() { --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$1" } -write_traefik_config() { - local node_ip="$1" - local traefik_dir="${DATA_DIR}/traefik" - mkdir -p "${traefik_dir}" - - cat > "${traefik_dir}/dynamic.yml" < /dev/null; then - log_info "Traefik gateway is ready" + if curl --noproxy '*' -fkSs "https://${gateway_ip}/healthz" > /dev/null; then + log_info "Edge gateway is ready" return 0 fi if ! "${DOCKER_PREFIX[@]}" ${DOCKER_CMD} inspect \ - --format '{{.State.Running}}' "${TRAEFIK_CONTAINER_NAME}" 2> /dev/null \ + --format '{{.State.Running}}' "${NODE_CONTAINER_NAME}" 2> /dev/null \ | grep -q true; then - log_error "Traefik exited during startup" - "${DOCKER_PREFIX[@]}" ${DOCKER_CMD} logs "${TRAEFIK_CONTAINER_NAME}" || true + log_error "Edge exited during startup" + "${DOCKER_PREFIX[@]}" ${DOCKER_CMD} logs "${NODE_CONTAINER_NAME}" || true return 1 fi if [[ ${i} -eq ${retries} ]]; then - log_error "Traefik gateway did not become ready" + log_error "Edge gateway did not become ready" return 1 fi sleep ${delay} @@ -504,21 +442,17 @@ wait_for_gateway() { # Show status show_status() { local node_ip="$1" - local traefik_ip="$2" echo "" log_info "Container status:" "${DOCKER_PREFIX[@]}" ${DOCKER_CMD} ps -a \ - --filter "name=${NODE_CONTAINER_NAME}" \ - --filter "name=${TRAEFIK_CONTAINER_NAME}" + --filter "name=${NODE_CONTAINER_NAME}" echo "" log_info "Useful commands:" echo " AKernel logs: ${DOCKER_CMD} logs -f ${NODE_CONTAINER_NAME}" - echo " Traefik logs: ${DOCKER_CMD} logs -f ${TRAEFIK_CONTAINER_NAME}" echo " Enter AKernel: ${DOCKER_CMD} exec -it ${NODE_CONTAINER_NAME} bash" echo " AKernel IP: ${node_ip}" - echo " Traefik IP: ${traefik_ip}" echo " SDK token: ${TOKEN_FILE}" } @@ -527,7 +461,6 @@ check_prerequisites cleanup_existing configure_auth ensure_image "${IMAGE}" -ensure_image "${TRAEFIK_IMAGE}" configure_container_proxy configure_gpu configure_network @@ -539,18 +472,9 @@ if [[ -z "${NODE_IP}" ]]; then log_error "Could not determine the AKernel container IP" exit 1 fi -write_traefik_config "${NODE_IP}" -TRAEFIK_PROVIDER_ENDPOINT="http://${NODE_IP}:22770/global-scheduler/traefik/config" -log_info "Using FunctionMaster route provider: ${TRAEFIK_PROVIDER_ENDPOINT}" -start_traefik_container "${TRAEFIK_PROVIDER_ENDPOINT}" -TRAEFIK_IP="$(container_ip "${TRAEFIK_CONTAINER_NAME}")" -if [[ -z "${TRAEFIK_IP}" ]]; then - log_error "Could not determine the Traefik container IP" - exit 1 -fi -wait_for_gateway "${TRAEFIK_IP}" -show_status "${NODE_IP}" "${TRAEFIK_IP}" +wait_for_gateway "${NODE_IP}" +show_status "${NODE_IP}" log_info "AKernel started successfully in standalone mode" -log_info "Set AKERNEL_SERVER_ADDRESS=${TRAEFIK_IP}" +log_info "Set AKERNEL_SERVER_ADDRESS=${NODE_IP}" log_info "Set AKERNEL_TOKEN=\$(cat ${TOKEN_FILE})" diff --git a/deploy/standalone/stop.sh b/deploy/standalone/stop.sh index eb19800..38f2781 100755 --- a/deploy/standalone/stop.sh +++ b/deploy/standalone/stop.sh @@ -7,7 +7,11 @@ set -e -CONTAINER_NAMES=("akernel-traefik" "akernel-node") +CONTAINER_NAMES=("${NODE_CONTAINER_NAME:-akernel-node}") +# Remove the gateway container left by earlier standalone releases on upgrade. +if [[ "${NODE_CONTAINER_NAME:-akernel-node}" == akernel-node ]]; then + CONTAINER_NAMES=("akernel-traefik" "${CONTAINER_NAMES[@]}") +fi # Container runtime command (docker or pouch) DOCKER_CMD="" @@ -52,8 +56,7 @@ else exit 1 fi -# Stop the gateway before the AKernel container so no new requests arrive -# while the runtime is shutting down. +# The Go CLI supervises Edge, Node Proxy, and runtime shutdown. for container in "${CONTAINER_NAMES[@]}"; do if "${DOCKER_PREFIX[@]}" ${DOCKER_CMD} container inspect "${container}" &> /dev/null; then log_info "Stopping container: ${container}" diff --git a/deploy/terraform/aliyun/README.md b/deploy/terraform/aliyun/README.md index 228f6ff..d197a8f 100644 --- a/deploy/terraform/aliyun/README.md +++ b/deploy/terraform/aliyun/README.md @@ -179,28 +179,22 @@ image registry settings before applying the plan. ## Public endpoint model The default cloud deployment uses a split frontend plus a two-entrypoint -Traefik LoadBalancer: +Edge LoadBalancer: - `websecure:443` routes frontend API and exec websocket traffic over TLS. - `web:80` routes function port-forwarding traffic over plain HTTP/WS. -Use the Traefik LoadBalancer host or IP directly with the SDK: +Use the Edge LoadBalancer host or IP directly with the SDK: ```bash -export AKERNEL_SERVER_ADDRESS= +export AKERNEL_SERVER_ADDRESS= ``` -`traefik_tls_enabled` is only for mounting a custom default certificate. It is -not required for the `websecure` router on port 443; Traefik serves its default -certificate when the variable is `false`. - -To use the legacy single-entrypoint mode, set -`traefik_enable_web_entrypoint=false` and configure `traefik_tcp_port`. In that -mode SDK clients must include the port explicitly: - -```bash -export AKERNEL_SERVER_ADDRESS=: -``` +Edge uses the chart's component certificate by default. Set +`edge_tls_secret_name` for a dedicated TLS Secret, or also set +`edge_tls_create_secret`, `edge_tls_cert`, and `edge_tls_key` to create it. +Customize public ports with `edge_http_port` and `edge_https_port`; configure +the SDK API and gateway addresses explicitly when using custom ports. Enable OSS auth injection for AKernel node secret: diff --git a/deploy/terraform/aliyun/main.tf b/deploy/terraform/aliyun/main.tf index c23bf15..c2b8ed0 100644 --- a/deploy/terraform/aliyun/main.tf +++ b/deploy/terraform/aliyun/main.tf @@ -102,7 +102,7 @@ locals { slb_security_group_annotations = length(var.security_group_id) > 0 ? { "service.beta.kubernetes.io/alibaba-cloud-loadbalancer-security-group-ids" = var.security_group_id } : {} - effective_traefik_service_annotations = merge(local.slb_security_group_annotations, var.traefik_service_annotations) + effective_edge_service_annotations = merge(local.slb_security_group_annotations, var.edge_service_annotations) effective_storage_class = var.storage_class effective_monitor_storage_class = length(var.monitor_storage_class) > 0 ? var.monitor_storage_class : local.effective_storage_class @@ -128,11 +128,9 @@ locals { auths = { for host, cred in var.registry_auths : host => { auth = base64encode("${cred.username}:${cred.password}") } } } - etcd_image_repo = length(var.etcd_image_repository) > 0 ? var.etcd_image_repository : "public.ecr.aws/bitnami/etcd" - master_image_repo = length(var.master_image_repository) > 0 ? var.master_image_repository : "${local.acr_registry}/all-in-one" - node_image_repo = length(var.node_image_repository) > 0 ? var.node_image_repository : "${local.acr_registry}/all-in-one" - traefik_image_repo = length(var.traefik_image_repository) > 0 ? var.traefik_image_repository : "traefik" - traefik_internal_stats_image = length(var.traefik_internal_stats_image) > 0 ? var.traefik_internal_stats_image : "${local.acr_registry}/busybox:1.37.0-musl" + etcd_image_repo = length(var.etcd_image_repository) > 0 ? var.etcd_image_repository : "public.ecr.aws/bitnami/etcd" + master_image_repo = length(var.master_image_repository) > 0 ? var.master_image_repository : "${local.acr_registry}/all-in-one" + node_image_repo = length(var.node_image_repository) > 0 ? var.node_image_repository : "${local.acr_registry}/all-in-one" core_values = templatefile("${path.module}/values-akernel.yaml.tmpl", { acr_registry = local.acr_registry @@ -146,12 +144,9 @@ locals { master_image_tag = var.master_image_tag node_image_repository = local.node_image_repo node_image_tag = var.node_image_tag - traefik_image_repository = local.traefik_image_repo - traefik_image_tag = var.traefik_image_tag iam_litebus_data_key = var.iam_litebus_data_key enable_kruise = var.install_prereqs - master_service_type = (var.master_public_access_8888 && !var.traefik_enabled) ? var.master_service_type : "ClusterIP" - traefik_enabled = var.traefik_enabled + master_service_type = var.master_public_access_8888 ? var.master_service_type : "ClusterIP" sandboxd_nat_backend = var.sandboxd_nat_backend enable_runc = var.enable_runc node_secret_create = var.node_secret_create @@ -189,23 +184,20 @@ locals { frontend_cpu = var.frontend_cpu frontend_memory = var.frontend_memory - install_traefik = var.install_traefik - traefik_replicas = var.traefik_replicas - traefik_tcp_port = var.traefik_tcp_port - traefik_enable_web_entrypoint = var.traefik_enable_web_entrypoint - traefik_web_port = var.traefik_web_port - traefik_websecure_port = var.traefik_websecure_port - traefik_service_type = var.traefik_service_type - traefik_service_annotations = local.effective_traefik_service_annotations - traefik_tls_enabled = var.traefik_tls_enabled - traefik_tls_create_secret = var.traefik_tls_create_secret - traefik_tls_cert = var.traefik_tls_cert - traefik_tls_key = var.traefik_tls_key - traefik_internal_stats = var.traefik_internal_stats_enabled - traefik_internal_stats_image = local.traefik_internal_stats_image - traefik_grafana_enabled = var.install_monitor - traefik_grafana_url = var.install_monitor ? "http://grafana.${var.monitor_namespace}.svc:3000" : "" - + edge_service_name = var.edge_service_name + edge_service_type = var.edge_service_type + edge_service_loadbalancer_ip = var.edge_service_loadbalancer_ip + edge_http_port = var.edge_http_port + edge_https_port = var.edge_https_port + edge_tls_secret_name = var.edge_tls_secret_name + edge_tls_create_secret = var.edge_tls_create_secret + edge_tls_cert = var.edge_tls_cert + edge_tls_key = var.edge_tls_key + edge_allowed_client_cidrs = var.edge_allowed_client_cidrs + node_proxy_allowed_target_cidrs = var.node_proxy_allowed_target_cidrs + node_proxy_allowed_edge_cidrs = var.node_proxy_allowed_edge_cidrs + edge_service_annotations = local.effective_edge_service_annotations + edge_grafana_url = var.install_monitor && !var.grafana_public_access ? "http://grafana.${var.monitor_namespace}.svc:3000" : "" }) monitor_image_registry = var.monitor_image_registry diff --git a/deploy/terraform/aliyun/terraform.tfvars.example b/deploy/terraform/aliyun/terraform.tfvars.example index eb411f9..113e2c9 100644 --- a/deploy/terraform/aliyun/terraform.tfvars.example +++ b/deploy/terraform/aliyun/terraform.tfvars.example @@ -97,8 +97,6 @@ core_namespace = "akernel" # master_image_tag = "" # node_image_repository = "my-registry.com/akernel/all-in-one" # node_image_tag = "" -# traefik_image_repository = "my-registry.com/traefik" -# traefik_image_tag = "v3.6.8" # Optional mirror registry for Grafana, Prometheus, Loki, Tempo, and BusyBox. # Leave empty to use the projects' official public images. @@ -139,28 +137,13 @@ core_namespace = "akernel" # Optional: master replicas (for HA when frontend is enabled). # master_replicas = 1 -# Frontend Deployment. Traefik routes API traffic to akernel-frontend when it is enabled. +# Frontend Deployment. Edge runs alongside akernel-frontend when it is enabled. frontend_enabled = true frontend_replicas = 1 frontend_cpu = "1" frontend_memory = "2Gi" -# Traefik ingress controller (deployed via core chart). -traefik_enabled = true -install_traefik = true -traefik_service_type = "LoadBalancer" -traefik_enable_web_entrypoint = true -traefik_websecure_port = 443 -traefik_web_port = 80 -# traefik_replicas = 1 -# Optional: mount a custom default certificate. The websecure router still -# serves TLS when this is false, using Traefik's default certificate. -traefik_tls_enabled = false -traefik_tls_create_secret = false -# traefik_tls_cert = "" -# traefik_tls_key = "" -# traefik_internal_stats_enabled = true -# traefik_internal_stats_image = "registry-vpc.cn-hangzhou.aliyuncs.com/my-namespace/busybox:1.37.0-musl" +# Edge ingress is deployed by the core chart (HTTP 80 and HTTPS 443). # Install prerequisite components (OpenKruise) before AKernel charts. install_prereqs = false @@ -266,3 +249,9 @@ dragonfly_server_node_pool = { # Example for using existing cluster instead of creating ACK: # create_cluster = false # kubeconfig_path = "/abs/path/to/kubeconfig" + +# Edge ingress and Node Proxy are enabled by default. +edge_service_type = "LoadBalancer" +# edge_tls_secret_name = "edge-tls" +# node_proxy_allowed_edge_cidrs = "10.0.0.0/8" +# node_proxy_allowed_target_cidrs = "172.16.0.0/12" diff --git a/deploy/terraform/aliyun/values-akernel.yaml.tmpl b/deploy/terraform/aliyun/values-akernel.yaml.tmpl index 0ce6d32..b2052b6 100644 --- a/deploy/terraform/aliyun/values-akernel.yaml.tmpl +++ b/deploy/terraform/aliyun/values-akernel.yaml.tmpl @@ -221,42 +221,23 @@ node: provider="kubernetes" sock_path="/var/run/resource.sock" -traefik: - enabled: ${install_traefik} - replicas: ${traefik_replicas} - enableWebEntrypoint: ${traefik_enable_web_entrypoint} - image: - repository: "${traefik_image_repository}" - tag: "${traefik_image_tag}" - ports: -%{ if traefik_enable_web_entrypoint ~} - web: ${traefik_web_port} - websecure: ${traefik_websecure_port} -%{ else ~} - tcp: ${traefik_tcp_port} -%{ endif ~} - service: - type: "${traefik_service_type}" -%{ if length(traefik_service_annotations) > 0 ~} - annotations: -%{ for k, v in traefik_service_annotations ~} - ${k}: "${v}" -%{ endfor ~} -%{ endif ~} - tls: - enabled: ${traefik_tls_enabled} - createSecret: ${traefik_tls_create_secret} -%{ if traefik_tls_cert != "" ~} - moduleCrt: ${jsonencode(traefik_tls_cert)} -%{ endif ~} -%{ if traefik_tls_key != "" ~} - moduleKey: ${jsonencode(traefik_tls_key)} -%{ endif ~} - internalStats: - enabled: ${traefik_internal_stats} - image: "${traefik_internal_stats_image}" - grafana: - enabled: ${traefik_grafana_enabled} -%{ if traefik_grafana_url != "" ~} - url: "${traefik_grafana_url}" -%{ endif ~} +dataPlane: + enabled: true + edge: + tlsSecretName: "${edge_tls_secret_name}" + tls: + createSecret: ${edge_tls_create_secret} + cert: ${jsonencode(edge_tls_cert)} + key: ${jsonencode(edge_tls_key)} + allowedClientCIDRs: "${edge_allowed_client_cidrs}" + grafanaURL: "${edge_grafana_url}" + service: + name: "${edge_service_name}" + type: "${edge_service_type}" + annotations: ${jsonencode(edge_service_annotations)} + loadBalancerIP: "${edge_service_loadbalancer_ip}" + httpPort: ${edge_http_port} + httpsPort: ${edge_https_port} + nodeProxy: + allowedTargetCIDRs: "${node_proxy_allowed_target_cidrs}" + allowedEdgeCIDRs: "${node_proxy_allowed_edge_cidrs}" diff --git a/deploy/terraform/aliyun/variables.tf b/deploy/terraform/aliyun/variables.tf index d390e09..b971c3f 100644 --- a/deploy/terraform/aliyun/variables.tf +++ b/deploy/terraform/aliyun/variables.tf @@ -408,6 +408,10 @@ variable "frontend_enabled" { type = bool description = "Whether to enable frontend Deployment (splits from master for independent scaling)." default = true + validation { + condition = var.frontend_enabled + error_message = "Edge ingress requires frontend_enabled=true." + } } variable "frontend_replicas" { @@ -440,18 +444,6 @@ variable "node_image_tag" { default = "" } -variable "traefik_image_repository" { - type = string - description = "Image repository for Traefik. Empty uses the official traefik image." - default = "" -} - -variable "traefik_image_tag" { - type = string - description = "Image tag for Traefik." - default = "v3.6.8" -} - variable "monitor_image_registry" { type = string description = "Optional mirror registry prefix for Grafana, Prometheus, Loki, Tempo, and BusyBox. Empty uses their official public images." @@ -508,102 +500,10 @@ variable "master_service_type" { variable "master_public_access_8888" { type = bool - description = "Whether to expose akernel-master port 8888 publicly. Ignored (forced ClusterIP) when traefik_enabled=true." - default = false -} - - -variable "traefik_enabled" { - type = bool - description = "Whether to deploy Traefik as the cluster ingress. When true, akernel-master is kept ClusterIP and Traefik exposes port 8888 via LoadBalancer." - default = true -} - -variable "install_traefik" { - type = bool - description = "Whether to enable Traefik ingress controller in the core chart." - default = true -} - -variable "traefik_replicas" { - type = number - description = "Number of Traefik replicas." - default = 1 -} - -variable "traefik_tcp_port" { - type = number - description = "TCP port for Traefik websecure entrypoint (legacy single-entrypoint mode; ignored when traefik_enable_web_entrypoint=true)." - default = 8888 -} - -variable "traefik_enable_web_entrypoint" { - type = bool - description = "Enable dual entrypoints: 'websecure' (TLS, frontend API) on traefik_websecure_port and 'web' (plain HTTP, port forwarding) on traefik_web_port. When false, falls back to legacy single-entrypoint mode using traefik_tcp_port." - default = true -} - -variable "traefik_web_port" { - type = number - description = "Port for Traefik 'web' (plain HTTP) entrypoint. Only used when traefik_enable_web_entrypoint=true." - default = 80 -} - -variable "traefik_websecure_port" { - type = number - description = "Port for Traefik 'websecure' (TLS) entrypoint. Only used when traefik_enable_web_entrypoint=true." - default = 443 -} - -variable "traefik_service_type" { - type = string - description = "Service type for Traefik. Use 'LoadBalancer' for cloud deployments." - default = "LoadBalancer" -} - -variable "traefik_tls_enabled" { - type = bool - description = "Whether to mount a custom default certificate for Traefik. The websecure router still serves TLS when this is false." - default = false -} - -variable "traefik_tls_cert" { - type = string - description = "TLS certificate content (PEM) for Traefik. Only used when traefik_tls_enabled=true and traefik_tls_create_secret=true." - default = "" - sensitive = true -} - -variable "traefik_tls_key" { - type = string - description = "TLS private key content (PEM) for Traefik. Only used when traefik_tls_enabled=true and traefik_tls_create_secret=true." - default = "" - sensitive = true -} - -variable "traefik_tls_create_secret" { - type = bool - description = "Whether to create TLS secret from traefik_tls_cert/traefik_tls_key." - default = false -} - -variable "traefik_service_annotations" { - type = map(string) - description = "Extra annotations for Traefik LoadBalancer Service." - default = {} -} - -variable "traefik_internal_stats_enabled" { - type = bool - description = "Whether to enable the /internal-stats endpoint on Traefik." + description = "Whether to expose akernel-master port 8888 publicly. Defaults to internal access; SDK ingress is provided by Edge." default = true } -variable "traefik_internal_stats_image" { - type = string - description = "Optional BusyBox image for the Traefik /internal-stats sidecar. Empty uses the deployment ACR VPC endpoint." - default = "" -} variable "monitor_namespace" { type = string @@ -955,3 +855,83 @@ variable "tempo_resources" { }) default = {} } + +variable "edge_service_name" { + type = string + description = "Name of the Edge ingress Service. Reuse the existing ingress Service name during migration." + default = "akernel-edge" +} + +variable "edge_service_type" { + type = string + description = "Kubernetes Service type for Edge ingress." + default = "LoadBalancer" +} + +variable "edge_service_loadbalancer_ip" { + type = string + description = "Optional existing ingress LoadBalancer IP." + default = "" +} + +variable "edge_http_port" { + type = number + description = "External HTTP and WS sandbox-port Service port." + default = 80 +} + +variable "edge_https_port" { + type = number + description = "External HTTPS and WSS API Service port." + default = 443 +} + +variable "edge_tls_secret_name" { + type = string + description = "TLS Secret containing tls.crt and tls.key. Empty uses the component certificate." + default = "" +} + +variable "edge_tls_create_secret" { + type = bool + description = "Create the named TLS Secret from edge_tls_cert and edge_tls_key." + default = false +} + +variable "edge_tls_cert" { + type = string + description = "PEM certificate for the Edge TLS Secret." + default = "" + sensitive = true +} + +variable "edge_tls_key" { + type = string + description = "PEM private key for the Edge TLS Secret." + default = "" + sensitive = true +} + +variable "edge_allowed_client_cidrs" { + type = string + description = "Comma-separated client CIDRs allowed to reach Edge." + default = "0.0.0.0/0" +} + +variable "node_proxy_allowed_target_cidrs" { + type = string + description = "Comma-separated sandbox target CIDRs allowed by Node Proxy." + default = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" +} + +variable "node_proxy_allowed_edge_cidrs" { + type = string + description = "Comma-separated Edge Pod CIDRs allowed by Node Proxy." + default = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" +} + +variable "edge_service_annotations" { + type = map(string) + description = "Additional annotations for the Edge Service." + default = {} +} diff --git a/deploy/terraform/huaweicloud/README.md b/deploy/terraform/huaweicloud/README.md index b03ef62..0fc66fd 100644 --- a/deploy/terraform/huaweicloud/README.md +++ b/deploy/terraform/huaweicloud/README.md @@ -3,7 +3,7 @@ This Terraform module creates a Huawei Cloud CCE cluster and installs the AKernel core and optional monitor charts. It follows the same deployment contract as the Aliyun module: one all-in-one AKernel image, a generated IAM -seed, dual-entrypoint Traefik, optional public Grafana, and local state under +seed, Edge ingress and Node Proxy, optional public Grafana, and local state under `.akernel//`. ## Pod PID budget @@ -65,15 +65,15 @@ login is required. The two settings are mutually exclusive. The generated profile enables the public CCE API endpoint and node-subnet SNAT so Terraform can reach the cluster and worker nodes can pull public images. -Traefik is exposed through a public ELB with two entrypoints: +Edge is exposed through a public ELB with two entrypoints: - `websecure:443` serves the authenticated frontend API and exec websocket. - `web:80` serves sandbox port-forwarding traffic. -The SDK therefore needs only the Traefik ELB address: +The SDK therefore needs only the Edge ELB address: ```bash -export AKERNEL_SERVER_ADDRESS= +export AKERNEL_SERVER_ADDRESS= ``` Grafana uses a separate public ELB when monitoring and public Grafana access @@ -83,7 +83,7 @@ are enabled. Its generated administrator password is stored at ## Images Master, frontend, and node use the same configured AKernel all-in-one image. -etcd, Traefik, Grafana, Prometheus, Loki, Tempo, and BusyBox use pinned official +etcd, Grafana, Prometheus, Loki, Tempo, and BusyBox use pinned official public images by default. Use the component image variables or `monitor_image_registry` only when the cluster requires private mirrors. diff --git a/deploy/terraform/huaweicloud/main.tf b/deploy/terraform/huaweicloud/main.tf index da5e56b..5f0997f 100644 --- a/deploy/terraform/huaweicloud/main.tf +++ b/deploy/terraform/huaweicloud/main.tf @@ -55,11 +55,11 @@ locals { eip_type = var.master_elb_eip_type }) } : {} - huaweicloud_traefik_elb_annotations = var.traefik_public_access ? { + huaweicloud_edge_elb_annotations = var.edge_public_access ? { "kubernetes.io/elb.class" = "union" "kubernetes.io/elb.autocreate" = jsonencode({ type = "public" - bandwidth_name = "${var.cluster_name}-traefik-elb" + bandwidth_name = "${var.cluster_name}-edge-elb" bandwidth_chargemode = var.master_elb_bandwidth_charge_mode bandwidth_size = var.master_elb_bandwidth_size bandwidth_sharetype = "PER" @@ -104,8 +104,6 @@ locals { master_image_tag = var.master_image_tag node_image_repository = var.node_image_repository node_image_tag = var.node_image_tag - traefik_image_repository = var.traefik_image_repository - traefik_image_tag = var.traefik_image_tag iam_litebus_data_key = var.iam_litebus_data_key enable_kruise = var.install_prereqs master_service_type = var.master_public_access_8888 ? var.master_service_type : "ClusterIP" @@ -146,23 +144,21 @@ locals { frontend_cpu = var.frontend_cpu frontend_memory = var.frontend_memory - install_traefik = var.install_traefik - traefik_replicas = var.traefik_replicas - traefik_tcp_port = var.traefik_tcp_port - traefik_enable_web_entrypoint = var.traefik_enable_web_entrypoint - traefik_web_port = var.traefik_web_port - traefik_websecure_port = var.traefik_websecure_port - traefik_service_type = var.traefik_service_type - traefik_service_annotations = local.huaweicloud_traefik_elb_annotations - traefik_service_loadbalancer_ip = "" - traefik_tls_enabled = var.traefik_tls_enabled - traefik_tls_create_secret = var.traefik_tls_create_secret - traefik_tls_cert = var.traefik_tls_cert - traefik_tls_key = var.traefik_tls_key - traefik_internal_stats = var.traefik_internal_stats_enabled - traefik_internal_stats_image = var.traefik_internal_stats_image - traefik_grafana_enabled = var.install_monitor - traefik_grafana_url = var.install_monitor ? "http://grafana.${var.monitor_namespace}.svc:3000" : "" + edge_service_name = var.edge_service_name + edge_service_type = var.edge_service_type + edge_service_loadbalancer_ip = var.edge_service_loadbalancer_ip + edge_http_port = var.edge_http_port + edge_https_port = var.edge_https_port + edge_tls_secret_name = var.edge_tls_secret_name + edge_tls_create_secret = var.edge_tls_create_secret + edge_tls_cert = var.edge_tls_cert + edge_tls_key = var.edge_tls_key + edge_allowed_client_cidrs = var.edge_allowed_client_cidrs + node_proxy_allowed_target_cidrs = var.node_proxy_allowed_target_cidrs + node_proxy_allowed_edge_cidrs = var.node_proxy_allowed_edge_cidrs + edge_service_annotations = merge(local.huaweicloud_edge_elb_annotations, var.edge_service_annotations) + edge_grafana_url = var.install_monitor && !var.grafana_public_access ? "http://grafana.${var.monitor_namespace}.svc:3000" : "" + }) monitor_values = templatefile("${path.module}/values-monitor.yaml.tmpl", { diff --git a/deploy/terraform/huaweicloud/terraform.tfvars.example b/deploy/terraform/huaweicloud/terraform.tfvars.example index c0f9e1c..d3f6488 100644 --- a/deploy/terraform/huaweicloud/terraform.tfvars.example +++ b/deploy/terraform/huaweicloud/terraform.tfvars.example @@ -82,8 +82,6 @@ master_image_repository = "swr.cn-north-4.myhuaweicloud.com/akernel/all-in-one master_image_tag = "" node_image_repository = "swr.cn-north-4.myhuaweicloud.com/akernel/all-in-one" node_image_tag = "" -# traefik_image_repository = "traefik" -# traefik_image_tag = "v3.6.8" # Generate a unique seed for each deployment. Guided `make config` writes this # automatically; direct Terraform users can use `openssl rand -hex 32`. @@ -93,9 +91,6 @@ node_image_tag = "" # matching AKERNEL_ENABLE_RUNC=true profile setting. enable_runc = false -# Enable the /internal-stats endpoint sidecar on Traefik. -# traefik_internal_stats_enabled = true -# traefik_internal_stats_image = "my-registry.example.com/busybox:1.37.0-musl" # Optional mirror registry for Grafana, Prometheus, Loki, Tempo, and BusyBox. # Leave empty to use the projects' official public images. @@ -154,15 +149,12 @@ enable_runc = false # Optional: master replicas (for HA when frontend is enabled). # master_replicas = 1 -# Optional: enable frontend Deployment (splits from master for independent scaling). -# When enabled, Traefik routes API traffic to akernel-frontend instead of akernel-master. -# frontend_enabled = false +# Frontend runs Edge and scales independently from master. +# frontend_enabled = true # frontend_replicas = 2 # frontend_cpu = "4" # frontend_memory = "8Gi" -# Optional: traefik replicas. -# traefik_replicas = 1 # Restrict SSH CIDRs in production. allowed_ssh_cidrs = ["0.0.0.0/0"] @@ -195,3 +187,9 @@ prereq_kruise_chart_version = "1.8.3" # password = "" # } # } + +# Edge ingress and Node Proxy are enabled by default. +edge_service_type = "LoadBalancer" +# edge_tls_secret_name = "edge-tls" +# node_proxy_allowed_edge_cidrs = "10.0.0.0/8" +# node_proxy_allowed_target_cidrs = "172.16.0.0/12" diff --git a/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl b/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl index 89c3602..aae86a7 100644 --- a/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl +++ b/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl @@ -50,10 +50,9 @@ master: cpu: "${master_cpu}" memory: "${master_memory}" ephemeral-storage: "${master_ephemeral}" -%{ if frontend_enabled ~} - frontend: - enabled: true + enabled: ${frontend_enabled} +%{ if frontend_enabled ~} replicas: ${frontend_replicas} image: repository: "${master_image_repository}" @@ -199,38 +198,23 @@ node: provider="kubernetes" sock_path="/var/run/resource.sock" -traefik: - enabled: ${install_traefik} - replicas: ${traefik_replicas} - enableWebEntrypoint: ${traefik_enable_web_entrypoint} - image: - repository: "${traefik_image_repository}" - tag: "${traefik_image_tag}" - ports: -%{ if traefik_enable_web_entrypoint ~} - web: ${traefik_web_port} - websecure: ${traefik_websecure_port} -%{ else ~} - tcp: ${traefik_tcp_port} -%{ endif ~} - service: - type: "${traefik_service_type}" - annotations: ${jsonencode(traefik_service_annotations)} - loadBalancerIP: "${traefik_service_loadbalancer_ip}" - tls: - enabled: ${traefik_tls_enabled} - createSecret: ${traefik_tls_create_secret} -%{ if traefik_tls_cert != "" ~} - moduleCrt: ${jsonencode(traefik_tls_cert)} -%{ endif ~} -%{ if traefik_tls_key != "" ~} - moduleKey: ${jsonencode(traefik_tls_key)} -%{ endif ~} - internalStats: - enabled: ${traefik_internal_stats} - image: "${traefik_internal_stats_image}" - grafana: - enabled: ${traefik_grafana_enabled} -%{ if traefik_grafana_url != "" ~} - url: "${traefik_grafana_url}" -%{ endif ~} +dataPlane: + enabled: true + edge: + tlsSecretName: "${edge_tls_secret_name}" + tls: + createSecret: ${edge_tls_create_secret} + cert: ${jsonencode(edge_tls_cert)} + key: ${jsonencode(edge_tls_key)} + allowedClientCIDRs: "${edge_allowed_client_cidrs}" + grafanaURL: "${edge_grafana_url}" + service: + name: "${edge_service_name}" + type: "${edge_service_type}" + annotations: ${jsonencode(edge_service_annotations)} + loadBalancerIP: "${edge_service_loadbalancer_ip}" + httpPort: ${edge_http_port} + httpsPort: ${edge_https_port} + nodeProxy: + allowedTargetCIDRs: "${node_proxy_allowed_target_cidrs}" + allowedEdgeCIDRs: "${node_proxy_allowed_edge_cidrs}" diff --git a/deploy/terraform/huaweicloud/variables.tf b/deploy/terraform/huaweicloud/variables.tf index 5c668bb..2a45f4f 100644 --- a/deploy/terraform/huaweicloud/variables.tf +++ b/deploy/terraform/huaweicloud/variables.tf @@ -448,18 +448,6 @@ variable "node_image_tag" { default = "" } -variable "traefik_image_repository" { - type = string - description = "Image repository for Traefik." - default = "traefik" -} - -variable "traefik_image_tag" { - type = string - description = "Image tag for Traefik." - default = "v3.6.8" -} - variable "monitor_image_registry" { type = string description = "Optional mirror registry prefix for Grafana, Prometheus, Loki, Tempo, and BusyBox. Empty uses their official public images." @@ -560,7 +548,11 @@ variable "master_replicas" { variable "frontend_enabled" { type = bool description = "Whether to enable frontend Deployment (splits from master for independent scaling)." - default = false + default = true + validation { + condition = var.frontend_enabled + error_message = "Edge ingress requires frontend_enabled=true." + } } variable "frontend_replicas" { @@ -581,92 +573,6 @@ variable "frontend_memory" { default = "8Gi" } -variable "install_traefik" { - type = bool - description = "Whether to enable Traefik ingress controller in the core chart." - default = true -} - -variable "traefik_replicas" { - type = number - description = "Number of Traefik replicas." - default = 1 -} - -variable "traefik_tcp_port" { - type = number - description = "TCP port for Traefik websecure entrypoint (legacy single-entrypoint mode; ignored when traefik_enable_web_entrypoint=true)." - default = 8888 -} - -variable "traefik_enable_web_entrypoint" { - type = bool - description = "Enable dual entrypoints: 'websecure' (TLS, frontend API) on traefik_websecure_port and 'web' (plain HTTP, port forwarding) on traefik_web_port. When false, falls back to legacy single-entrypoint mode using traefik_tcp_port." - default = false -} - -variable "traefik_web_port" { - type = number - description = "Port for Traefik 'web' (plain HTTP) entrypoint. Only used when traefik_enable_web_entrypoint=true." - default = 80 -} - -variable "traefik_websecure_port" { - type = number - description = "Port for Traefik 'websecure' (TLS) entrypoint. Only used when traefik_enable_web_entrypoint=true." - default = 443 -} - -variable "traefik_service_type" { - type = string - description = "Service type for Traefik. Use 'LoadBalancer' for cloud deployments." - default = "LoadBalancer" -} - -variable "traefik_public_access" { - type = bool - description = "Whether Traefik receives a public Huawei Cloud LoadBalancer." - default = true -} - -variable "traefik_tls_enabled" { - type = bool - description = "Whether to enable TLS for Traefik." - default = false -} - -variable "traefik_tls_cert" { - type = string - description = "TLS certificate content (PEM) for Traefik. Only used when traefik_tls_enabled=true and traefik_tls_create_secret=true." - default = "" - sensitive = true -} - -variable "traefik_tls_key" { - type = string - description = "TLS private key content (PEM) for Traefik. Only used when traefik_tls_enabled=true and traefik_tls_create_secret=true." - default = "" - sensitive = true -} - -variable "traefik_tls_create_secret" { - type = bool - description = "Whether to create TLS secret from traefik_tls_cert/traefik_tls_key." - default = false -} - -variable "traefik_internal_stats_enabled" { - type = bool - description = "Whether to enable the /internal-stats endpoint on Traefik." - default = false -} - -variable "traefik_internal_stats_image" { - type = string - description = "BusyBox image for the Traefik /internal-stats sidecar." - default = "busybox:1.37.0-musl" -} - variable "monitor_namespace" { type = string description = "Namespace for monitor resources." @@ -989,3 +895,89 @@ variable "tempo_resources" { }) default = {} } + +variable "edge_service_name" { + type = string + description = "Name of the Edge ingress Service. Reuse the existing ingress Service name during migration." + default = "akernel-edge" +} + +variable "edge_service_type" { + type = string + description = "Kubernetes Service type for Edge ingress." + default = "LoadBalancer" +} + +variable "edge_service_loadbalancer_ip" { + type = string + description = "Optional existing ingress LoadBalancer IP." + default = "" +} + +variable "edge_http_port" { + type = number + description = "External HTTP and WS sandbox-port Service port." + default = 80 +} + +variable "edge_https_port" { + type = number + description = "External HTTPS and WSS API Service port." + default = 443 +} + +variable "edge_tls_secret_name" { + type = string + description = "TLS Secret containing tls.crt and tls.key. Empty uses the component certificate." + default = "" +} + +variable "edge_tls_create_secret" { + type = bool + description = "Create the named TLS Secret from edge_tls_cert and edge_tls_key." + default = false +} + +variable "edge_tls_cert" { + type = string + description = "PEM certificate for the Edge TLS Secret." + default = "" + sensitive = true +} + +variable "edge_tls_key" { + type = string + description = "PEM private key for the Edge TLS Secret." + default = "" + sensitive = true +} + +variable "edge_allowed_client_cidrs" { + type = string + description = "Comma-separated client CIDRs allowed to reach Edge." + default = "0.0.0.0/0" +} + +variable "node_proxy_allowed_target_cidrs" { + type = string + description = "Comma-separated sandbox target CIDRs allowed by Node Proxy." + default = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" +} + +variable "node_proxy_allowed_edge_cidrs" { + type = string + description = "Comma-separated Edge Pod CIDRs allowed by Node Proxy." + default = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" +} + +variable "edge_service_annotations" { + type = map(string) + description = "Additional annotations for the Edge Service." + default = {} +} + +variable "edge_public_access" { + type = bool + description = "Create a public Huawei Cloud ELB for Edge." + default = true +} diff --git a/sdk/python/README.md b/sdk/python/README.md index c18d5ae..1ba6c16 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -428,7 +428,7 @@ with Sandbox(port_forwardings=[8080]) as sandbox: ``` `get_port_url()` rejects undeclared ports. Pass `internal=True` only when a -deployment operator explicitly wants the direct Traefik address instead of the +deployment operator explicitly wants the direct Edge address instead of the public gateway. ## Local failover and reload @@ -483,7 +483,7 @@ with Sandbox(reverse_tunnel=tunnel) as sandbox: ) ``` -`reverse_port` carries the WebSocket tunnel through Traefik. `listen_port` is +`reverse_port` carries the WebSocket tunnel through Edge. `listen_port` is the loopback HTTP listener used inside the sandbox. Consequently, `sandbox.reverse_tunnel.url` is always `http://127.0.0.1:`, even when `target` uses HTTPS. diff --git a/sdk/python/akernel_sdk/_addresses.py b/sdk/python/akernel_sdk/_addresses.py index 6b6a375..3b4a1f9 100644 --- a/sdk/python/akernel_sdk/_addresses.py +++ b/sdk/python/akernel_sdk/_addresses.py @@ -121,7 +121,7 @@ def gateway_endpoint_from_env() -> Endpoint: """Return the public port-forwarding gateway endpoint. An explicit gateway override is parsed as plain HTTP by default because - standalone exposes Traefik's web entrypoint without TLS. Without an + standalone exposes Edge's HTTP listener without TLS. Without an explicit gateway, host-only server addresses use public 80, while host:port server addresses reuse the API port with plain HTTP. """ diff --git a/sdk/python/akernel_sdk/sandbox.py b/sdk/python/akernel_sdk/sandbox.py index d939611..cd9eb93 100644 --- a/sdk/python/akernel_sdk/sandbox.py +++ b/sdk/python/akernel_sdk/sandbox.py @@ -40,7 +40,7 @@ SandboxInfo, ) -_traefik_internal_ip_cache: str | None = None +_gateway_internal_address_cache: dict[tuple[str, Endpoint], tuple[str, int]] = {} logger = logging.getLogger(__name__) @@ -141,14 +141,14 @@ def _validate_integer( raise ValueError(f"{name} must be greater than or equal to {minimum}") -def _get_traefik_internal_ip(gateway: Endpoint) -> tuple[str, int]: - """Resolve Traefik's direct address for ``internal=True`` URLs.""" - - global _traefik_internal_ip_cache - if _traefik_internal_ip_cache is not None: - return _traefik_internal_ip_cache, gateway.port +def _get_gateway_internal_address(gateway: Endpoint) -> tuple[str, int]: + """Resolve the gateway's direct address for ``internal=True`` URLs.""" server = api_endpoint_from_env() + cache_key = (server.base_url(), gateway) + if cache_key in _gateway_internal_address_cache: + return _gateway_internal_address_cache[cache_key] + context = ssl.create_default_context() context.check_hostname = False context.verify_mode = ssl.CERT_NONE @@ -161,8 +161,13 @@ def _get_traefik_internal_ip(gateway: Endpoint) -> tuple[str, int]: pod_ip = payload.get("pod_ip") if not isinstance(pod_ip, str) or not pod_ip: raise RuntimeError("/internal-stats response does not contain pod_ip") - _traefik_internal_ip_cache = pod_ip - return pod_ip, gateway.port + port_key = "https_port" if gateway.use_tls else "http_port" + port = payload.get(port_key, gateway.port) + if isinstance(port, bool) or not isinstance(port, int) or not 1 <= port <= 65535: + raise RuntimeError(f"/internal-stats response contains invalid {port_key}") + address = (pod_ip, port) + _gateway_internal_address_cache[cache_key] = address + return address class Sandbox: @@ -538,7 +543,7 @@ def get_port_url(self, port: int, *, internal: bool = False) -> str: Args: port: Port included in ``port_forwardings`` at sandbox creation. - internal: Resolve Traefik's directly reachable address instead of + internal: Resolve the gateway's directly reachable address instead of the public gateway address. Raises: @@ -554,7 +559,7 @@ def get_port_url(self, port: int, *, internal: bool = False) -> str: gateway = gateway_endpoint_from_env() if internal: - pod_ip, gateway_port = _get_traefik_internal_ip(gateway) + pod_ip, gateway_port = _get_gateway_internal_address(gateway) direct = Endpoint( host=pod_ip, port=gateway_port, diff --git a/sdk/python/tests/unit/test_gateway_metadata.py b/sdk/python/tests/unit/test_gateway_metadata.py new file mode 100644 index 0000000..5bf4fd7 --- /dev/null +++ b/sdk/python/tests/unit/test_gateway_metadata.py @@ -0,0 +1,53 @@ +# Copyright (c) 2026 Ant Group Corporation. +# SPDX-License-Identifier: Apache-2.0 + +import json +import unittest +from unittest.mock import MagicMock, patch + +from akernel_sdk import sandbox +from akernel_sdk._addresses import Endpoint + + +class GatewayMetadataTest(unittest.TestCase): + def setUp(self): + sandbox._gateway_internal_address_cache.clear() + self.addCleanup(sandbox._gateway_internal_address_cache.clear) + + def resolve(self, payload, gateway): + response = MagicMock() + response.__enter__.return_value.read.return_value = json.dumps(payload).encode() + with patch.object(sandbox.urllib.request, "urlopen", return_value=response): + return sandbox._get_gateway_internal_address(gateway) + + @patch.object(sandbox, "api_endpoint_from_env", + return_value=Endpoint("public.example", 443, "https", False)) + def test_internal_ports_follow_listener_instead_of_service_port(self, _server): + payload = {"pod_ip": "10.0.0.4", "http_port": 8080, "https_port": 8443} + for scheme, expected in (("http", 8080), ("https", 8443)): + with self.subTest(scheme=scheme): + gateway = Endpoint("public.example", 10000, scheme, True) + self.assertEqual(self.resolve(payload, gateway), ("10.0.0.4", expected)) + + @patch.object(sandbox, "api_endpoint_from_env", + return_value=Endpoint("public.example", 443, "https", False)) + def test_legacy_metadata_uses_configured_gateway_port(self, _server): + gateway = Endpoint("public.example", 8888, "http", True) + self.assertEqual(self.resolve({"pod_ip": "10.0.0.4"}, gateway), + ("10.0.0.4", 8888)) + + @patch.object(sandbox, "api_endpoint_from_env") + def test_cache_is_scoped_to_api_endpoint(self, server): + gateway = Endpoint("public.example", 80, "http", False) + for index in (1, 2): + server.return_value = Endpoint(f"cluster{index}", 443, "https", False) + ip = f"10.0.0.{index}" + self.assertEqual(self.resolve({"pod_ip": ip}, gateway), (ip, 80)) + + @patch.object(sandbox, "api_endpoint_from_env", + return_value=Endpoint("public.example", 443, "https", False)) + def test_invalid_port_is_rejected(self, _server): + gateway = Endpoint("public.example", 80, "http", False) + for port in (0, 65536, True, "8080"): + with self.subTest(port=port), self.assertRaises(RuntimeError): + self.resolve({"pod_ip": "10.0.0.4", "http_port": port}, gateway) diff --git a/src/sandboxd b/src/sandboxd index b892414..b5d25ee 160000 --- a/src/sandboxd +++ b/src/sandboxd @@ -1 +1 @@ -Subproject commit b892414a3b21f3c2a4481322767362af14fc701f +Subproject commit b5d25eede172983cbfef9975e919219a7643e658