From 7f09b8ff21549bff358eb1ed7b16907cf56d409f Mon Sep 17 00:00:00 2001 From: mhsong1998-dot <258010372+mhsong1998-dot@users.noreply.github.com> Date: Wed, 9 Sep 2026 12:35:00 +0800 Subject: [PATCH] feat(deploy): migrate to openyuanrong python cli Replace the legacy Go launcher with the Python CLI for master, frontend, node, and standalone while preserving fixed deployment paths and runtime configuration. Wait for the sandbox bridge before starting YuanRong, track foreground sandboxd through systemd, restore cluster DataSystem timeouts, and allow control-plane startup recovery. Place DataSystem health files under the log directory prepared by the Python CLI. Signed-off-by: mhsong1998-dot <258010372+mhsong1998-dot@users.noreply.github.com> --- AGENTS.md | 7 + ...penyuanrong-core-0.10.2rc2.constraints.txt | 30 +++ builder/config/yr/config.toml.jinja | 199 +++++++++++++++++ builder/node.Dockerfile | 38 +++- builder/scripts/akernel-entrypoint.sh | 22 +- builder/scripts/master_entrypoint.sh | 155 ++++++------- builder/scripts/sandboxd_network_ready.sh | 21 ++ builder/scripts/yr_node_bootstrap.sh | 206 +++++++++--------- builder/systemd_services/sandboxd.service | 3 +- builder/systemd_services/yuanrong.service | 5 +- .../templates/frontend/akernel_frontend.yaml | 26 ++- .../core/templates/master/akernel_master.yaml | 22 +- .../charts/core/templates/node/configmap.yaml | 2 +- .../charts/core/templates/node/daemonset.yaml | 23 +- deploy/akernel/charts/core/values.yaml | 7 +- .../terraform/aliyun/values-akernel.yaml.tmpl | 3 +- deploy/terraform/huaweicloud/main.tf | 6 +- .../huaweicloud/values-akernel.yaml.tmpl | 13 +- 18 files changed, 552 insertions(+), 236 deletions(-) create mode 100644 builder/config/openyuanrong-core-0.10.2rc2.constraints.txt create mode 100644 builder/config/yr/config.toml.jinja create mode 100755 builder/scripts/sandboxd_network_ready.sh diff --git a/AGENTS.md b/AGENTS.md index fca1d2a..c16e130 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -409,6 +409,13 @@ the sandbox bridge. YuanRong receives `INSTANCE_IP` in Kubernetes or the default-route interface address in standalone mode; `AKERNEL_NODE_IP` is the explicit override for multi-homed environments. +Node deployments keep openYuanRong's in-node address aligned with sandboxd's +actual bridge state. Kubernetes renders `node.sandboxIPRange` into sandboxd's +`ip_range`; standalone reads the same setting from `sandboxd_config.toml` when +preparing networking. The sandboxd systemd startup waits with a bounded +timeout for `sandbox0`; YuanRong starts afterward and passes the bridge's +assigned IPv4 address through `values.local_ip` in the Python CLI config. + The standalone sandboxd filestore is a loop-mounted ext4 image under the bind-mounted `deploy/standalone/data/` directory. Explicit `storage_mb` quotas for runsc and Firecracker use this local-disk filestore. Without an diff --git a/builder/config/openyuanrong-core-0.10.2rc2.constraints.txt b/builder/config/openyuanrong-core-0.10.2rc2.constraints.txt new file mode 100644 index 0000000..9a77ef5 --- /dev/null +++ b/builder/config/openyuanrong-core-0.10.2rc2.constraints.txt @@ -0,0 +1,30 @@ +# Runtime dependency lock for openyuanrong-core 0.10.2rc2 on Python 3.12. +# Regenerate and validate this file for both linux/amd64 and linux/arm64 when +# OPEN_YR_VERSION changes. +aiohappyeyeballs==2.7.1 +aiohttp==3.14.3 +aiosignal==1.4.0 +anyio==4.14.2 +attrs==26.1.0 +certifi==2026.7.22 +charset-normalizer==3.5.1 +click==8.4.2 +cloudpickle==3.1.2 +frozenlist==1.8.0 +h11==0.16.0 +httpcore==1.0.9 +httpx==0.28.1 +idna==3.19 +jinja2==3.1.6 +markupsafe==3.0.3 +msgpack==1.2.1 +multidict==6.7.1 +propcache==0.5.2 +protobuf==7.36.0 +pyyaml==6.0.3 +requests==2.34.2 +tomli-w==1.2.0 +typing-extensions==4.16.0 +urllib3==2.7.0 +websockets==17.0.1 +yarl==1.24.5 diff --git a/builder/config/yr/config.toml.jinja b/builder/config/yr/config.toml.jinja new file mode 100644 index 0000000..4e2fcde --- /dev/null +++ b/builder/config/yr/config.toml.jinja @@ -0,0 +1,199 @@ +{# + AKernel runtime profile template. + + Deployment values are resolved from the CLI-provided env mapping. Strings + containing Python CLI runtime expressions remain escaped for its second + rendering pass. +#} +{% macro toml_string(value) -%} +"{%- for char in value -%} +{%- set encoded = char | tojson -%} +{{- char if encoded | length == 14 else encoded[1:-1] -}} +{%- endfor -%}" +{%- endmacro %} +{% set role = env["AKERNEL_ROLE"] %} +{% set cluster_roles = ["master", "frontend"] %} +{% set node_roles = ["node", "standalone"] %} +{% if role not in cluster_roles + node_roles %} +{{ unsupported_akernel_role }} +{% endif %} +{% set is_cluster = role in cluster_roles %} +{% set is_node = role in node_roles %} +{% set is_agent = role == "node" %} +{% set is_standalone = role == "standalone" %} +{% set has_master_mode = not is_agent %} +{% set enable_metrics = env.get("ENABLE_METRICS", "false") | lower %} +{% set enable_trace = env.get("ENABLE_TRACE", "false") | lower %} +{% set traefik_enable_tls = "true" if env.get("TRAEFIK_ENABLE_TLS", "false") | lower == "true" else "false" %} +{% set configured_host_ip = env.get("YR_NODE_IP", "") | trim %} +{% set configured_local_ip = env.get("YR_LOCAL_IP", "") | trim %} +{% set enable_faas_frontend = env.get("ENABLE_FAAS_FRONTEND", "true") | lower %} +{% set enable_iam_server = env.get("ENABLE_IAM_SERVER", "true") | lower %} +{% set runtime_hostname = "{{ hostname }}" %} +{% set deploy_path_value = env["DEPLOY_PATH"] | trim %} +{% set component_log_path = env["YR_LOG_PATH"] | trim %} +{% set ds_worker_log_path = component_log_path ~ "/data_system/worker" %} + +# Generated at startup from builder/config/yr/config.toml.jinja. + +[values] +node_id = "{{ runtime_hostname }}" +deploy_path = {{ toml_string(deploy_path_value) }} +{% if configured_host_ip %} +host_ip = {{ toml_string(configured_host_ip) }} +{% endif %} +{% if configured_local_ip %} +local_ip = {{ toml_string(configured_local_ip) }} +{% endif %} +{% if is_cluster %} +cpu_num = 1 +memory_num = 3904 +shared_memory_num = 4096 +{% endif %} + +[values.fs.log] +path = {{ toml_string(component_log_path) }} + +[values.fs.tls] +base_path = "/home/yuanrong/.cert" + +{% if not is_standalone %} +[values.etcd] +enable_multi_master = true +{% endif %} + +{% if is_cluster or is_node %} +[[values.etcd.address]] +ip = {{ toml_string(configured_host_ip if is_standalone else env["ETCD_ADDRESS"]) }} +port = {{ env.get("ETCD_PORT", "2379") }} +peer_port = {{ env.get("ETCD_PEER_PORT", "2378") }} +{% endif %} + +[values.frontend] +ssl_enable = true +client_auth_type = "NoClientCert" +frontend_lease_bypass = true +enable_function_token_auth = true +enable_func_token_auth = true +{% if role == "frontend" %} +meta_service_address = {{ toml_string(env["META_SERVICE_ADDRESS"]) }} +{% endif %} +iam_server_address = "127.0.0.1:31113" + +[values.meta_service] +port = 31111 + +{% if has_master_mode %} +[mode.master] +etcd = {{ "true" if is_standalone else "false" }} +ds_master = {{ "true" if is_standalone else "false" }} +frontend = {{ enable_faas_frontend if role == "master" else "true" if role in ["frontend", "standalone"] else "false" }} +function_master = {{ "true" if role in ["master", "standalone"] else "false" }} +function_scheduler = false +meta_service = {{ "true" if role in ["master", "standalone"] else "false" }} +iam_server = {{ enable_iam_server if role == "master" else "true" if role in ["frontend", "standalone"] else "false" }} +{% endif %} + +[ds_worker.health_check] +endpoint = {{ toml_string(ds_worker_log_path ~ "/health") }} + +[ds_worker.args] +log_dir = {{ toml_string(ds_worker_log_path) }} +health_check_path = {{ toml_string(ds_worker_log_path ~ "/health") }} +ready_check_path = {{ toml_string(ds_worker_log_path ~ "/ready") }} +{% if is_cluster %} +rpc_thread_num = 128 +node_timeout_s = 10 +node_dead_timeout_s = 30 +heartbeat_interval_ms = 3000 +{% endif %} +{% if is_node %} +node_timeout_s = 30 +client_dead_timeout_s = 60 +heartbeat_interval_ms = 1000 +node_dead_timeout_s = 120 +{% endif %} + +[ds_master.args] +log_dir = {{ toml_string(component_log_path ~ "/data_system/master") }} + +[function_master.args] +services_path = "/home/yuanrong/deploy/process/services.yaml" +metrics_config_file = "/home/yuanrong/metrics/metrics_config.json" +traefik_enable_tls = {{ traefik_enable_tls }} +traefik_http_entry_point = {{ toml_string(env.get("TRAEFIK_HTTP_ENTRYPOINT", "websecure")) }} +traefik_forward_timeout_ms = 3000 +{% if enable_metrics == "true" and (is_cluster or is_standalone) %} +enable_metrics = true +{% endif %} +{% if enable_trace == "true" and (is_cluster or is_standalone) %} +enable_trace = true +trace_config = {{ toml_string(env["YR_TRACE_CONFIG_CONTENT"]) }} +{% endif %} +{% if is_cluster or is_standalone %} +enable_traefik_provider = {{ "true" if env.get("TRAEFIK_MODE", "etcd") == "http" else "false" }} +{% endif %} +{% if is_cluster %} +system_timeout = 300000 +schedule_relaxed = 20 +{% elif is_standalone %} +system_timeout = 60000 +{% endif %} + +[function_proxy.args] +services_path = "/home/yuanrong/deploy/process/services.yaml" +enable_inherit_env = false +npu_collection_mode = "off" +metrics_config_file = "/home/yuanrong/metrics/metrics_config.json" +enable_direct_routing = false +force_low_reliability_instance = true +traefik_enable_tls = {{ traefik_enable_tls }} +{% if enable_metrics == "true" %} +enable_metrics = true +{% endif %} +{% if enable_trace == "true" %} +enable_trace = true +trace_config = {{ toml_string(env["YR_TRACE_CONFIG_CONTENT"]) }} +{% endif %} +{% if is_node %} +enable_traefik_registry = {{ "true" if env.get("TRAEFIK_MODE", "etcd") == "etcd" else "false" }} +traefik_http_entrypoint = {{ toml_string(env.get("TRAEFIK_HTTP_ENTRYPOINT", "websecure")) }} +{% endif %} +log_expiration_enable = true +log_expiration_time_threshold = {{ 10 if is_agent else 7200 }} +log_expiration_cleanup_interval = {{ 10 if is_agent else 600 }} +log_expiration_max_file_count = {{ 50 if is_agent else 256 }} +{% if is_cluster %} +system_timeout = 300000 +pseudo_data_plane = true +{% elif is_node %} +system_timeout = 60000 +fc_agent_mgr_retry_times = 30 +fc_agent_mgr_retry_cycle = 60000 +{% endif %} +runtime_logs_dir = {{ toml_string(component_log_path) }} +{% if is_node %} +metrics_collector_type = "external" +snapshot_storage_mode = "local_only" +checkpoint_dir = "/home/akernel/checkpoints" +{% endif %} + +# The CLI exposes the meta-service component port separately from values.meta_service. +[meta_service] +port = 31111 + +[iam_server.args] +token_expired_time_span = 604800 +ssl_enable = true +iam_ssl_enable = true +local_listen_port = 31113 +local_ip = "127.0.0.1" +{% if enable_trace == "true" and (is_cluster or is_standalone) %} +enable_trace = true +{% endif %} + +{% if enable_trace == "true" and (is_cluster or is_standalone) %} +[frontend.env] +ENABLE_TRACE = "true" +TRACE_CONFIG = {{ toml_string(env["YR_TRACE_CONFIG_CONTENT"]) }} +{% endif %} diff --git a/builder/node.Dockerfile b/builder/node.Dockerfile index 19669be..c19f3a9 100644 --- a/builder/node.Dockerfile +++ b/builder/node.Dockerfile @@ -254,6 +254,7 @@ RUN apt-get update && \ procps \ python3 \ python3-pip \ + python3-venv \ systemd \ systemd-sysv \ tzdata \ @@ -296,6 +297,10 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ ENV YR_INSTALLATION_DIR=/home/yuanrong +ENV PATH=/opt/openyuanrong/bin:${PATH} + +COPY ./builder/config/yr/config.toml.jinja /etc/yuanrong/config.toml.jinja +COPY ./builder/config/openyuanrong-core-0.10.2rc2.constraints.txt /tmp/openyuanrong-core.constraints.txt # Install the complete, language-runtime-free openYuanRong control plane from # its checksum-pinned core wheel. A URL and checksum pair may override the @@ -325,21 +330,30 @@ RUN set -eux; \ test -z "${OPEN_YR_CORE_WHEEL_SHA256}"; \ fi; \ wheel="/tmp/${wheel_name}"; \ - target=/tmp/openyuanrong-core; \ curl -fSL --retry 10 --retry-delay 2 --retry-all-errors \ "${wheel_url}" -o "${wheel}"; \ echo "${wheel_sha} ${wheel}" | sha256sum -c -; \ - python3 -m pip install \ - --break-system-packages \ + python3 -m venv /opt/openyuanrong; \ + /opt/openyuanrong/bin/python -m pip install \ --no-cache-dir \ - --no-deps \ - --target "${target}" \ + --index-url "${PIP_INDEX_URL}" \ + --constraint /tmp/openyuanrong-core.constraints.txt \ "${wheel}"; \ - test -x "${target}/yr/functionsystem/bin/yr"; \ - mkdir -p "${YR_INSTALLATION_DIR}"; \ - cp -a "${target}/yr/." "${YR_INSTALLATION_DIR}/"; \ - rm -rf "${target}" "${wheel}"; \ - ln -sfn "${YR_INSTALLATION_DIR}/functionsystem/bin/yr" /usr/bin/yr + site_packages="$(/opt/openyuanrong/bin/python -c 'import site; print(site.getsitepackages()[0])')"; \ + base_py="${site_packages}/yr/cli/component/base.py"; \ + launcher_py="${site_packages}/yr/cli/system_launcher.py"; \ + grep -Fq 'logger.info(f"Environment: {full_env}")' "${base_py}"; \ + sed -i \ + 's/logger.info(f"Environment: {full_env}")/logger.info(f"Environment keys: {sorted(full_env)}")/' \ + "${base_py}"; \ + grep -Fq 'logger.info(f"Environment keys: {sorted(full_env)}")' "${base_py}"; \ + ! grep -Fq 'logger.info(f"Environment: {full_env}")' "${base_py}"; \ + grep -Fq '"env_vars": comp.env_vars,' "${launcher_py}"; \ + sed -i 's/"env_vars": comp.env_vars,/"env_vars": {},/' "${launcher_py}"; \ + grep -Fq '"env_vars": {},' "${launcher_py}"; \ + ! grep -Fq '"env_vars": comp.env_vars,' "${launcher_py}"; \ + test -x /opt/openyuanrong/bin/yr; \ + rm -f "${wheel}" /tmp/openyuanrong-core.constraints.txt COPY --from=runtime-image /yr-runtime-rootfs.img ${YR_INSTALLATION_DIR}/yr-runtime-rootfs.img @@ -358,6 +372,7 @@ RUN if [ "${AKERNEL_ENABLE_KATA}" = "true" ]; then \ COPY ./builder/scripts/akernel-entrypoint.sh /usr/local/bin/akernel-entrypoint COPY ./builder/scripts/ensure-component-cert.sh /usr/local/bin/ensure-component-cert COPY ./builder/scripts/sandboxd_network_prepare.sh /usr/local/bin/sandboxd-network-prepare +COPY ./builder/scripts/sandboxd_network_ready.sh /usr/local/bin/sandboxd-network-ready RUN chmod 0755 \ /usr/local/bin/runsc \ /usr/local/bin/sandboxd \ @@ -366,7 +381,8 @@ RUN chmod 0755 \ /usr/local/bin/distill_fs \ /usr/local/bin/akernel-entrypoint \ /usr/local/bin/ensure-component-cert \ - /usr/local/bin/sandboxd-network-prepare + /usr/local/bin/sandboxd-network-prepare \ + /usr/local/bin/sandboxd-network-ready RUN if [ "${AKERNEL_ENABLE_KATA}" = "true" ]; then chmod 0755 /usr/local/bin/containerd-shim-kata-v2; fi RUN if [ "${AKERNEL_ENABLE_RUNC}" = "true" ]; then \ chmod 0755 /usr/local/bin/runc /usr/local/bin/runc-shim; \ diff --git a/builder/scripts/akernel-entrypoint.sh b/builder/scripts/akernel-entrypoint.sh index 1df691c..c134180 100644 --- a/builder/scripts/akernel-entrypoint.sh +++ b/builder/scripts/akernel-entrypoint.sh @@ -8,12 +8,8 @@ set -euo pipefail role="${AKERNEL_ROLE:-}" if [ -z "${role}" ] && [ "$#" -gt 0 ]; then - case "$1" in - master|frontend|node|standalone) - role="$1" - shift - ;; - esac + role="$1" + shift fi if [ -z "${role}" ]; then @@ -25,6 +21,16 @@ if [ -z "${role}" ]; then fi fi +case "${role}" in + master|frontend|node|standalone) + ;; + *) + echo "unsupported AKERNEL_ROLE: ${role}; expected master, frontend, node, or standalone" >&2 + exit 1 + ;; +esac +export AKERNEL_ROLE="${role}" + case "${role}" in master|frontend) /usr/local/bin/ensure-component-cert @@ -38,8 +44,4 @@ case "${role}" in /usr/local/bin/ensure-component-cert exec /usr/sbin/init "$@" ;; - *) - echo "unsupported AKERNEL_ROLE: ${role}" >&2 - exit 1 - ;; esac diff --git a/builder/scripts/master_entrypoint.sh b/builder/scripts/master_entrypoint.sh index 862d1b6..b0dda3d 100644 --- a/builder/scripts/master_entrypoint.sh +++ b/builder/scripts/master_entrypoint.sh @@ -3,46 +3,94 @@ # Copyright (c) 2026 Ant Group Corporation. # # SPDX-License-Identifier: Apache-2.0 -set -e +set -euo pipefail ulimit -n 32768 -BASE_DIR=$( - cd "$(dirname "$0")" - pwd -) -export DEPLOY_PATH="/home/yuanrong/master/" -mkdir -p "$DEPLOY_PATH" -export YR_LOG_PATH="$DEPLOY_PATH/log" + +YR_CLI=/opt/openyuanrong/bin/yr +YR_CONFIG_TEMPLATE=/etc/yuanrong/config.toml.jinja +YR_CONFIG_PATH="${YR_RENDERED_CONFIG_PATH:-/run/yuanrong/config.toml}" +export DEPLOY_PATH="${DEPLOY_PATH:-/home/yuanrong/master}" +# The all-in-one image supplies the node log path; master/frontend use their +# own fixed role directory, as in the legacy launcher. +export YR_LOG_PATH="${DEPLOY_PATH}/log" export YR_IMAGE_PROCESS_CONFIG="${YR_IMAGE_PROCESS_CONFIG:-/run/akernel/yr-image-process.json}" -# If ConfigMap-mounted config exists, symlink it to override the baked-in default -[ -f /etc/otel-collector/otel_config.yaml ] && ln -sf /etc/otel-collector/otel_config.yaml /home/yuanrong/otel_config.yaml +if [ -z "${LITEBUS_DATA_KEY:-}" ]; then + echo "LITEBUS_DATA_KEY is required for akernel master/frontend" >&2 + exit 1 +fi -# otel watchdog: monitor and restart otelcol-contrib if it crashes +if [ ! -x "${YR_CLI}" ]; then + echo "yr binary not found or not executable: ${YR_CLI}" >&2 + exit 1 +fi + +if [ "${ENABLE_TRACE:-false}" = "true" ]; then + trace_config_file="${TRACE_CONFIG_FILE:-/home/yuanrong/trace/trace_config.json}" + if [ ! -r "${trace_config_file}" ]; then + echo "trace config file is not readable: ${trace_config_file}" >&2 + exit 1 + fi + YR_TRACE_CONFIG_CONTENT="$(cat "${trace_config_file}")" + export YR_TRACE_CONFIG_CONTENT +else + unset YR_TRACE_CONFIG_CONTENT +fi + +mkdir -p "$(dirname "${YR_CONFIG_PATH}")" +"${YR_CLI}" config render \ + -t "${YR_CONFIG_TEMPLATE}" \ + -o "${YR_CONFIG_PATH}" + +YR_CLI_ARGS=( + "${YR_CLI}" + --config "${YR_CONFIG_PATH}" + start --master --block true + --port-policy FIX + --function-proxy-merge-process-enable +) +if [ "${YR_CLI_DRY_RUN:-false}" = "true" ]; then + if [ -z "${YR_CLI_CAPTURE_FILE:-}" ]; then + echo "YR_CLI_CAPTURE_FILE is required when YR_CLI_DRY_RUN=true" >&2 + exit 1 + fi + mkdir -p "$(dirname "${YR_CLI_CAPTURE_FILE}")" + printf '%s\0' "${YR_CLI_ARGS[@]}" > "${YR_CLI_CAPTURE_FILE}" + exit 0 +fi + +mkdir -p "${DEPLOY_PATH}" "${YR_LOG_PATH}" + +# If ConfigMap-mounted config exists, symlink it to override the baked-in default. +[ -f /etc/otel-collector/otel_config.yaml ] && + ln -sf /etc/otel-collector/otel_config.yaml /home/yuanrong/otel_config.yaml + +# Monitor and restart the collector when observability is enabled. otel_watchdog() { - local otel_log="$DEPLOY_PATH/otelcol.log" + local otel_log="${DEPLOY_PATH}/otelcol.log" local max_restart_interval=60 local restart_count=0 while true; do - otelcol-contrib --config="/home/yuanrong/otel_config.yaml" >> "$otel_log" 2>&1 & + otelcol-contrib --config=/home/yuanrong/otel_config.yaml >> "${otel_log}" 2>&1 & local otel_pid=$! - echo $otel_pid > $DEPLOY_PATH/otelcol.pid - echo "[$(date '+%Y-%m-%d %H:%M:%S')] otelcol started, PID: $otel_pid (restart count: $restart_count)" >> "$otel_log" + echo "${otel_pid}" > "${DEPLOY_PATH}/otelcol.pid" + echo "[$(date '+%Y-%m-%d %H:%M:%S')] otelcol started, PID: ${otel_pid} (restart count: ${restart_count})" >> "${otel_log}" - wait $otel_pid - local exit_code=$? + local exit_code=0 + wait "${otel_pid}" || exit_code=$? restart_count=$((restart_count + 1)) - echo "[$(date '+%Y-%m-%d %H:%M:%S')] otelcol exited with code $exit_code, restarting in 5s (restart count: $restart_count)" >> "$otel_log" + echo "[$(date '+%Y-%m-%d %H:%M:%S')] otelcol exited with code ${exit_code}, restarting (restart count: ${restart_count})" >> "${otel_log}" - # Exponential backoff with a cap local delay=$((2 ** restart_count)) - [ $delay -gt $max_restart_interval ] && delay=$max_restart_interval - sleep $delay + [ "${delay}" -gt "${max_restart_interval}" ] && delay="${max_restart_interval}" + sleep "${delay}" done } export -f otel_watchdog -if { [ "${ENABLE_METRICS:-false}" = "true" ] || [ "${ENABLE_TRACE:-false}" = "true" ]; } && command -v otelcol-contrib >/dev/null 2>&1; then +if { [ "${ENABLE_METRICS:-false}" = "true" ] || [ "${ENABLE_TRACE:-false}" = "true" ]; } && + command -v otelcol-contrib >/dev/null 2>&1; then nohup bash -c otel_watchdog & echo "otelcol watchdog started" echo "otel log: ${DEPLOY_PATH}/otelcol.log" @@ -50,65 +98,4 @@ else echo "otelcol watchdog skipped" fi -# Set enable_traefik_provider based on TRAEFIK_MODE -if [ "${TRAEFIK_MODE:-etcd}" = "http" ]; then - ENABLE_TRAEFIK_PROVIDER=true -else - ENABLE_TRAEFIK_PROVIDER=false -fi - -if [ -z "${LITEBUS_DATA_KEY:-}" ]; then - echo "LITEBUS_DATA_KEY is required for akernel master/frontend" >&2 - exit 1 -fi - -YR_BIN="${YR_BIN:-/usr/bin/yr}" -if [ ! -x "${YR_BIN}" ]; then - echo "yr binary not found or not executable: ${YR_BIN}" >&2 - exit 1 -fi - -exec "${YR_BIN}" start --master --block true \ - -e -c 0 -m 8000 -s 4096 -n $HOSTNAME \ - -d $DEPLOY_PATH \ - --fs_health_check_retry_interval 1 \ - --schedule_relaxed 20 \ - --enable_faas_frontend ${ENABLE_FAAS_FRONTEND:-true} \ - --enable_function_scheduler ${ENABLE_FUNCTION_SCHEDULER:-false} \ - --enable_meta_service ${ENABLE_META_SERVICE:-true} \ - --enable_iam_server ${ENABLE_IAM_SERVER:-true} \ - --iam_token_expired_time_span 604800 \ - --ssl_base_path=/home/yuanrong/.cert/ \ - --frontend_ssl_enable=true \ - --frontend_client_auth_type NoClientCert \ - --enable_function_token_auth ${ENABLE_FUNCTION_TOKEN_AUTH:-true} \ - --enable_inherit_env false \ - --npu_collection_mode off \ - --port_policy FIX \ - --system_timeout 300000 \ - --enable_distributed_master false \ - --etcd_mode outter \ - --etcd_addr_list $ETCD_ADDRESS \ - --etcd_port ${ETCD_PORT} \ - --etcd_peer_port 2378 \ - --enable_metrics ${ENABLE_METRICS} \ - --metrics_config_file "/home/yuanrong/metrics/metrics_config.json" \ - --enable_trace ${ENABLE_TRACE} \ - --trace_config "$(cat /home/yuanrong/trace/trace_config.json)" \ - --ds_rpc_thread_num 128 \ - --function_proxy_merge_process_enable true \ - --force_low_reliability_instance true \ - --enable_traefik_provider=${ENABLE_TRAEFIK_PROVIDER} \ - --traefik_http_entry_point=${TRAEFIK_HTTP_ENTRYPOINT:-websecure} \ - --traefik_enable_tls=${TRAEFIK_ENABLE_TLS:-false} \ - --traefik_forward_timeout_ms=3000 \ - --frontend_lease_bypass true \ - --iam_ssl_enable true \ - --ssl_root_file ca.crt \ - --ssl_cert_file module.crt \ - --ssl_key_file module.key \ - --iam_local_listen_port 31113 \ - --iam_local_ip 127.0.0.1 \ - --enable_direct_routing false \ - --enable_sandbox_router true \ - ${META_SERVICE_ADDRESS:+--meta_service_address $META_SERVICE_ADDRESS} +exec "${YR_CLI_ARGS[@]}" diff --git a/builder/scripts/sandboxd_network_ready.sh b/builder/scripts/sandboxd_network_ready.sh new file mode 100755 index 0000000..81a1338 --- /dev/null +++ b/builder/scripts/sandboxd_network_ready.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Copyright (c) 2026 Ant Group Corporation. +# +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +for ((attempt = 0; attempt < 60; attempt++)); do + address="$( + { ip -4 -o address show dev sandbox0 scope global 2>/dev/null || true; } | + awk 'NR == 1 { split($4, value, "/"); print value[1] }' + )" + if [ -n "${address}" ]; then + echo "sandbox0 is ready with IPv4 address ${address}" + exit 0 + fi + sleep 1 +done + +echo "timed out after 60s waiting for sandbox0 to have an IPv4 address" >&2 +exit 1 diff --git a/builder/scripts/yr_node_bootstrap.sh b/builder/scripts/yr_node_bootstrap.sh index 4b8b8e4..bfbb65c 100755 --- a/builder/scripts/yr_node_bootstrap.sh +++ b/builder/scripts/yr_node_bootstrap.sh @@ -3,7 +3,13 @@ # Copyright (c) 2026 Ant Group Corporation. # # SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + ulimit -n 32768 + +YR_CLI=/opt/openyuanrong/bin/yr +YR_CONFIG_TEMPLATE=/etc/yuanrong/config.toml.jinja +YR_CONFIG_PATH="${YR_RENDERED_CONFIG_PATH:-/run/yuanrong/config.toml}" export YR_RUNTIME_BACKEND=sandboxd export YR_IMAGE_PROCESS_CONFIG="${YR_IMAGE_PROCESS_CONFIG:-/run/akernel/yr-image-process.json}" @@ -40,117 +46,111 @@ resolve_node_ip() { printf '%s\n' "${node_ip}" } +resolve_sandbox_ip() { + local address + + address="$( + ip -4 -o address show dev sandbox0 scope global 2>/dev/null | + awk 'NR == 1 { split($4, value, "/"); print value[1] }' + )" + if [ -z "${address}" ]; then + echo "sandbox0 has no global IPv4 address after sandboxd startup" >&2 + return 1 + fi + printf '%s\n' "${address}" +} + YR_NODE_IP="$(resolve_node_ip)" +export YR_NODE_IP echo "Using ${YR_NODE_IP} as the YuanRong node address" CHECKPOINT_DIR="/home/akernel/checkpoints" mkdir -p "${CHECKPOINT_DIR}" -# Select the legacy etcd registry or the FunctionMaster HTTP provider. -if [ "${TRAEFIK_MODE:-etcd}" = "etcd" ]; then - ENABLE_TRAEFIK_REGISTRY=${ENABLE_TRAEFIK_REGISTRY:-true} - ENABLE_TRAEFIK_PROVIDER=false -else - ENABLE_TRAEFIK_REGISTRY=false - ENABLE_TRAEFIK_PROVIDER=true -fi +YR_LOCAL_IP="$(resolve_sandbox_ip)" +export YR_LOCAL_IP +echo "Using ${YR_LOCAL_IP} as the YuanRong sandbox-local service address" -if [ "x${AKS_LOCAL_MODE}" == "xtrue" ]; then - if [ -z "${LITEBUS_DATA_KEY:-}" ] && [ -r /home/akernel/iam-seed ]; then - LITEBUS_DATA_KEY="$(tr -d '[:space:]' < /home/akernel/iam-seed)" - export LITEBUS_DATA_KEY +role="${AKERNEL_ROLE:-}" +if [ -z "${role}" ]; then + if [ "${AKS_LOCAL_MODE:-false}" = "true" ]; then + role=standalone + else + role=node fi - if [ -z "${LITEBUS_DATA_KEY:-}" ]; then - echo "LITEBUS_DATA_KEY is required in standalone mode" >&2 + export AKERNEL_ROLE="${role}" +fi + +case "${role}" in + node) + ;; + standalone) + if [ -z "${LITEBUS_DATA_KEY:-}" ] && [ -r /home/akernel/iam-seed ]; then + LITEBUS_DATA_KEY="$(tr -d '[:space:]' < /home/akernel/iam-seed)" + export LITEBUS_DATA_KEY + fi + if [ -z "${LITEBUS_DATA_KEY:-}" ]; then + echo "LITEBUS_DATA_KEY is required in standalone mode" >&2 + exit 1 + fi + ;; + *) + echo "AKERNEL_ROLE must be node or standalone" >&2 + exit 1 + ;; +esac + +export DEPLOY_PATH="${DEPLOY_PATH:-/home/yuanrong}" +export YR_LOG_PATH="${YR_LOG_PATH:-${DEPLOY_PATH}/logs}" + +if [ ! -f "${YR_CONFIG_TEMPLATE}" ]; then + echo "YuanRong CLI config template not found: ${YR_CONFIG_TEMPLATE}" >&2 + exit 1 +fi +if [ ! -x "${YR_CLI}" ]; then + echo "YuanRong CLI not executable: ${YR_CLI}" >&2 + exit 1 +fi + +if [ "${ENABLE_TRACE:-false}" = "true" ]; then + trace_config_file="${TRACE_CONFIG_FILE:-/home/yuanrong/trace/trace_config.json}" + if [ ! -r "${trace_config_file}" ]; then + echo "trace config file is not readable: ${trace_config_file}" >&2 exit 1 fi - /usr/bin/yr start --master \ - --ip_address "${YR_NODE_IP}" \ - --port_policy FIX \ - --enable_function_scheduler=false \ - --enable_faas_frontend=true \ - --enable_meta_service=true \ - --enable_iam_server=true \ - --iam_token_expired_time_span 604800 \ - --ssl_base_path=/home/yuanrong/.cert/ \ - --frontend_ssl_enable=true \ - --frontend_client_auth_type NoClientCert \ - --enable_function_token_auth true \ - --ds_node_timeout_s 30 \ - --ds_client_dead_timeout_s 60 \ - --ds_heartbeat_interval_ms 1000 \ - --ds_node_dead_timeout_s 120 \ - --system_timeout 60000 \ - --block true \ - --etcd_port ${ETCD_PORT:-2379} \ - --etcd_peer_port ${ETCD_PEER_PORT:-2378} \ - --enable_inherit_env false \ - --npu_collection_mode off \ - --enable_distributed_master false \ - --metrics_collector_type external \ - --enable_traefik_registry=${ENABLE_TRAEFIK_REGISTRY} \ - --enable_traefik_provider=${ENABLE_TRAEFIK_PROVIDER} \ - --traefik_enable_tls=${TRAEFIK_ENABLE_TLS:-false} \ - --traefik_etcd_prefix=traefik \ - --traefik_lease_ttl=300000 \ - --traefik_http_entrypoint=${TRAEFIK_HTTP_ENTRYPOINT:-websecure} \ - --traefik_http_entry_point=${TRAEFIK_HTTP_ENTRYPOINT:-websecure} \ - --enable_metrics ${ENABLE_METRICS} \ - --metrics_config_file "/home/yuanrong/metrics/metrics_config.json" \ - --enable_trace ${ENABLE_TRACE} \ - --trace_config "$(cat /home/yuanrong/trace/trace_config.json)" \ - --log_root "${YR_LOG_PATH}" \ - --function_proxy_merge_process_enable true \ - --fc_agent_mgr_retry_times 30 \ - --fc_agent_mgr_retry_cycle 60000 \ - --iam_ssl_enable true \ - --ssl_root_file ca.crt \ - --ssl_cert_file module.crt \ - --ssl_key_file module.key \ - --iam_local_listen_port 31113 \ - --iam_local_ip 127.0.0.1 \ - --frontend_lease_bypass true \ - --force_low_reliability_instance true \ - --snapshot_storage_mode local_only \ - --checkpoint_dir "${CHECKPOINT_DIR}" \ - --enable_sandbox_router true \ - --enable_direct_routing false + YR_TRACE_CONFIG_CONTENT="$(cat "${trace_config_file}")" + export YR_TRACE_CONFIG_CONTENT else - /usr/bin/yr start \ - --ip_address "${YR_NODE_IP}" \ - --port_policy FIX \ - --ds_node_timeout_s 30 \ - --ds_client_dead_timeout_s 60 \ - --ds_heartbeat_interval_ms 1000 \ - --ds_node_dead_timeout_s 120 \ - --etcd_addr_list ${ETCD_ADDRESS} \ - --etcd_mode outter \ - --etcd_port ${ETCD_PORT} \ - --etcd_peer_port ${ETCD_PEER_PORT:-2378} \ - --system_timeout 60000 \ - --enable_inherit_env false \ - --npu_collection_mode off \ - --enable_distributed_master false \ - --metrics_collector_type external \ - --enable_metrics ${ENABLE_METRICS} \ - --metrics_config_file "/home/yuanrong/metrics/metrics_config.json" \ - --enable_trace ${ENABLE_TRACE} \ - --trace_config "$(cat /home/yuanrong/trace/trace_config.json)" \ - -n ${HOSTNAME} \ - --enable_traefik_registry=${ENABLE_TRAEFIK_REGISTRY} \ - --traefik_enable_tls=${TRAEFIK_ENABLE_TLS:-false} \ - --traefik_etcd_prefix=traefik \ - --traefik_lease_ttl=300000 \ - --traefik_http_entrypoint=${TRAEFIK_HTTP_ENTRYPOINT:-websecure} \ - --log_root "${YR_LOG_PATH}" \ - --fc_agent_mgr_retry_times 30 \ - --fc_agent_mgr_retry_cycle 60000 \ - --log_expiration_time_threshold 10 \ - --log_expiration_cleanup_interval 10 \ - --log_expiration_max_file_count 50 \ - --function_proxy_merge_process_enable true \ - --enable_direct_routing false \ - --force_low_reliability_instance true \ - --snapshot_storage_mode local_only \ - --checkpoint_dir "${CHECKPOINT_DIR}" \ - --block true + unset YR_TRACE_CONFIG_CONTENT fi + +mkdir -p "$(dirname "${YR_CONFIG_PATH}")" +"${YR_CLI}" config render \ + -t "${YR_CONFIG_TEMPLATE}" \ + -o "${YR_CONFIG_PATH}" + +YR_CLI_ARGS=( + "${YR_CLI}" + --config "${YR_CONFIG_PATH}" + start +) +if [ "${role}" = "standalone" ]; then + YR_CLI_ARGS+=(--master) +fi +YR_CLI_ARGS+=( + --block true + --port-policy FIX + --function-proxy-merge-process-enable +) + +if [ "${YR_CLI_DRY_RUN:-false}" = "true" ]; then + if [ -z "${YR_CLI_CAPTURE_FILE:-}" ]; then + echo "YR_CLI_CAPTURE_FILE is required when YR_CLI_DRY_RUN=true" >&2 + exit 1 + fi + mkdir -p "$(dirname "${YR_CLI_CAPTURE_FILE}")" + printf '%s\0' "${YR_CLI_ARGS[@]}" > "${YR_CLI_CAPTURE_FILE}" + exit 0 +fi + +mkdir -p "${YR_LOG_PATH:-/home/yuanrong/logs}" +exec "${YR_CLI_ARGS[@]}" diff --git a/builder/systemd_services/sandboxd.service b/builder/systemd_services/sandboxd.service index 7642142..5a366a1 100644 --- a/builder/systemd_services/sandboxd.service +++ b/builder/systemd_services/sandboxd.service @@ -4,7 +4,7 @@ After=network-online.target Before=yuanrong.service [Service] -PIDFile=/var/run/sandboxd.pid +Type=simple PassEnvironment=NODE_NAME KUBERNETES_SERVICE_HOST KUBERNETES_SERVICE_PORT Environment=GOGC=300 @@ -17,6 +17,7 @@ ExecStart=/bin/bash -ce "exec /usr/local/bin/sandboxd \ -log-level debug \ -log-file /home/akernel/logs/sandboxd/sandboxd.log \ -config /home/akernel/sandboxd/config.toml" +ExecStartPost=/usr/local/bin/sandboxd-network-ready KillMode=process Restart=always diff --git a/builder/systemd_services/yuanrong.service b/builder/systemd_services/yuanrong.service index 4572f5b..34d9bbd 100644 --- a/builder/systemd_services/yuanrong.service +++ b/builder/systemd_services/yuanrong.service @@ -1,10 +1,12 @@ [Unit] Description=yuanrong.service +Requires=sandboxd.service +After=sandboxd.service [Service] #Type=simple PIDFile=/run/yuanrong.pid -PassEnvironment=ETCD_PORT ETCD_PEER_PORT ETCD_ADDRESS HOSTNAME AKS_LOCAL_MODE AKERNEL_NODE_IP INSTANCE_IP LITEBUS_DATA_KEY YR_LOG_PATH YR_INSTALLATION_DIR YR_RRT_CONTROL_SOCKET_PATH YR_IMAGE_PROCESS_CONFIG ENABLE_METRICS ENABLE_TRACE TRAEFIK_MODE TRAEFIK_ENABLE_TLS TRAEFIK_HTTP_ENTRYPOINT +PassEnvironment=AKERNEL_ROLE AKERNEL_NODE_IP INSTANCE_IP ETCD_ADDRESS ETCD_PORT ETCD_PEER_PORT HOSTNAME AKS_LOCAL_MODE LITEBUS_DATA_KEY DEPLOY_PATH YR_LOG_PATH YR_INSTALLATION_DIR YR_RRT_CONTROL_SOCKET_PATH YR_IMAGE_PROCESS_CONFIG ENABLE_METRICS ENABLE_TRACE TRACE_CONFIG_FILE TRAEFIK_MODE TRAEFIK_ENABLE_TLS TRAEFIK_HTTP_ENTRYPOINT Environment="CONTAINER_EP=unix:///run/sandboxd/sandboxd.sock" Environment="RUNTIME_HOME_DIR=/home/yuanrong/runtime" Environment="YR_NOSET_CUDA_VISIBLE_DEVICES=1" @@ -14,6 +16,7 @@ ExecStart=/usr/bin/bash /home/yuanrong/yr_node_bootstrap.sh ExecReload=/bin/kill -15 $MAINPID KillMode=control-group Restart=always +RestartSec=5s Delegate=yes UMask=000 TasksMax=infinity diff --git a/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml b/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml index 2b02967..472555a 100644 --- a/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml +++ b/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml @@ -94,18 +94,8 @@ spec: secretKeyRef: name: {{ include "core.litebusSecretName" . }} key: litebus-data-key - - name: ENABLE_FUNCTION_MASTER - value: "false" - - name: ENABLE_FUNCTION_SCHEDULER - value: "false" - - name: ENABLE_META_SERVICE - value: "false" - - name: ENABLE_IAM_SERVER - value: "true" - name: META_SERVICE_ADDRESS value: {{ .Values.frontend.master.metaServiceAddress | default (printf "akernel-master.%s.svc.cluster.local:31111" .Release.Namespace) | quote }} - - name: IAM_SERVER_ADDRESS - value: {{ .Values.frontend.master.iamServerAddress | default (printf "akernel-master.%s.svc.cluster.local:31112" .Release.Namespace) | quote }} - name: ETCD_ADDRESS value: {{ get $frontendEtcd "host" | default (printf "akernel-etcd.%s.svc.cluster.local" .Release.Namespace) | quote }} - name: ETCD_PORT @@ -124,13 +114,25 @@ spec: {{- end }} - name: AKERNEL_ENV value: {{ .Values.monitoring.akernelEnv | default "default" | quote }} + {{- if .Values.monitoring.prometheusEndpoint }} - name: ENABLE_METRICS - value: {{ ne (.Values.monitoring.prometheusEndpoint | default "") "" | ternary "true" "false" | quote }} + value: "true" + {{- end }} + {{- if .Values.monitoring.tempoEndpoint }} - name: ENABLE_TRACE - value: {{ ne (.Values.monitoring.tempoEndpoint | default "") "" | ternary "true" "false" | quote }} + value: "true" + {{- end }} ports: - name: http containerPort: 8888 + startupProbe: + httpGet: + path: /healthz + port: 8888 + scheme: HTTPS + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 18 livenessProbe: httpGet: path: /healthz diff --git a/deploy/akernel/charts/core/templates/master/akernel_master.yaml b/deploy/akernel/charts/core/templates/master/akernel_master.yaml index de37591..9f873fe 100644 --- a/deploy/akernel/charts/core/templates/master/akernel_master.yaml +++ b/deploy/akernel/charts/core/templates/master/akernel_master.yaml @@ -108,16 +108,24 @@ spec: {{- end }} - name: AKERNEL_ENV value: {{ .Values.monitoring.akernelEnv | default "default" | quote }} + {{- if .Values.monitoring.prometheusEndpoint }} - name: ENABLE_METRICS - value: {{ ne (.Values.monitoring.prometheusEndpoint | default "") "" | ternary "true" "false" | quote }} + value: "true" + {{- end }} + {{- if .Values.monitoring.tempoEndpoint }} - name: ENABLE_TRACE - value: {{ ne (.Values.monitoring.tempoEndpoint | default "") "" | ternary "true" "false" | quote }} + value: "true" + {{- end }} - name: TRAEFIK_MODE value: {{ .Values.traefik.mode | default "http" | quote }} + {{- if and (not (.Values.traefik.enableWebEntrypoint | default false)) (.Values.traefik.enableTLS | default false) }} - name: TRAEFIK_ENABLE_TLS - value: {{ .Values.traefik.enableWebEntrypoint | default false | ternary false (.Values.traefik.enableTLS | default false) | quote }} + value: "true" + {{- end }} + {{- if .Values.traefik.enableWebEntrypoint }} - name: TRAEFIK_HTTP_ENTRYPOINT - value: {{ .Values.traefik.enableWebEntrypoint | default false | ternary "web" "websecure" | quote }} + value: "web" + {{- end }} ports: - name: http containerPort: 8888 @@ -125,6 +133,12 @@ spec: containerPort: 31111 - name: global-sched containerPort: 22770 + startupProbe: + tcpSocket: + port: 22770 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 18 livenessProbe: tcpSocket: port: 22770 diff --git a/deploy/akernel/charts/core/templates/node/configmap.yaml b/deploy/akernel/charts/core/templates/node/configmap.yaml index bfc8989..b2d643f 100644 --- a/deploy/akernel/charts/core/templates/node/configmap.yaml +++ b/deploy/akernel/charts/core/templates/node/configmap.yaml @@ -13,7 +13,7 @@ data: CPUQuota={{ .Values.node.config.resourceControl.cpuQuota | default "800%" }} sandboxd_config.toml: | -{{- $sandboxdConfig := .Values.node.config.sandboxd.config }} +{{- $sandboxdConfig := tpl .Values.node.config.sandboxd.config . }} {{- $runcMarker := "# AKERNEL_RUNTIME_RUNC" }} {{- if .Values.node.config.sandboxd.enableRunc }} {{- if not (contains $runcMarker $sandboxdConfig) }} diff --git a/deploy/akernel/charts/core/templates/node/daemonset.yaml b/deploy/akernel/charts/core/templates/node/daemonset.yaml index 01eb11d..f3b7041 100644 --- a/deploy/akernel/charts/core/templates/node/daemonset.yaml +++ b/deploy/akernel/charts/core/templates/node/daemonset.yaml @@ -59,6 +59,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} {{- $nodeEtcd := .Values.node.etcd | default dict }} + {{- $sandboxResolverIP := first (splitList "/" .Values.node.sandboxIPRange) }} containers: - name: akernel-node image: "{{ include "core.image" (dict "root" . "image" .Values.node.image) }}" @@ -72,7 +73,11 @@ spec: - bash - -c - | - cp /etc/resolv.conf /etc/resolv_akernel.conf && sed -i 's/127.0.0.1/10.88.0.1/g' /etc/resolv_akernel.conf + sandbox_resolver_ip={{ $sandboxResolverIP | quote }} + awk -v resolver="${sandbox_resolver_ip}" ' + $1 == "nameserver" && $2 ~ /^127\./ { $2 = resolver } + { print } + ' /etc/resolv.conf > /etc/resolv_akernel.conf resources: {{- toYaml .Values.node.resources | nindent 10 }} env: @@ -122,16 +127,24 @@ spec: {{- end }} - name: AKERNEL_ENV value: {{ .Values.monitoring.akernelEnv | default "default" | quote }} + {{- if .Values.monitoring.prometheusEndpoint }} - name: ENABLE_METRICS - value: {{ ne (.Values.monitoring.prometheusEndpoint | default "") "" | ternary "true" "false" | quote }} + value: "true" + {{- end }} + {{- if .Values.monitoring.tempoEndpoint }} - name: ENABLE_TRACE - value: {{ ne (.Values.monitoring.tempoEndpoint | default "") "" | ternary "true" "false" | quote }} + value: "true" + {{- end }} - name: TRAEFIK_MODE value: {{ .Values.traefik.mode | default "http" | quote }} + {{- if and (not (.Values.traefik.enableWebEntrypoint | default false)) (.Values.traefik.enableTLS | default false) }} - name: TRAEFIK_ENABLE_TLS - value: {{ .Values.traefik.enableWebEntrypoint | default false | ternary false (.Values.traefik.enableTLS | default false) | quote }} + value: "true" + {{- end }} + {{- if .Values.traefik.enableWebEntrypoint }} - name: TRAEFIK_HTTP_ENTRYPOINT - value: {{ .Values.traefik.enableWebEntrypoint | default false | ternary "web" "websecure" | quote }} + value: "web" + {{- end }} securityContext: privileged: true volumeMounts: diff --git a/deploy/akernel/charts/core/values.yaml b/deploy/akernel/charts/core/values.yaml index ab638fd..2136d83 100644 --- a/deploy/akernel/charts/core/values.yaml +++ b/deploy/akernel/charts/core/values.yaml @@ -123,6 +123,11 @@ frontend: iamServerAddress: "" node: + # sandboxd allocates sandbox addresses from this CIDR. + sandboxIPRange: "10.88.0.1/16" + etcd: + host: "" + port: "2379" secret: create: true oss_auths: {} @@ -459,7 +464,7 @@ node: stream_server_port="" [plugin.network] - ip_range="10.88.0.1/16" + ip_range="{{ .Values.node.sandboxIPRange }}" nat_backend="iptables" enable_network_acl=true diff --git a/deploy/terraform/aliyun/values-akernel.yaml.tmpl b/deploy/terraform/aliyun/values-akernel.yaml.tmpl index 0ce6d32..51359ac 100644 --- a/deploy/terraform/aliyun/values-akernel.yaml.tmpl +++ b/deploy/terraform/aliyun/values-akernel.yaml.tmpl @@ -77,6 +77,7 @@ frontend: %{ endif ~} node: + sandboxIPRange: "10.88.0.1/16" affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -156,7 +157,7 @@ node: stream_server_port="" [plugin.network] - ip_range="10.88.0.1/16" + ip_range="{{ .Values.node.sandboxIPRange }}" nat_backend="${sandboxd_nat_backend}" enable_network_acl=true diff --git a/deploy/terraform/huaweicloud/main.tf b/deploy/terraform/huaweicloud/main.tf index da5e56b..5eeff98 100644 --- a/deploy/terraform/huaweicloud/main.tf +++ b/deploy/terraform/huaweicloud/main.tf @@ -39,8 +39,10 @@ locals { } : {} oss_auths = merge(local.generated_oss_auths, var.oss_auths) registry_auths = { - auths = { for host, cred in var.registry_auths : host => { username = cred.username, password = cred.password } } + auths = { for host, cred in var.registry_auths : host => { auth = base64encode("${cred.username}:${cred.password}") } } } + registry_auths_enabled = length(var.registry_auths) > 0 + dockerconfigjson = base64encode(jsonencode(local.registry_auths)) # When auto-creating ELB on Huawei Cloud CCE, inject required annotations # so the cloud-controller-manager provisions the ELB automatically. @@ -119,6 +121,8 @@ locals { node_home_csi_size = var.node_home_csi_size oss_auths = local.oss_auths registry_auths = local.registry_auths + registry_auths_enabled = local.registry_auths_enabled + dockerconfigjson = local.dockerconfigjson etcd_cpu = var.etcd_resources.cpu etcd_memory = var.etcd_resources.memory diff --git a/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl b/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl index 89c3602..7fa2941 100644 --- a/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl +++ b/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl @@ -1,3 +1,13 @@ +%{ if registry_auths_enabled ~} +imagePullSecrets: + - name: registry-secret + +imagePullSecret: + create: true + name: "registry-secret" + dockerconfigjson: "${dockerconfigjson}" +%{ endif ~} + kruise: enabled: ${enable_kruise} @@ -68,6 +78,7 @@ frontend: %{ endif ~} node: + sandboxIPRange: "10.88.0.1/16" image: repository: "${node_image_repository}" tag: "${node_image_tag}" @@ -136,7 +147,7 @@ node: stream_server_port="" [plugin.network] - ip_range="10.88.0.1/16" + ip_range="{{ .Values.node.sandboxIPRange }}" nat_backend="${sandboxd_nat_backend}" enable_network_acl=true