From d6621390a9a4206521737c01e93ebebdd7a9654b Mon Sep 17 00:00:00 2001 From: Kyle Felter Date: Thu, 9 Jul 2026 10:30:51 -0500 Subject: [PATCH 1/8] feat: Integrate REST services into DevSpace Signed-off-by: Kyle Felter --- dev/deployment/devspace/README.md | 62 ++++- dev/deployment/devspace/bootstrap-prereqs.sh | 196 ++++++++++++++++ .../devspace/setup-rest-integration.sh | 195 ++++++++++++++++ devspace.yaml | 220 +++++++++++++++++- rest-api/scripts/setup-local.sh | 83 +++++-- 5 files changed, 729 insertions(+), 27 deletions(-) create mode 100755 dev/deployment/devspace/setup-rest-integration.sh diff --git a/dev/deployment/devspace/README.md b/dev/deployment/devspace/README.md index 57793b8c7a..b62c073d5b 100644 --- a/dev/deployment/devspace/README.md +++ b/dev/deployment/devspace/README.md @@ -1,6 +1,6 @@ # Local Development with DevSpace -You can use [DevSpace](https://www.devspace.sh) to deploy infra-controller core locally using mock hosts. +You can use [DevSpace](https://www.devspace.sh) to deploy the complete local infra-controller stack. The deployment connects the REST services to the real Core gRPC API, while `machine-a-tron` supplies the mock hosts. The process is broken into two steps: @@ -14,6 +14,8 @@ The intent is that the app deploy path stays the same whether the prerequisites ## Prerequisites Bootstrap +The full-stack path requires kind with a current kube context named `kind-`. The bootstrap and deploy steps stop before applying the REST stack when another context is selected. + Run: ```bash @@ -27,12 +29,16 @@ By default this script assumes an empty cluster and will idempotently: - deploy a simple PostgreSQL instance - deploy a simple Vault dev server - configure Vault mounts and a local PKI role +- create a separate REST database in the local PostgreSQL instance +- deploy Temporal and create its `cloud` and `site` namespaces +- deploy the local Keycloak realm +- share the Core CA with REST so the site agent can use mTLS with Core - create the Secrets and ConfigMaps that the Helm chart expects - write [`values.generated.yaml`](values.generated.yaml) for the app deploy step It is safe to re-run. It uses `helm upgrade --install`, `kubectl apply`, and Vault checks before writing mounts/roles/secrets. -The bootstrap script is responsible for cluster-facing dependencies and generated wiring only. The repo deploy step does not install PostgreSQL, Vault, or cert-manager. +The bootstrap script is responsible for cluster-facing dependencies and generated wiring only. The repo deploy step does not install PostgreSQL, Vault, cert-manager, Temporal, or Keycloak. ### Bring Your Own @@ -42,6 +48,7 @@ Examples: ```bash LOCAL_DEV_INSTALL_POSTGRES=0 \ +LOCAL_DEV_INSTALL_REST_PREREQS=0 \ LOCAL_DEV_POSTGRES_HOST=my-postgres.postgres.svc.cluster.local \ LOCAL_DEV_POSTGRES_PORT=5432 \ LOCAL_DEV_POSTGRES_DB=nico \ @@ -63,6 +70,7 @@ dev/deployment/devspace/bootstrap-prereqs.sh ```bash LOCAL_DEV_INSTALL_CERT_MANAGER=0 \ LOCAL_DEV_INSTALL_LOCAL_ISSUER=0 \ +LOCAL_DEV_INSTALL_REST_PREREQS=0 \ LOCAL_DEV_CERT_ISSUER_KIND=ClusterIssuer \ LOCAL_DEV_CERT_ISSUER_NAME=my-existing-issuer \ LOCAL_DEV_CERT_ISSUER_GROUP=cert-manager.io \ @@ -74,6 +82,10 @@ Important: - The script writes the generated Helm values file from these settings. - For local Vault, the app uses root-token auth by setting `automountServiceAccountToken: false`. - For external Vault, either keep `VAULT_AUTH_MODE=root-token` or supply your own compatible auth setup. +- `LOCAL_DEV_INSTALL_TEMPORAL=0` and `LOCAL_DEV_INSTALL_KEYCLOAK=0` skip those managed services. +- `LOCAL_DEV_INSTALL_REST_PREREQS=0` preserves the Core-only bootstrap behavior. +- A full-stack deployment expects PostgreSQL at `postgres.postgres.svc.cluster.local` and requires the `nico_rest`, `keycloak`, `temporal`, and `temporal_visibility` databases and roles when the local PostgreSQL installation is skipped. A nondefault PostgreSQL host is supported only by the Core-only path. +- The Core and REST services share one PostgreSQL server but use separate `nico` and `nico_rest` databases because both schemas contain tables such as `machines` and `instances`. ## Build And Deploy @@ -86,9 +98,12 @@ devspace deploy DevSpace will: - build the local runtime images from [`Dockerfile.api`](Dockerfile.api), [`Dockerfile.bmc-proxy`](Dockerfile.bmc-proxy), and [`Dockerfile.machine-a-tron`](Dockerfile.machine-a-tron) +- build the REST API, workflow, site-manager, site-agent, database migration, certificate-manager, and MCP images from [`rest-api/docker/local`](../../../rest-api/docker/local) - deploy the Helm chart in [`helm/`](../../../helm) +- deploy the REST umbrella, site-agent, and MCP charts in [`helm/rest`](../../../helm/rest) - apply the local-only `machine-a-tron` Kubernetes objects from [`machine-a-tron.yaml`](machine-a-tron.yaml) with `kubectl` - inject the built image names and DevSpace-generated tags into both deployments at runtime +- register a local REST site, configure its Temporal namespace, and confirm that the site agent establishes a Core gRPC connection The image builds are configured in [`devspace.yaml`](../../../devspace.yaml). The Dockerfiles are multi-stage builds: the builder stage compiles the Rust binary inside Docker from the local `build-container-localdev` image, and the runtime stage copies only the finished binary and required runtime assets. DevSpace first checks whether `build-container-localdev` already exists locally and reuses it if present; otherwise it builds it from [`dev/docker/Dockerfile.build-container-x86_64`](../../../dev/docker/Dockerfile.build-container-x86_64). BuildKit cache mounts are used for Cargo registry, Cargo git checkouts, and Cargo target output so rebuilds stay fast without copying host build artifacts into the image. @@ -96,6 +111,8 @@ The DevSpace images also use Dockerfile-specific ignore files: [`Dockerfile.api. DevSpace watches the Rust workspace, toolchain metadata, and the runtime Dockerfiles to decide when images need rebuilding. +The REST images use the existing local Dockerfiles and one shared per-run tag because the REST Helm charts intentionally consume a common repository and tag. On kind clusters, the pre-deploy hooks load all Core and REST images into the cluster selected by the current kube context. + The production Helm chart is still only responsible for the product services. `machine-a-tron` is deployed separately as plain local-only Kubernetes objects in [`machine-a-tron.yaml`](machine-a-tron.yaml), with DevSpace wiring in the local image tag and certificate issuer from [`devspace.yaml`](../../../devspace.yaml). The local API and BMC proxy configs in [`values.base.yaml`](values.base.yaml) point BMC traffic at `machine-a-tron-bmc-mock.nico-system.svc.cluster.local:1266`. Common usage: @@ -106,6 +123,34 @@ devspace deploy -n nico-system devspace deploy --force-build ``` +The post-deploy setup uses temporary port-forwards to register the site and verifies that machines from Core are visible through the REST API. To keep the REST API and Keycloak available on localhost after `devspace deploy` exits, run these in separate terminals: + +```bash +kubectl -n nico-rest port-forward service/nico-rest-api 18388:8388 +kubectl -n nico-rest port-forward service/keycloak 18082:8082 +``` + +Then acquire a local token and list the machines discovered through `machine-a-tron`: + +```bash +TOKEN=$(curl -fsS -X POST http://localhost:18082/realms/nico-dev/protocol/openid-connect/token \ + -H 'Content-Type: application/x-www-form-urlencoded' \ + -d 'client_id=nico-api' \ + -d 'client_secret=nico-local-secret' \ + -d 'grant_type=password' \ + -d 'username=admin@example.com' \ + -d 'password=adminpassword' | jq -r .access_token) +curl -fsS http://localhost:18388/v2/org/test-org/nico/machine \ + -H "Authorization: Bearer ${TOKEN}" | jq +``` + +To run the original Core-only deployment, skip the REST prerequisites during bootstrap and use the `core-only` profile: + +```bash +LOCAL_DEV_INSTALL_REST_PREREQS=0 dev/deployment/devspace/bootstrap-prereqs.sh +devspace deploy --profile core-only +``` + ## Manual Equivalent If you want to understand what DevSpace is doing for the runtime images, the configured build is effectively: @@ -117,27 +162,27 @@ docker build -t "nico-bmc-proxy:" -f dev/deployment/devs docker build -t "machine-a-tron:" -f dev/deployment/devspace/Dockerfile.machine-a-tron . ``` -DevSpace then deploys the Helm chart with the built `nico-api` image wired into `global.image.repository` and `global.image.tag`, the built `nico-bmc-proxy` image wired into the `nico-bmc-proxy` chart values, and applies the local-only `machine-a-tron` manifest with its image wired into the `Deployment` spec. +DevSpace then deploys the Helm chart with the built `nico-api` image wired into `global.image.repository` and `global.image.tag`, the built `nico-bmc-proxy` image wired into the `nico-bmc-proxy` chart values, and applies the local-only `machine-a-tron` manifest with its image wired into the `Deployment` spec. The REST images are built from the existing `rest-api/docker/local` Dockerfiles and are passed to the three existing REST Helm charts with the same generated tag. -## Re-initializing infra-controller to a clean slate +## Re-initializing Core state Once deployed, the `nico-api` container will run and initialize its database, and the `machine-a-tron` container will run a set of mock machines, which will be discovered and ingested into the database, and run through the state machine until they reach a Ready state. -You can start over again (purging the resources from k8s) by running: +You can purge the deployed product resources by running: ```bash devspace purge -n nico-system ``` -and it will delete the NICo Helm release and machine-a-tron deployments. +and it will delete the Core and REST Helm releases and the machine-a-tron deployment. Bootstrap-owned services such as PostgreSQL, Vault, Temporal, Keycloak, and cert-manager remain installed. -To clear out the nico database to start from scratch again, run the nuke-postgres.sh helper script: +To clear only the Core `nico` database, run the nuke-postgres.sh helper script: ```bash dev/deployment/devspace/nuke-postgres.sh ``` -and the postgres database will be reset to an empty state, allowing you to deploy again: +This helper does not reset the REST, Keycloak, or Temporal databases, the REST site registration, or Temporal namespaces. After resetting Core state, deploy again with: ```bash devspace deploy -n nico-system @@ -146,6 +191,7 @@ devspace deploy -n nico-system ## Files - [`bootstrap-prereqs.sh`](bootstrap-prereqs.sh) +- [`setup-rest-integration.sh`](setup-rest-integration.sh) - [`devspace.yaml`](../../../devspace.yaml) - [`values.base.yaml`](values.base.yaml) - [`values.generated.yaml`](values.generated.yaml) diff --git a/dev/deployment/devspace/bootstrap-prereqs.sh b/dev/deployment/devspace/bootstrap-prereqs.sh index 5da40723d2..e1d73b12ff 100755 --- a/dev/deployment/devspace/bootstrap-prereqs.sh +++ b/dev/deployment/devspace/bootstrap-prereqs.sh @@ -27,6 +27,9 @@ INSTALL_CERT_MANAGER="${LOCAL_DEV_INSTALL_CERT_MANAGER:-1}" INSTALL_LOCAL_ISSUER="${LOCAL_DEV_INSTALL_LOCAL_ISSUER:-1}" INSTALL_POSTGRES="${LOCAL_DEV_INSTALL_POSTGRES:-1}" INSTALL_VAULT="${LOCAL_DEV_INSTALL_VAULT:-1}" +INSTALL_REST_PREREQS="${LOCAL_DEV_INSTALL_REST_PREREQS:-1}" +INSTALL_TEMPORAL="${LOCAL_DEV_INSTALL_TEMPORAL:-1}" +INSTALL_KEYCLOAK="${LOCAL_DEV_INSTALL_KEYCLOAK:-1}" POSTGRES_NAMESPACE="${POSTGRES_NAMESPACE:-postgres}" POSTGRES_HOST="${LOCAL_DEV_POSTGRES_HOST:-postgres.${POSTGRES_NAMESPACE}.svc.cluster.local}" @@ -36,6 +39,12 @@ POSTGRES_USER="${LOCAL_DEV_POSTGRES_USER:-nico}" POSTGRES_PASSWORD="${LOCAL_DEV_POSTGRES_PASSWORD:-nico}" POSTGRES_SSL_MODE="${LOCAL_DEV_POSTGRES_SSL_MODE:-disable}" +REST_NAMESPACE="nico-rest" +REST_POSTGRES_DB="${LOCAL_DEV_REST_POSTGRES_DB:-nico_rest}" +REST_POSTGRES_USER="${LOCAL_DEV_REST_POSTGRES_USER:-nico_rest}" +REST_POSTGRES_PASSWORD="${LOCAL_DEV_REST_POSTGRES_PASSWORD:-nico_rest}" +TEMPORAL_NAMESPACE="temporal" + VAULT_NAMESPACE="${VAULT_NAMESPACE:-vault}" VAULT_ADDR="${LOCAL_DEV_VAULT_ADDR:-http://vault.${VAULT_NAMESPACE}.svc.cluster.local:8200}" VAULT_TOKEN="${LOCAL_DEV_VAULT_TOKEN:-root}" @@ -59,6 +68,66 @@ require_bin() { } } +require_kind_context_for_rest() { + if [[ "${INSTALL_REST_PREREQS}" != "1" ]]; then + return + fi + + local current_context + current_context="$(kubectl config current-context 2>/dev/null || true)" + case "${current_context}" in + kind-*) ;; + *) + printf 'REST prerequisites require a kind context; current context is %s\n' \ + "${current_context:-unset}" >&2 + exit 1 + ;; + esac +} + +rest_postgres_sql() { + printf '%s\n' \ + 'DO $$' \ + 'BEGIN' \ + " IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '${REST_POSTGRES_USER}') THEN" \ + " EXECUTE format('CREATE ROLE %I LOGIN PASSWORD %L', '${REST_POSTGRES_USER}', '${REST_POSTGRES_PASSWORD}');" \ + ' END IF;' \ + " EXECUTE format('ALTER ROLE %I WITH LOGIN PASSWORD %L', '${REST_POSTGRES_USER}', '${REST_POSTGRES_PASSWORD}');" \ + " IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'keycloak') THEN" \ + " CREATE ROLE keycloak LOGIN PASSWORD 'keycloak';" \ + ' END IF;' \ + " ALTER ROLE keycloak WITH LOGIN PASSWORD 'keycloak';" \ + " IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'temporal') THEN" \ + " CREATE ROLE temporal LOGIN PASSWORD 'temporal' CREATEDB;" \ + ' END IF;' \ + " ALTER ROLE temporal WITH LOGIN PASSWORD 'temporal' CREATEDB;" \ + 'END' \ + '$$;' \ + "SELECT format('CREATE DATABASE %I OWNER %I', '${REST_POSTGRES_DB}', '${REST_POSTGRES_USER}')" \ + "WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = '${REST_POSTGRES_DB}')" \ + '\gexec' \ + "\\connect ${REST_POSTGRES_DB}" \ + "GRANT ALL ON SCHEMA public TO ${REST_POSTGRES_USER};" \ + 'CREATE EXTENSION IF NOT EXISTS pg_trgm;' \ + '\connect postgres' \ + "SELECT 'CREATE DATABASE keycloak OWNER keycloak'" \ + "WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'keycloak')" \ + '\gexec' \ + '\connect keycloak' \ + 'GRANT ALL ON SCHEMA public TO keycloak;' \ + '\connect postgres' \ + "SELECT 'CREATE DATABASE temporal OWNER temporal'" \ + "WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'temporal')" \ + '\gexec' \ + "SELECT 'CREATE DATABASE temporal_visibility OWNER temporal'" \ + "WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'temporal_visibility')" \ + '\gexec' \ + '\connect temporal' \ + 'GRANT ALL ON SCHEMA public TO temporal;' \ + '\connect temporal_visibility' \ + 'GRANT ALL ON SCHEMA public TO temporal;' +} + install_cert_manager() { if [[ "${INSTALL_CERT_MANAGER}" != "1" ]]; then return @@ -130,10 +199,19 @@ EOF } apply_local_postgres() { + local rest_postgres_init="" + if [[ "${INSTALL_POSTGRES}" != "1" ]]; then return fi + if [[ "${INSTALL_REST_PREREQS}" == "1" ]]; then + rest_postgres_init="$( + printf ' 002-create-rest-databases.sql: |\n' + rest_postgres_sql | sed 's/^/ /' + )" + fi + log "Applying local PostgreSQL deployment" kubectl apply -f - </dev/null + + if [[ "${INSTALL_REST_PREREQS}" == "1" ]]; then + log "Ensuring REST, Temporal, and Keycloak databases exist" + rest_postgres_sql | kubectl exec -i -n "${POSTGRES_NAMESPACE}" statefulset/postgres -- \ + psql -v ON_ERROR_STOP=1 -U postgres -d postgres >/dev/null + fi } apply_local_vault() { @@ -393,6 +478,112 @@ data: EOF } +apply_rest_ca() { + local ca_crt_b64="" + local ca_key_b64="" + + if [[ "${INSTALL_REST_PREREQS}" != "1" ]]; then + return + fi + + ca_crt_b64="$(kubectl get secret nico-local-ca -n "${NAMESPACE}" -o jsonpath='{.data.tls\.crt}')" + ca_key_b64="$(kubectl get secret nico-local-ca -n "${NAMESPACE}" -o jsonpath='{.data.tls\.key}')" + if [[ -z "${ca_crt_b64}" || -z "${ca_key_b64}" ]]; then + printf 'missing local Core CA secret in namespace %s\n' "${NAMESPACE}" >&2 + exit 1 + fi + + log "Sharing the local Core CA with REST services" + kubectl apply -f - </dev/null 2>&1; then + kubectl apply -f - </dev/null +} + +install_temporal() { + if [[ "${INSTALL_REST_PREREQS}" != "1" || "${INSTALL_TEMPORAL}" != "1" ]]; then + return + fi + + log "Installing Temporal" + kubectl create namespace "${TEMPORAL_NAMESPACE}" --dry-run=client -o yaml | kubectl apply -f - >/dev/null + kubectl apply -k "${REPO_ROOT}/rest-api/deploy/kustomize/base/temporal-helm" >/dev/null + kubectl wait --for=condition=Ready certificate/server-interservice-cert \ + -n "${TEMPORAL_NAMESPACE}" --timeout=240s >/dev/null + kubectl wait --for=condition=Ready certificate/server-cloud-cert \ + -n "${TEMPORAL_NAMESPACE}" --timeout=240s >/dev/null + kubectl wait --for=condition=Ready certificate/server-site-cert \ + -n "${TEMPORAL_NAMESPACE}" --timeout=240s >/dev/null + + helm upgrade --install temporal "${REPO_ROOT}/rest-api/temporal-helm/temporal" \ + --namespace "${TEMPORAL_NAMESPACE}" \ + --values "${REPO_ROOT}/rest-api/temporal-helm/temporal/values-kind.yaml" \ + --wait \ + --timeout 16m >/dev/null + + kubectl rollout status deployment/temporal-frontend -n "${TEMPORAL_NAMESPACE}" --timeout=360s >/dev/null + kubectl rollout status deployment/temporal-history -n "${TEMPORAL_NAMESPACE}" --timeout=360s >/dev/null + kubectl rollout status deployment/temporal-matching -n "${TEMPORAL_NAMESPACE}" --timeout=360s >/dev/null + kubectl rollout status deployment/temporal-worker -n "${TEMPORAL_NAMESPACE}" --timeout=360s >/dev/null + + for temporal_namespace in cloud site; do + "${REPO_ROOT}/rest-api/scripts/setup-local.sh" temporal-namespace "${temporal_namespace}" + done +} + +install_keycloak() { + if [[ "${INSTALL_REST_PREREQS}" != "1" || "${INSTALL_KEYCLOAK}" != "1" ]]; then + return + fi + + log "Installing Keycloak" + kubectl apply -k "${REPO_ROOT}/rest-api/deploy/kustomize/base/keycloak" >/dev/null + kubectl rollout status deployment/keycloak -n "${REST_NAMESPACE}" --timeout=240s >/dev/null +} + write_generated_values() { local disable_tls_enforcement="" local automount="true" @@ -447,6 +638,7 @@ Generated values: ${VALUES_FILE} Postgres endpoint: ${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} Vault address: ${VAULT_ADDR} Cert issuer: ${CERT_ISSUER_KIND}/${CERT_ISSUER_NAME} +REST prerequisites: ${INSTALL_REST_PREREQS} Next step: cd ${REPO_ROOT} && devspace deploy -n ${NAMESPACE} @@ -457,6 +649,7 @@ main() { require_bin kubectl require_bin helm require_bin base64 + require_kind_context_for_rest install_cert_manager apply_core_objects @@ -464,6 +657,9 @@ main() { apply_local_vault apply_local_issuer sync_nico_roots_secret + apply_rest_ca + install_temporal + install_keycloak write_generated_values print_summary } diff --git a/dev/deployment/devspace/setup-rest-integration.sh b/dev/deployment/devspace/setup-rest-integration.sh new file mode 100755 index 0000000000..75542276e6 --- /dev/null +++ b/dev/deployment/devspace/setup-rest-integration.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../../.." && pwd)" + +CORE_NAMESPACE="${LOCAL_DEV_NAMESPACE:-nico-system}" +REST_NAMESPACE="nico-rest" +API_FORWARD_PORT="${LOCAL_DEV_REST_API_FORWARD_PORT:-18388}" +KEYCLOAK_FORWARD_PORT="${LOCAL_DEV_KEYCLOAK_FORWARD_PORT:-18082}" +WORK_DIR="${LOCAL_DEV_REST_WORK_DIR:-${HOME}/Developer/_agent-tmp/devspace-rest}" + +api_forward_pid="" +keycloak_forward_pid="" + +cleanup() { + if [[ -n "${api_forward_pid}" ]]; then + kill "${api_forward_pid}" >/dev/null 2>&1 || true + fi + if [[ -n "${keycloak_forward_pid}" ]]; then + kill "${keycloak_forward_pid}" >/dev/null 2>&1 || true + fi +} + +require_bin() { + command -v "$1" >/dev/null 2>&1 || { + printf 'missing required binary: %s\n' "$1" >&2 + exit 1 + } +} + +trap cleanup EXIT INT TERM + +require_bin curl +require_bin jq +require_bin kubectl +require_bin base64 + +mkdir -p "${WORK_DIR}" + +kubectl rollout status deployment/nico-api -n "${CORE_NAMESPACE}" --timeout=300s >/dev/null +kubectl rollout status deployment/nico-rest-api -n "${REST_NAMESPACE}" --timeout=300s >/dev/null +kubectl rollout status deployment/nico-rest-cert-manager -n "${REST_NAMESPACE}" --timeout=300s >/dev/null +kubectl rollout status deployment/nico-rest-cloud-worker -n "${REST_NAMESPACE}" --timeout=300s >/dev/null +kubectl rollout status deployment/nico-rest-site-worker -n "${REST_NAMESPACE}" --timeout=300s >/dev/null +kubectl rollout status deployment/nico-rest-site-manager -n "${REST_NAMESPACE}" --timeout=300s >/dev/null +kubectl rollout status deployment/nico-mcp -n "${REST_NAMESPACE}" --timeout=300s >/dev/null +kubectl wait --for=condition=Ready certificate/core-grpc-client-site-agent-certs \ + -n "${REST_NAMESPACE}" --timeout=240s >/dev/null + +kubectl port-forward --address 127.0.0.1 -n "${REST_NAMESPACE}" \ + service/nico-rest-api "${API_FORWARD_PORT}:8388" \ + >"${WORK_DIR}/api-port-forward.log" 2>&1 & +api_forward_pid=$! + +kubectl port-forward --address 127.0.0.1 -n "${REST_NAMESPACE}" \ + service/keycloak "${KEYCLOAK_FORWARD_PORT}:8082" \ + >"${WORK_DIR}/keycloak-port-forward.log" 2>&1 & +keycloak_forward_pid=$! + +curl --fail --silent --show-error --retry 120 --retry-connrefused --retry-delay 1 \ + --max-time 5 "http://localhost:${KEYCLOAK_FORWARD_PORT}/realms/nico-dev" >/dev/null +curl --fail --silent --show-error --retry 120 --retry-connrefused --retry-delay 1 \ + --max-time 5 "http://localhost:${API_FORWARD_PORT}/healthz" >/dev/null + +api_forward_log="$(<"${WORK_DIR}/api-port-forward.log")" +keycloak_forward_log="$(<"${WORK_DIR}/keycloak-port-forward.log")" +if ! kill -0 "${api_forward_pid}" >/dev/null 2>&1 || \ + [[ "${api_forward_log}" != *"Forwarding from 127.0.0.1:${API_FORWARD_PORT}"* ]]; then + printf 'REST API port-forward failed; port %s may already be in use\n' \ + "${API_FORWARD_PORT}" >&2 + sed -n '1,120p' "${WORK_DIR}/api-port-forward.log" >&2 + exit 1 +fi +if ! kill -0 "${keycloak_forward_pid}" >/dev/null 2>&1 || \ + [[ "${keycloak_forward_log}" != *"Forwarding from 127.0.0.1:${KEYCLOAK_FORWARD_PORT}"* ]]; then + printf 'Keycloak port-forward failed; port %s may already be in use\n' \ + "${KEYCLOAK_FORWARD_PORT}" >&2 + sed -n '1,120p' "${WORK_DIR}/keycloak-port-forward.log" >&2 + exit 1 +fi + +setup_started_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + +NAMESPACE="${REST_NAMESPACE}" \ +API_URL="http://localhost:${API_FORWARD_PORT}" \ +KEYCLOAK_URL="http://localhost:${KEYCLOAK_FORWARD_PORT}" \ +RESTART_API=0 \ + "${REPO_ROOT}/rest-api/scripts/setup-local.sh" site-agent + +kubectl rollout status statefulset/nico-rest-site-agent \ + -n "${REST_NAMESPACE}" --timeout=300s >/dev/null + +site_agent_logs="" +core_connected=0 +for attempt in {1..60}; do + site_agent_logs="$(kubectl logs statefulset/nico-rest-site-agent \ + -n "${REST_NAMESPACE}" --since-time="${setup_started_at}" 2>/dev/null || true)" + case "${site_agent_logs}" in + *"CoreGrpcClient: Successfully connected to server"*) + core_connected=1 + break + ;; + esac + sleep 5 +done +if [[ "${core_connected}" != "1" ]]; then + printf 'site-agent did not establish a Core gRPC connection\n' >&2 + printf '%s\n' "${site_agent_logs}" >&2 + exit 1 +fi + +site_id="$(kubectl get configmap nico-rest-site-agent-config -n "${REST_NAMESPACE}" \ + -o jsonpath='{.data.CLUSTER_ID}')" +secret_site_id="$(kubectl get secret site-registration -n "${REST_NAMESPACE}" \ + -o jsonpath='{.data.site-uuid}' 2>/dev/null | base64 -d 2>/dev/null || true)" +if [[ -z "${site_id}" || "${site_id}" == "00000000-0000-4000-8000-000000000001" || \ + "${secret_site_id}" != "${site_id}" ]]; then + printf 'site-agent ConfigMap and registration Secret do not contain the same registered site ID\n' >&2 + exit 1 +fi + +inventory_started_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + +site_ready=false +machines_ready=false +machine_count=0 +fresh_cycle=false +for attempt in {1..60}; do + site_ready=false + machines_ready=false + machine_count=0 + token="$(curl --fail --silent --max-time 5 -X POST \ + "http://localhost:${KEYCLOAK_FORWARD_PORT}/realms/nico-dev/protocol/openid-connect/token" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "client_id=nico-api" \ + -d "client_secret=nico-local-secret" \ + -d "grant_type=password" \ + -d "username=admin@example.com" \ + -d "password=adminpassword" 2>/dev/null | jq -er '.access_token' 2>/dev/null || true)" + if [[ -n "${token}" ]]; then + site="$(curl --fail --silent --max-time 5 \ + "http://localhost:${API_FORWARD_PORT}/v2/org/test-org/nico/site/${site_id}" \ + -H "Authorization: Bearer ${token}" 2>/dev/null || true)" + machines="$(curl --fail --silent --max-time 5 \ + "http://localhost:${API_FORWARD_PORT}/v2/org/test-org/nico/machine?siteId=${site_id}&isMissingOnSite=false&pageSize=100" \ + -H "Authorization: Bearer ${token}" 2>/dev/null || true)" + site_ready="$(jq -r --arg site_id "${site_id}" \ + '.id == $site_id and .isOnline == true and .status == "Registered"' \ + <<<"${site}" 2>/dev/null || printf 'false')" + machines_ready="$(jq -r --arg site_id "${site_id}" \ + 'type == "array" and length > 0 and all(.[]; .siteId == $site_id)' \ + <<<"${machines}" 2>/dev/null || printf 'false')" + machine_count="$(jq -r 'if type == "array" then length else 0 end' \ + <<<"${machines}" 2>/dev/null || printf '0')" + fi + + cloud_worker_logs="$(kubectl logs deployment/nico-rest-cloud-worker \ + -n "${REST_NAMESPACE}" --since-time="${inventory_started_at}" 2>/dev/null || true)" + fresh_cycle="$(jq -Rrs --arg site_id "${site_id}" ' + [splits("\n") | fromjson? + | select(.Activity == "UpdateMachinesInDB" and .["Site ID"] == $site_id) + | .msg] | + reduce .[] as $msg ( + {saw_last_nonempty_page: false, complete: false}; + if ($msg | startswith("Received Machine inventory page:")) then + ($msg | capture("^Received Machine inventory page: (?[0-9]+) of (?[0-9]+), page size: [0-9]+, total count: (?[0-9]+)$")) as $page | + .saw_last_nonempty_page = ( + ($page.current | tonumber) == ($page.pages | tonumber) and + ($page.items | tonumber) > 0 + ) + elif .saw_last_nonempty_page and $msg == "completed activity" then + .complete = true + else . end + ) | .complete + ' <<<"${cloud_worker_logs}" 2>/dev/null || printf 'false')" + + if [[ "${site_ready}" == "true" && "${machines_ready}" == "true" && \ + "${fresh_cycle}" == "true" ]]; then + printf 'REST API reports site %s online with %s machine(s) from a current Core inventory\n' \ + "${site_id}" "${machine_count}" + break + fi + if [[ "${attempt}" == "60" ]]; then + printf 'REST integration verification failed: site_ready=%s machines_ready=%s machines=%s fresh_cycle=%s\n' \ + "${site_ready}" "${machines_ready}" "${machine_count}" "${fresh_cycle}" >&2 + exit 1 + fi + sleep 5 +done + +printf 'REST integration setup complete\n' diff --git a/devspace.yaml b/devspace.yaml index ef66eff130..0a48f30dd1 100644 --- a/devspace.yaml +++ b/devspace.yaml @@ -14,6 +14,21 @@ vars: LOCAL_DEV_CERT_ISSUER_GROUP: source: env default: cert-manager.io + LOCAL_DEV_REST_POSTGRES_DB: + source: env + default: nico_rest + LOCAL_DEV_REST_POSTGRES_USER: + source: env + default: nico_rest + LOCAL_DEV_REST_POSTGRES_PASSWORD: + source: env + default: nico_rest + LOCAL_DEV_REST_API_FORWARD_PORT: + source: env + default: "18388" + LOCAL_DEV_KEYCLOAK_FORWARD_PORT: + source: env + default: "18082" images: nico-api: @@ -65,6 +80,55 @@ images: - include - dev/docker/Dockerfile.build-container-x86_64 - dev/deployment/devspace/Dockerfile.machine-a-tron + nico-rest-api: + image: localhost:5000/nico-rest-api + tags: + - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_RANDOM} + dockerfile: rest-api/docker/local/Dockerfile.nico-rest-api + context: rest-api + skipPush: true + nico-rest-workflow: + image: localhost:5000/nico-rest-workflow + tags: + - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_RANDOM} + dockerfile: rest-api/docker/local/Dockerfile.nico-rest-workflow + context: rest-api + skipPush: true + nico-rest-site-manager: + image: localhost:5000/nico-rest-site-manager + tags: + - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_RANDOM} + dockerfile: rest-api/docker/local/Dockerfile.nico-rest-site-manager + context: rest-api + skipPush: true + nico-rest-site-agent: + image: localhost:5000/nico-rest-site-agent + tags: + - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_RANDOM} + dockerfile: rest-api/docker/local/Dockerfile.nico-rest-site-agent + context: rest-api + skipPush: true + nico-rest-db: + image: localhost:5000/nico-rest-db + tags: + - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_RANDOM} + dockerfile: rest-api/docker/local/Dockerfile.nico-rest-db + context: rest-api + skipPush: true + nico-rest-cert-manager: + image: localhost:5000/nico-rest-cert-manager + tags: + - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_RANDOM} + dockerfile: rest-api/docker/local/Dockerfile.nico-rest-cert-manager + context: rest-api + skipPush: true + nico-mcp: + image: localhost:5000/nico-mcp + tags: + - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_RANDOM} + dockerfile: rest-api/docker/local/Dockerfile.nico-mcp + context: rest-api + skipPush: true deployments: carbide-local: @@ -115,6 +179,105 @@ deployments: op: replace path: spec.issuerRef.group value: ${LOCAL_DEV_CERT_ISSUER_GROUP} + nico-rest: + namespace: nico-rest + updateImageTags: false + helm: + chart: + path: ./helm/rest/nico-rest + values: + global: + image: + repository: localhost:5000 + tag: ${runtime.images.nico-rest-api.tag} + pullPolicy: Never + imagePullSecrets: [] + certificate: + issuerRef: + kind: ClusterIssuer + name: nico-rest-ca-issuer + group: cert-manager.io + nico-rest-common: + secrets: + dbCreds: + username: ${LOCAL_DEV_REST_POSTGRES_USER} + password: ${LOCAL_DEV_REST_POSTGRES_PASSWORD} + nico-rest-api: + nodePort: + enabled: true + port: 30388 + config: + db: + host: postgres.postgres.svc.cluster.local + name: ${LOCAL_DEV_REST_POSTGRES_DB} + user: ${LOCAL_DEV_REST_POSTGRES_USER} + keycloak: + enabled: true + baseURL: http://keycloak:8082 + externalBaseURL: http://localhost:${LOCAL_DEV_KEYCLOAK_FORWARD_PORT} + realm: nico-dev + clientID: nico-api + serviceAccount: true + nico-rest-workflow: + secrets: + dbCreds: db-creds + config: + db: + host: postgres.postgres.svc.cluster.local + name: ${LOCAL_DEV_REST_POSTGRES_DB} + user: ${LOCAL_DEV_REST_POSTGRES_USER} + nico-rest-db: + db: + host: postgres.postgres.svc.cluster.local + name: ${LOCAL_DEV_REST_POSTGRES_DB} + user: ${LOCAL_DEV_REST_POSTGRES_USER} + nico-rest-site-agent: + namespace: nico-rest + updateImageTags: false + helm: + chart: + path: ./helm/rest/nico-rest-site-agent + values: + global: + image: + repository: localhost:5000 + tag: ${runtime.images.nico-rest-api.tag} + pullPolicy: Never + imagePullSecrets: [] + certificate: + issuerRef: + kind: ClusterIssuer + name: nico-rest-ca-issuer + group: cert-manager.io + certificate: + uris: + - spiffe://nico.local/nico-system/sa/elektra-site-agent + envConfig: + NICO_ADDRESS: nico-api.${LOCAL_DEV_NAMESPACE}.svc.cluster.local:1079 + NICO_SEC_OPT: "2" + CORE_GRPC_ADDRESS: nico-api.${LOCAL_DEV_NAMESPACE}.svc.cluster.local:1079 + CORE_GRPC_SEC_OPT: "2" + FLOW_GRPC_ENABLED: "false" + CLUSTER_ID: 00000000-0000-4000-8000-000000000001 + TEMPORAL_SERVER: site.server.temporal.local + TEMPORAL_SUBSCRIBE_NAMESPACE: 00000000-0000-4000-8000-000000000001 + TEMPORAL_SUBSCRIBE_QUEUE: site + nico-mcp: + namespace: nico-rest + updateImageTags: false + helm: + chart: + path: ./helm/rest/nico-mcp + values: + global: + image: + repository: localhost:5000 + tag: ${runtime.images.nico-rest-api.tag} + pullPolicy: Never + imagePullSecrets: [] + config: + baseURL: http://nico-rest-api:8388 + apiName: nico hooks: - name: load-images-into-local-cluster @@ -124,10 +287,65 @@ hooks: case "$CONTEXT" in kind-*) echo "Loading images into kind cluster..." - kind load docker-image \ + kind load docker-image --name "${CONTEXT#kind-}" \ "${runtime.images.nico-api.image}:${runtime.images.nico-api.tag}" \ "${runtime.images.nico-bmc-proxy.image}:${runtime.images.nico-bmc-proxy.tag}" \ "${runtime.images.machine-a-tron.image}:${runtime.images.machine-a-tron.tag}" ;; # Add new local K8s tools here esac + - name: load-rest-images-into-local-cluster + events: ["before:deploy"] + command: |- + CONTEXT=$(kubectl config current-context 2>/dev/null || echo "") + case "$CONTEXT" in + kind-*) + echo "Loading REST images into kind cluster..." + kind load docker-image --name "${CONTEXT#kind-}" \ + "${runtime.images.nico-rest-api.image}:${runtime.images.nico-rest-api.tag}" \ + "${runtime.images.nico-rest-workflow.image}:${runtime.images.nico-rest-workflow.tag}" \ + "${runtime.images.nico-rest-site-manager.image}:${runtime.images.nico-rest-site-manager.tag}" \ + "${runtime.images.nico-rest-site-agent.image}:${runtime.images.nico-rest-site-agent.tag}" \ + "${runtime.images.nico-rest-db.image}:${runtime.images.nico-rest-db.tag}" \ + "${runtime.images.nico-rest-cert-manager.image}:${runtime.images.nico-rest-cert-manager.tag}" \ + "${runtime.images.nico-mcp.image}:${runtime.images.nico-mcp.tag}" + ;; + *) + echo "Full-stack deployment requires a kind context; current context is ${CONTEXT:-unset}" >&2 + exit 1 + ;; + esac + - name: setup-rest-integration + events: ["after:deploy"] + command: |- + LOCAL_DEV_REST_API_FORWARD_PORT=${LOCAL_DEV_REST_API_FORWARD_PORT} \ + LOCAL_DEV_KEYCLOAK_FORWARD_PORT=${LOCAL_DEV_KEYCLOAK_FORWARD_PORT} \ + dev/deployment/devspace/setup-rest-integration.sh + +profiles: + - name: core-only + patches: + - op: remove + path: images.nico-rest-api + - op: remove + path: images.nico-rest-workflow + - op: remove + path: images.nico-rest-site-manager + - op: remove + path: images.nico-rest-site-agent + - op: remove + path: images.nico-rest-db + - op: remove + path: images.nico-rest-cert-manager + - op: remove + path: images.nico-mcp + - op: remove + path: deployments.nico-rest + - op: remove + path: deployments.nico-rest-site-agent + - op: remove + path: deployments.nico-mcp + - op: remove + path: hooks.name=load-rest-images-into-local-cluster + - op: remove + path: hooks.name=setup-rest-integration diff --git a/rest-api/scripts/setup-local.sh b/rest-api/scripts/setup-local.sh index 1da2a09b2b..c056b3bd79 100755 --- a/rest-api/scripts/setup-local.sh +++ b/rest-api/scripts/setup-local.sh @@ -8,6 +8,7 @@ NAMESPACE="${NAMESPACE:-nico-rest}" API_URL="${API_URL:-http://localhost:8388}" KEYCLOAK_URL="${KEYCLOAK_URL:-http://localhost:8082}" ORG="${ORG:-test-org}" +RESTART_API="${RESTART_API:-1}" usage() { echo "Usage: $0 " @@ -15,6 +16,7 @@ usage() { echo "Commands:" echo " pki Setup PKI secrets and CA" echo " site-agent Setup site-agent with a real site" + echo " temporal-namespace Ensure a Temporal namespace exists" echo " all Run both pki and site-agent setup" echo " verify Verify local deployment health" exit 1 @@ -127,13 +129,16 @@ wait_for_services() { echo "Waiting for Keycloak... $i/240" done - # Once Keycloak is ready we need to restart the API server because if Keycloak wasn't ready + # Once Keycloak is ready we normally restart the API server because if Keycloak wasn't ready # when it started it would have failed to fetch the JWKS, and therefore it will automatically - # disable Keycloak support. + # disable Keycloak support. Integrated deployments can skip the restart when they guarantee + # that Keycloak was ready before the API started. echo "Waiting for API ..." - kubectl -n $NAMESPACE rollout restart deployment nico-rest-api + if [ "$RESTART_API" = "1" ]; then + kubectl -n $NAMESPACE rollout restart deployment nico-rest-api + fi if ! kubectl -n $NAMESPACE rollout status deployment nico-rest-api --timeout=240s; then - echo "ERROR: Failed to restart API" + echo "ERROR: API is not ready" exit 1 fi @@ -221,15 +226,44 @@ create_site() { exit 1 } +ensure_temporal_namespace() { + local temporal_namespace=$1 + local temporal_command=( + kubectl -n temporal exec deployment/temporal-admintools -- + temporal operator namespace + ) + + echo "Registering Temporal namespace: $temporal_namespace" + for attempt in {1..30}; do + if "${temporal_command[@]}" describe --namespace "$temporal_namespace" \ + --address temporal-frontend.temporal:7233 \ + --tls-cert-path /var/secrets/temporal/certs/server-interservice/tls.crt \ + --tls-key-path /var/secrets/temporal/certs/server-interservice/tls.key \ + --tls-ca-path /var/secrets/temporal/certs/server-interservice/ca.crt \ + --tls-server-name interservice.server.temporal.local >/dev/null 2>&1; then + echo "Temporal namespace confirmed: $temporal_namespace" + return + fi + + "${temporal_command[@]}" create --namespace "$temporal_namespace" \ + --address temporal-frontend.temporal:7233 \ + --tls-cert-path /var/secrets/temporal/certs/server-interservice/tls.crt \ + --tls-key-path /var/secrets/temporal/certs/server-interservice/tls.key \ + --tls-ca-path /var/secrets/temporal/certs/server-interservice/ca.crt \ + --tls-server-name interservice.server.temporal.local >/dev/null 2>&1 || true + echo "Attempt $attempt: Temporal namespace not confirmed yet" + sleep 5 + done + + echo "ERROR: Temporal namespace $temporal_namespace was not created" >&2 + return 1 +} + configure_site_agent() { local site_id=$1 + local current_site_id="" - kubectl -n temporal exec deploy/temporal-admintools -- temporal operator namespace create --namespace "$site_id" \ - --address temporal-frontend.temporal:7233 \ - --tls-cert-path /var/secrets/temporal/certs/server-interservice/tls.crt \ - --tls-key-path /var/secrets/temporal/certs/server-interservice/tls.key \ - --tls-ca-path /var/secrets/temporal/certs/server-interservice/ca.crt \ - --tls-server-name interservice.server.temporal.local || true + ensure_temporal_namespace "$site_id" kubectl -n $NAMESPACE get configmap nico-rest-site-agent-config -o yaml | \ sed "s/CLUSTER_ID: .*/CLUSTER_ID: \"$site_id\"/" | \ @@ -237,16 +271,25 @@ configure_site_agent() { sed "s/TEMPORAL_SUBSCRIBE_QUEUE: .*/TEMPORAL_SUBSCRIBE_QUEUE: \"site\"/" | \ kubectl apply -f - - local reg_token="${SITE_REG_TOKEN:-local-dev-otp}" local sm_cacert sm_cacert=$(kubectl -n $NAMESPACE get secret site-manager-tls -o jsonpath='{.data.ca\.crt}' 2>/dev/null | base64 -d 2>/dev/null || echo "") - kubectl -n $NAMESPACE delete secret site-registration 2>/dev/null || true - kubectl -n $NAMESPACE create secret generic site-registration \ - --from-literal=site-uuid="$site_id" \ - --from-literal=otp="$reg_token" \ - --from-literal=creds-url="https://nico-rest-site-manager:8100/v1/sitecreds" \ - --from-literal=cacert="$sm_cacert" + if [ -n "${SITE_REG_TOKEN:-}" ] && [ "$SITE_REG_TOKEN" != "null" ]; then + kubectl -n $NAMESPACE create secret generic site-registration \ + --from-literal=site-uuid="$site_id" \ + --from-literal=otp="$SITE_REG_TOKEN" \ + --from-literal=creds-url="https://nico-rest-site-manager:8100/v1/sitecreds" \ + --from-literal=cacert="$sm_cacert" \ + --dry-run=client -o yaml | kubectl apply -f - + else + current_site_id=$(kubectl -n $NAMESPACE get secret site-registration \ + -o jsonpath='{.data.site-uuid}' 2>/dev/null | base64 -d 2>/dev/null || echo "") + if [ "$current_site_id" != "$site_id" ]; then + echo "ERROR: Registration token is unavailable and existing site credentials do not match $site_id" >&2 + return 1 + fi + echo "Registration token is no longer exposed; preserving the existing site credentials" + fi kubectl -n $NAMESPACE rollout restart sts/nico-rest-site-agent kubectl -n $NAMESPACE rollout status sts/nico-rest-site-agent --timeout=240s @@ -271,7 +314,7 @@ setup_site_agent() { -H "Authorization: Bearer $TOKEN" | jq -r '.id' )" -H "Authorization: Bearer $TOKEN" | jq -r '.registrationToken // empty' 2>/dev/null) if [ -z "$SITE_REG_TOKEN" ] || [ "$SITE_REG_TOKEN" = "null" ]; then - echo "WARNING: Could not retrieve registration token from API, using fallback" + echo "Registration token is no longer exposed by the API" else echo "Registration token acquired (${#SITE_REG_TOKEN} chars)" fi @@ -326,6 +369,10 @@ case "${1:-}" in site-agent) setup_site_agent ;; + temporal-namespace) + [ "$#" -eq 2 ] || usage + ensure_temporal_namespace "$2" + ;; all) setup_pki setup_site_agent From 1717aa11fe4c4914782fbcbc5cdc407dd42cb236 Mon Sep 17 00:00:00 2001 From: Kyle Felter Date: Thu, 9 Jul 2026 10:44:20 -0500 Subject: [PATCH 2/8] fix: Build REST images with BuildKit Signed-off-by: Kyle Felter --- devspace.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/devspace.yaml b/devspace.yaml index 0a48f30dd1..2dab2efd6e 100644 --- a/devspace.yaml +++ b/devspace.yaml @@ -86,6 +86,7 @@ images: - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_RANDOM} dockerfile: rest-api/docker/local/Dockerfile.nico-rest-api context: rest-api + buildKit: {} skipPush: true nico-rest-workflow: image: localhost:5000/nico-rest-workflow @@ -93,6 +94,7 @@ images: - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_RANDOM} dockerfile: rest-api/docker/local/Dockerfile.nico-rest-workflow context: rest-api + buildKit: {} skipPush: true nico-rest-site-manager: image: localhost:5000/nico-rest-site-manager @@ -100,6 +102,7 @@ images: - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_RANDOM} dockerfile: rest-api/docker/local/Dockerfile.nico-rest-site-manager context: rest-api + buildKit: {} skipPush: true nico-rest-site-agent: image: localhost:5000/nico-rest-site-agent @@ -107,6 +110,7 @@ images: - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_RANDOM} dockerfile: rest-api/docker/local/Dockerfile.nico-rest-site-agent context: rest-api + buildKit: {} skipPush: true nico-rest-db: image: localhost:5000/nico-rest-db @@ -114,6 +118,7 @@ images: - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_RANDOM} dockerfile: rest-api/docker/local/Dockerfile.nico-rest-db context: rest-api + buildKit: {} skipPush: true nico-rest-cert-manager: image: localhost:5000/nico-rest-cert-manager @@ -121,6 +126,7 @@ images: - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_RANDOM} dockerfile: rest-api/docker/local/Dockerfile.nico-rest-cert-manager context: rest-api + buildKit: {} skipPush: true nico-mcp: image: localhost:5000/nico-mcp @@ -128,6 +134,7 @@ images: - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_RANDOM} dockerfile: rest-api/docker/local/Dockerfile.nico-mcp context: rest-api + buildKit: {} skipPush: true deployments: From 5270c2fdc7cc7c585a127acaa87a50fd416345cd Mon Sep 17 00:00:00 2001 From: Kyle Felter Date: Thu, 9 Jul 2026 11:00:51 -0500 Subject: [PATCH 3/8] fix: Use Kubernetes-safe REST image tags Signed-off-by: Kyle Felter --- devspace.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/devspace.yaml b/devspace.yaml index 2dab2efd6e..7d1b624d3b 100644 --- a/devspace.yaml +++ b/devspace.yaml @@ -83,7 +83,7 @@ images: nico-rest-api: image: localhost:5000/nico-rest-api tags: - - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_RANDOM} + - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_TIMESTAMP} dockerfile: rest-api/docker/local/Dockerfile.nico-rest-api context: rest-api buildKit: {} @@ -91,7 +91,7 @@ images: nico-rest-workflow: image: localhost:5000/nico-rest-workflow tags: - - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_RANDOM} + - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_TIMESTAMP} dockerfile: rest-api/docker/local/Dockerfile.nico-rest-workflow context: rest-api buildKit: {} @@ -99,7 +99,7 @@ images: nico-rest-site-manager: image: localhost:5000/nico-rest-site-manager tags: - - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_RANDOM} + - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_TIMESTAMP} dockerfile: rest-api/docker/local/Dockerfile.nico-rest-site-manager context: rest-api buildKit: {} @@ -107,7 +107,7 @@ images: nico-rest-site-agent: image: localhost:5000/nico-rest-site-agent tags: - - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_RANDOM} + - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_TIMESTAMP} dockerfile: rest-api/docker/local/Dockerfile.nico-rest-site-agent context: rest-api buildKit: {} @@ -115,7 +115,7 @@ images: nico-rest-db: image: localhost:5000/nico-rest-db tags: - - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_RANDOM} + - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_TIMESTAMP} dockerfile: rest-api/docker/local/Dockerfile.nico-rest-db context: rest-api buildKit: {} @@ -123,7 +123,7 @@ images: nico-rest-cert-manager: image: localhost:5000/nico-rest-cert-manager tags: - - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_RANDOM} + - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_TIMESTAMP} dockerfile: rest-api/docker/local/Dockerfile.nico-rest-cert-manager context: rest-api buildKit: {} @@ -131,7 +131,7 @@ images: nico-mcp: image: localhost:5000/nico-mcp tags: - - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_RANDOM} + - ${DEVSPACE_GIT_COMMIT}-${DEVSPACE_TIMESTAMP} dockerfile: rest-api/docker/local/Dockerfile.nico-mcp context: rest-api buildKit: {} From dea204b991dd35095df61e3f54aa8ecc140ebfa0 Mon Sep 17 00:00:00 2001 From: Kyle Felter Date: Thu, 9 Jul 2026 11:09:13 -0500 Subject: [PATCH 4/8] fix: Verify inventory on the site worker Signed-off-by: Kyle Felter --- dev/deployment/devspace/setup-rest-integration.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/deployment/devspace/setup-rest-integration.sh b/dev/deployment/devspace/setup-rest-integration.sh index 75542276e6..5b285a8eb8 100755 --- a/dev/deployment/devspace/setup-rest-integration.sh +++ b/dev/deployment/devspace/setup-rest-integration.sh @@ -158,7 +158,7 @@ for attempt in {1..60}; do <<<"${machines}" 2>/dev/null || printf '0')" fi - cloud_worker_logs="$(kubectl logs deployment/nico-rest-cloud-worker \ + site_worker_logs="$(kubectl logs deployment/nico-rest-site-worker \ -n "${REST_NAMESPACE}" --since-time="${inventory_started_at}" 2>/dev/null || true)" fresh_cycle="$(jq -Rrs --arg site_id "${site_id}" ' [splits("\n") | fromjson? @@ -176,7 +176,7 @@ for attempt in {1..60}; do .complete = true else . end ) | .complete - ' <<<"${cloud_worker_logs}" 2>/dev/null || printf 'false')" + ' <<<"${site_worker_logs}" 2>/dev/null || printf 'false')" if [[ "${site_ready}" == "true" && "${machines_ready}" == "true" && \ "${fresh_cycle}" == "true" ]]; then From 5ca05f6e551943d0a27511d73bcb1e8a58c82852 Mon Sep 17 00:00:00 2001 From: Kyle Felter Date: Thu, 9 Jul 2026 11:29:58 -0500 Subject: [PATCH 5/8] fix: Preserve site agent Temporal credentials Signed-off-by: Kyle Felter --- .../templates/temporal-certs-secret.yaml | 22 +++---- .../tests/temporal_certs_secret_test.yaml | 60 +++++++++++++++++++ 2 files changed, 72 insertions(+), 10 deletions(-) create mode 100644 helm/rest/nico-rest-site-agent/tests/temporal_certs_secret_test.yaml diff --git a/helm/rest/nico-rest-site-agent/templates/temporal-certs-secret.yaml b/helm/rest/nico-rest-site-agent/templates/temporal-certs-secret.yaml index 6efa5b1e15..904cf724d7 100644 --- a/helm/rest/nico-rest-site-agent/templates/temporal-certs-secret.yaml +++ b/helm/rest/nico-rest-site-agent/templates/temporal-certs-secret.yaml @@ -1,21 +1,23 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Temporal client TLS certs for site-agent — placeholder values populated by bootstrap. -# resource-policy: keep prevents helm upgrade from overwriting certs written by the -# bootstrap process; deleting and re-installing the release creates a fresh placeholder. +# Temporal client TLS certs for site-agent. The bootstrap process populates these +# placeholders, and lookup preserves those values across Helm upgrades. +{{- $namespace := include "nico-rest-site-agent.namespace" . }} +{{- $secretName := .Values.secrets.temporalClientCerts }} +{{- $existing := lookup "v1" "Secret" $namespace $secretName | default dict }} apiVersion: v1 kind: Secret metadata: - name: {{ .Values.secrets.temporalClientCerts }} - namespace: {{ include "nico-rest-site-agent.namespace" . }} + name: {{ $secretName }} + namespace: {{ $namespace }} labels: {{- include "nico-rest-site-agent.labels" . | nindent 4 }} annotations: helm.sh/resource-policy: keep type: Opaque -stringData: - otp: "" - cacertificate: "" - certificate: "" - key: "" +data: + otp: {{ dig "data" "otp" "" $existing | quote }} + cacertificate: {{ dig "data" "cacertificate" "" $existing | quote }} + certificate: {{ dig "data" "certificate" "" $existing | quote }} + key: {{ dig "data" "key" "" $existing | quote }} diff --git a/helm/rest/nico-rest-site-agent/tests/temporal_certs_secret_test.yaml b/helm/rest/nico-rest-site-agent/tests/temporal_certs_secret_test.yaml new file mode 100644 index 0000000000..02772a88d9 --- /dev/null +++ b/helm/rest/nico-rest-site-agent/tests/temporal_certs_secret_test.yaml @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +suite: temporal client certificate persistence +templates: + - temporal-certs-secret.yaml +release: + namespace: nico-rest +skip: + minimumVersion: 0.5.1 +tests: + - it: creates empty placeholders on install + asserts: + - isKind: + of: Secret + - equal: + path: metadata.annotations["helm.sh/resource-policy"] + value: keep + - notExists: + path: stringData + - equal: + path: data + value: + otp: "" + cacertificate: "" + certificate: "" + key: "" + + - it: preserves populated credentials on upgrade + release: + namespace: nico-rest + upgrade: true + kubernetesProvider: + scheme: + "v1/Secret": + gvr: + version: v1 + resource: secrets + namespaced: true + objects: + - apiVersion: v1 + kind: Secret + metadata: + name: temporal-client-site-agent-certs + namespace: nico-rest + data: + otp: b3Rw + cacertificate: Y2E= + certificate: Y2VydA== + key: a2V5 + asserts: + - notExists: + path: stringData + - equal: + path: data + value: + otp: b3Rw + cacertificate: Y2E= + certificate: Y2VydA== + key: a2V5 From 89fe26f583420e28babe0481b5aa4743bc01ac80 Mon Sep 17 00:00:00 2001 From: Kyle Felter Date: Thu, 9 Jul 2026 12:17:32 -0500 Subject: [PATCH 6/8] fix: Reset DevSpace state on purge Signed-off-by: Kyle Felter --- dev/deployment/devspace/README.md | 18 +++-- dev/deployment/devspace/reset-kind-cluster.sh | 70 +++++++++++++++++++ devspace.yaml | 4 ++ 3 files changed, 88 insertions(+), 4 deletions(-) create mode 100755 dev/deployment/devspace/reset-kind-cluster.sh diff --git a/dev/deployment/devspace/README.md b/dev/deployment/devspace/README.md index b62c073d5b..d7f12ad716 100644 --- a/dev/deployment/devspace/README.md +++ b/dev/deployment/devspace/README.md @@ -4,7 +4,7 @@ You can use [DevSpace](https://www.devspace.sh) to deploy the complete local inf The process is broken into two steps: -1. Bootstrap Kubernetes prerequisites. (This only needs to be done once.) +1. Bootstrap Kubernetes prerequisites. (This only needs to be done once per cluster.) 2. Run `devspace deploy` to deploy code from this repo The intent is that the app deploy path stays the same whether the prerequisites are: @@ -120,6 +120,7 @@ Common usage: ```bash devspace deploy devspace deploy -n nico-system +devspace deploy --skip-build -n nico-system devspace deploy --force-build ``` @@ -164,17 +165,25 @@ docker build -t "machine-a-tron:" -f dev/deployment/devs DevSpace then deploys the Helm chart with the built `nico-api` image wired into `global.image.repository` and `global.image.tag`, the built `nico-bmc-proxy` image wired into the `nico-bmc-proxy` chart values, and applies the local-only `machine-a-tron` manifest with its image wired into the `Deployment` spec. The REST images are built from the existing `rest-api/docker/local` Dockerfiles and are passed to the three existing REST Helm charts with the same generated tag. -## Re-initializing Core state +## Resetting the local environment Once deployed, the `nico-api` container will run and initialize its database, and the `machine-a-tron` container will run a set of mock machines, which will be discovered and ingested into the database, and run through the state machine until they reach a Ready state. -You can purge the deployed product resources by running: +Reset the complete local environment by running: ```bash devspace purge -n nico-system ``` -and it will delete the Core and REST Helm releases and the machine-a-tron deployment. Bootstrap-owned services such as PostgreSQL, Vault, Temporal, Keycloak, and cert-manager remain installed. +The purge pipeline only runs when the current context is `kind-`. It deletes and recreates that kind cluster with the same node image, then bootstraps clean prerequisites. This removes all Kubernetes state, including the Core and REST databases, Temporal namespaces and history, Vault data, Keycloak data, certificates, site registration, Helm releases, CRDs, and persistent volumes. + +The host Docker images, BuildKit cache, and `.devspace` image metadata are outside the kind node and remain available. Redeploy the last built images without rebuilding them: + +```bash +devspace deploy --skip-build -n nico-system +``` + +The pre-deploy hooks load the cached Core and REST images from the host Docker store into the new kind node. Omit `--skip-build` when the source or image definitions have changed since the last build. To clear only the Core `nico` database, run the nuke-postgres.sh helper script: @@ -191,6 +200,7 @@ devspace deploy -n nico-system ## Files - [`bootstrap-prereqs.sh`](bootstrap-prereqs.sh) +- [`reset-kind-cluster.sh`](reset-kind-cluster.sh) - [`setup-rest-integration.sh`](setup-rest-integration.sh) - [`devspace.yaml`](../../../devspace.yaml) - [`values.base.yaml`](values.base.yaml) diff --git a/dev/deployment/devspace/reset-kind-cluster.sh b/dev/deployment/devspace/reset-kind-cluster.sh new file mode 100755 index 0000000000..35bb84d0e1 --- /dev/null +++ b/dev/deployment/devspace/reset-kind-cluster.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" + +log() { + printf '[local-dev] %s\n' "$*" +} + +require_bin() { + command -v "$1" >/dev/null 2>&1 || { + printf 'missing required binary: %s\n' "$1" >&2 + exit 1 + } +} + +main() { + require_bin docker + require_bin helm + require_bin kind + require_bin kubectl + require_bin base64 + + local current_context cluster_name control_plane node_image + current_context="$(kubectl config current-context 2>/dev/null || true)" + case "${current_context}" in + kind-*) cluster_name="${current_context#kind-}" ;; + *) + printf 'purge requires a kind context; current context is %s\n' \ + "${current_context:-unset}" >&2 + exit 1 + ;; + esac + + control_plane="${cluster_name}-control-plane" + node_image="$(docker inspect --format '{{.Config.Image}}' "${control_plane}" 2>/dev/null || true)" + if [[ -z "${node_image}" ]]; then + printf 'could not determine the node image from container %s\n' \ + "${control_plane}" >&2 + exit 1 + fi + + log "Deleting kind cluster ${cluster_name}" + kind delete cluster --name "${cluster_name}" + + log "Recreating kind cluster ${cluster_name} with ${node_image}" + kind create cluster --name "${cluster_name}" --image "${node_image}" + + log "Installing clean local prerequisites" + "${SCRIPT_DIR}/bootstrap-prereqs.sh" + + log "Reset complete; deploy with devspace deploy --skip-build" +} + +main "$@" diff --git a/devspace.yaml b/devspace.yaml index 7d1b624d3b..bdfa037a09 100644 --- a/devspace.yaml +++ b/devspace.yaml @@ -30,6 +30,10 @@ vars: source: env default: "18082" +pipelines: + purge: |- + LOCAL_DEV_NAMESPACE=${DEVSPACE_NAMESPACE} dev/deployment/devspace/reset-kind-cluster.sh + images: nico-api: image: nico-api From 3a2abf56e9ab4038445e7c5872843415d01dfd89 Mon Sep 17 00:00:00 2001 From: Kyle Felter Date: Thu, 9 Jul 2026 12:22:39 -0500 Subject: [PATCH 7/8] fix: Bootstrap the legacy service namespace Signed-off-by: Kyle Felter --- dev/deployment/devspace/bootstrap-prereqs.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dev/deployment/devspace/bootstrap-prereqs.sh b/dev/deployment/devspace/bootstrap-prereqs.sh index e1d73b12ff..e8cae69d7f 100755 --- a/dev/deployment/devspace/bootstrap-prereqs.sh +++ b/dev/deployment/devspace/bootstrap-prereqs.sh @@ -146,7 +146,7 @@ install_cert_manager() { } apply_core_objects() { - log "Applying namespace and connection objects in ${NAMESPACE}" + log "Applying Core namespaces and connection objects in ${NAMESPACE}" kubectl apply -f - < Date: Thu, 9 Jul 2026 12:59:39 -0500 Subject: [PATCH 8/8] fix: Allow the clean inventory cycle to finish Signed-off-by: Kyle Felter --- dev/deployment/devspace/setup-rest-integration.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dev/deployment/devspace/setup-rest-integration.sh b/dev/deployment/devspace/setup-rest-integration.sh index 5b285a8eb8..293ff703c2 100755 --- a/dev/deployment/devspace/setup-rest-integration.sh +++ b/dev/deployment/devspace/setup-rest-integration.sh @@ -129,7 +129,8 @@ site_ready=false machines_ready=false machine_count=0 fresh_cycle=false -for attempt in {1..60}; do +# A clean cluster may need a second three-minute inventory cycle after Core discovers machines. +for attempt in {1..90}; do site_ready=false machines_ready=false machine_count=0 @@ -184,7 +185,7 @@ for attempt in {1..60}; do "${site_id}" "${machine_count}" break fi - if [[ "${attempt}" == "60" ]]; then + if [[ "${attempt}" == "90" ]]; then printf 'REST integration verification failed: site_ready=%s machines_ready=%s machines=%s fresh_cycle=%s\n' \ "${site_ready}" "${machines_ready}" "${machine_count}" "${fresh_cycle}" >&2 exit 1