From 1ef138b2555e95dc9dff764ea0144361f833599e Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 3 Aug 2026 14:43:26 +0100 Subject: [PATCH 01/54] ci: add k8s stack PR preview via labeled deploy + Cloudflare Tunnel Labeling a PR with deploy-preview spins up the full NIC platform stack plus this chart in an ephemeral k3d cluster, then exposes JupyterHub through a per-PR Cloudflare Tunnel gated by Cloudflare Access (GitHub org SSO), posting the link as a sticky PR comment. Includes a standalone tunnel-only smoketest workflow for validating the Cloudflare plumbing without the full stack deploy. --- .../k8s-preview-tunnel-smoketest.yaml | 138 ++++++++ .github/workflows/k8s-preview.yaml | 309 ++++++++++++++++++ 2 files changed, 447 insertions(+) create mode 100644 .github/workflows/k8s-preview-tunnel-smoketest.yaml create mode 100644 .github/workflows/k8s-preview.yaml diff --git a/.github/workflows/k8s-preview-tunnel-smoketest.yaml b/.github/workflows/k8s-preview-tunnel-smoketest.yaml new file mode 100644 index 0000000..4392d0d --- /dev/null +++ b/.github/workflows/k8s-preview-tunnel-smoketest.yaml @@ -0,0 +1,138 @@ +name: K8s Preview Tunnel Smoketest +# Manual, on-demand check of the Cloudflare Tunnel + DNS + Access plumbing +# that k8s-preview.yaml depends on — without paying for a full k3d + +# platform-stack + helm deploy on every iteration. Serves a static +# index.html instead of JupyterHub; same tunnel/DNS create+configure+ +# delete API calls as the real workflow, same secrets. Run this first +# when validating the Cloudflare-side setup (token scopes, Access +# application, GitHub identity provider); once a visit to the printed +# URL round-trips through GitHub SSO successfully, k8s-preview.yaml's +# tunnel plumbing is known-good and any remaining issue is in the +# k3d/helm/chart side, not Cloudflare. + +on: + workflow_dispatch: + +env: + PREVIEW_DOMAIN: dspack.iakte.ch + CLOUDFLARED_VERSION: "2026.7.3" + CLOUDFLARED_SHA256: "9d71c677db00134c1bd4144b7783486b654ad281b1ea62b4972098d19f770f17" + +jobs: + smoketest: + name: Tunnel smoketest + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read + steps: + - name: Serve a trivial static page + run: | + mkdir -p /tmp/preview-test + cat > /tmp/preview-test/index.html <<'EOF' + +

Tunnel smoketest OK

+ EOF + python3 -m http.server 8000 --directory /tmp/preview-test \ + > /tmp/http-server.log 2>&1 & + + - name: Install cloudflared + run: | + curl -fsSL -o /tmp/cloudflared \ + "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64" + echo "${CLOUDFLARED_SHA256} /tmp/cloudflared" | sha256sum -c - + chmod +x /tmp/cloudflared + + - name: Create Cloudflare Tunnel for this run + id: cf_tunnel + env: + CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} + CF_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_TUNNEL_ACCOUNT_ID }} + PREVIEW_HOSTNAME: smoketest-${{ github.run_id }}.${{ env.PREVIEW_DOMAIN }} + run: | + tunnel_secret=$(openssl rand -base64 32) + echo "::add-mask::${tunnel_secret}" + + create_resp=$(curl -fsS -X POST \ + "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "$(jq -n --arg name "smoketest-${{ github.run_id }}" --arg secret "$tunnel_secret" \ + '{name: $name, config_src: "cloudflare", tunnel_secret: $secret}')") + tunnel_id=$(jq -r '.result.id' <<< "$create_resp") + if [ -z "$tunnel_id" ] || [ "$tunnel_id" = "null" ]; then + echo "::error::Tunnel creation failed: $create_resp" + exit 1 + fi + echo "TUNNEL_ID=${tunnel_id}" >> "$GITHUB_ENV" + + token_resp=$(curl -fsS \ + "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel/${tunnel_id}/token" \ + -H "Authorization: Bearer ${CF_API_TOKEN}") + tunnel_token=$(jq -r '.result' <<< "$token_resp") + echo "::add-mask::${tunnel_token}" + echo "TUNNEL_TOKEN=${tunnel_token}" >> "$GITHUB_ENV" + + curl -fsS -X PUT \ + "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel/${tunnel_id}/configurations" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "$(jq -n --arg host "$PREVIEW_HOSTNAME" \ + '{config: {ingress: [{hostname: $host, service: "http://localhost:8000"}, {service: "http_status:404"}]}}')" \ + > /dev/null + + - name: Point DNS at the tunnel + id: cf_dns + env: + CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} + PREVIEW_HOSTNAME: smoketest-${{ github.run_id }}.${{ env.PREVIEW_DOMAIN }} + run: | + zone_id=$(curl -fsS "https://api.cloudflare.com/client/v4/zones?name=iakte.ch" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" | jq -r '.result[0].id') + if [ -z "$zone_id" ] || [ "$zone_id" = "null" ]; then + echo "::error::Could not resolve zone id for iakte.ch" + exit 1 + fi + echo "ZONE_ID=${zone_id}" >> "$GITHUB_ENV" + + record_resp=$(curl -fsS -X POST "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "$(jq -n --arg host "$PREVIEW_HOSTNAME" --arg target "${TUNNEL_ID}.cfargotunnel.com" \ + '{type: "CNAME", name: $host, content: $target, proxied: true}')") + record_id=$(jq -r '.result.id' <<< "$record_resp") + if [ -z "$record_id" ] || [ "$record_id" = "null" ]; then + echo "::error::DNS record creation failed: $record_resp" + exit 1 + fi + echo "DNS_RECORD_ID=${record_id}" >> "$GITHUB_ENV" + + echo "## Smoketest URL" >> "$GITHUB_STEP_SUMMARY" + echo "https://${PREVIEW_HOSTNAME}" >> "$GITHUB_STEP_SUMMARY" + echo "Visiting it should challenge you with Cloudflare Access GitHub SSO," >> "$GITHUB_STEP_SUMMARY" + echo "then show 'Tunnel smoketest OK' once you're through." >> "$GITHUB_STEP_SUMMARY" + echo "Live for up to 15 minutes (this job's timeout)." >> "$GITHUB_STEP_SUMMARY" + + - name: Run tunnel until the job times out + run: /tmp/cloudflared tunnel run --token "${TUNNEL_TOKEN}" --no-autoupdate + + - name: Delete DNS record + if: always() + env: + CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} + run: | + [ -n "${ZONE_ID:-}" ] && [ -n "${DNS_RECORD_ID:-}" ] || exit 0 + curl -fsS -X DELETE \ + "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records/${DNS_RECORD_ID}" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" || true + + - name: Delete Cloudflare Tunnel + if: always() + env: + CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} + CF_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_TUNNEL_ACCOUNT_ID }} + run: | + [ -n "${TUNNEL_ID:-}" ] || exit 0 + curl -fsS -X DELETE \ + "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel/${TUNNEL_ID}" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" || true diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml new file mode 100644 index 0000000..cbf510f --- /dev/null +++ b/.github/workflows/k8s-preview.yaml @@ -0,0 +1,309 @@ +name: K8s Stack Preview +# Deploys the full Nebari platform stack (Keycloak + nic-operator + Envoy +# Gateway, via nebari-dev/action-nebari-sandbox's `platform` profile) plus +# this PR's chart into an ephemeral k3d cluster on the runner, then exposes +# JupyterHub through a per-PR Cloudflare Tunnel behind Cloudflare Access +# (GitHub-org SSO) so a reviewer can click a link and use it. +# +# This repo is public, so a plain shared secret posted in the PR comment +# (a quick-tunnel URL, a basic-auth password) is readable by anyone who +# opens the PR, not just intended reviewers. Access closes that gap by +# authenticating the *person*, not a string in the comment: Cloudflare +# challenges every request to *.dspack.iakte.ch with a GitHub SSO +# login and only lets it through to cloudflared if the signed-in account +# is a member of this GitHub org. The PR comment only ever contains a URL. +# +# iakte.ch is a placeholder domain used temporarily because it lives in +# the same Cloudflare account as the Tunnel and Zero Trust setup; swap +# PREVIEW_DOMAIN below once a permanent domain is available in that +# account. +# +# Scope, deliberately: the tunnel points straight at the `proxy-public` +# service (dummy-authenticator login, same as local Tilt dev), not the +# operator-provisioned NebariApp/OIDC route — that route exists for its +# own Keycloak realm and redirect URIs, unrelated to Access's GitHub SSO. +# The NebariApp CRD is still installed (auth.enabled=true) purely so the +# operator/Keycloak reconcile is exercised and its Ready condition +# reported in the run; it is not what the reviewer clicks through. +# +# The link only lives for the run's duration (bounded by timeout-minutes +# below) — it is not a persistent per-PR environment. Each run creates its +# own Cloudflare Tunnel + DNS record (so concurrent previews on different +# PRs don't collide on the same route) and deletes both on cleanup. +# +# One-time setup this workflow assumes already exists (Cloudflare Zero +# Trust dashboard, done by a repo admin, not scripted here). The tunnel, +# the `iakte.ch` zone, and Zero Trust/Access all live in ONE Cloudflare +# account (aktechlabs) — not the account behind CLOUDFLARE_API_TOKEN / +# CLOUDFLARE_ACCOUNT_ID, which docs.yml uses for Pages: +# - Zone `iakte.ch` in that account, with an Access self-hosted +# application for `*.dspack.iakte.ch`, GitHub as identity provider, +# policy scoped to this org. +# - Secret CLOUDFLARE_TUNNEL_ACCOUNT_ID: that account's id (the +# `cfd_tunnel` API is account-scoped; can't be derived from the +# token alone). +# - Secret CLOUDFLARE_TUNNEL_API_TOKEN: a custom token scoped to +# EXACTLY three permissions, nothing broader: +# * Account -> Cloudflare Tunnel -> Edit +# * Zone -> Zone -> Read (to resolve the zone id by name) +# * Zone -> DNS -> Edit (to create/delete the CNAME record) +# Zone Resources: Include -> Specific zone -> iakte.ch. +# Account Resources: Include -> Specific account -> aktechlabs. +# +# Only runs when a maintainer/collaborator adds the `deploy-preview` label +# (GitHub restricts who can label a PR) — arbitrary PR authors, including +# from forks, cannot trigger this themselves. Even so, a fork PR still runs +# attacker-authored code once labeled; the comment flags this so whoever +# labels it is doing so knowingly. +# +# Residual risk not covered here: k3d pods share the runner's Docker daemon +# rather than being hardware-isolated, and k3d's default CNI (flannel) +# does not enforce NetworkPolicy, so a container escape or outbound abuse +# from inside a spawned notebook pod is not blocked at the network layer. +# Per-job GITHUB_TOKEN permissions are scoped to the minimum each job +# needs so a compromised runner in the exposed 90-minute window can't use +# an ambient token to touch other workflows or repo state. + +on: + pull_request: + types: [labeled, unlabeled, synchronize] + +concurrency: + group: k8s-preview-${{ github.event.pull_request.number }} + cancel-in-progress: true + +env: + PREVIEW_LABEL: deploy-preview + PREVIEW_DOMAIN: dspack.iakte.ch + CLOUDFLARED_VERSION: "2026.7.3" + # sha256 of cloudflared-linux-amd64 for the pinned version above, + # computed from the official release asset at + # https://github.com/cloudflare/cloudflared/releases/tag/2026.7.3 + CLOUDFLARED_SHA256: "9d71c677db00134c1bd4144b7783486b654ad281b1ea62b4972098d19f770f17" + +jobs: + deploy-preview: + if: contains(github.event.pull_request.labels.*.name, 'deploy-preview') && github.event.action != 'unlabeled' + name: Deploy preview + runs-on: ubuntu-24.04 + timeout-minutes: 90 + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Provision sandbox (k3d + full NIC platform stack) + id: sandbox + uses: nebari-dev/action-nebari-sandbox@dbdb054c16efee3b29e26aee406c12fb9639bfd0 # v2.2.0 + with: + profile: platform + cluster-name: pr-preview-${{ github.event.pull_request.number }} + + - name: Build hub image from this PR + run: docker build --target jupyterhub -t nebari-data-science-pack-jupyterhub:preview images/ + + - name: Install k3d CLI (for image import) + run: | + curl -fsSL https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | \ + TAG=v5.8.3 bash + + - name: Side-load hub image into the sandbox cluster + run: k3d image import nebari-data-science-pack-jupyterhub:preview -c ${{ steps.sandbox.outputs.cluster-name }} + + - name: Deploy chart + id: deploy + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + run: | + helm upgrade --install preview . \ + --namespace pr-preview --create-namespace \ + --set jupyterhub.hub.image.name=nebari-data-science-pack-jupyterhub \ + --set jupyterhub.hub.image.tag=preview \ + --set nebariapp.enabled=true \ + --set nebariapp.hostname="pr-${{ github.event.pull_request.number }}.${{ steps.sandbox.outputs.domain }}" \ + --set nebariapp.auth.enabled=true \ + --wait --timeout 5m + + - name: Wait for hub + proxy + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + run: | + kubectl -n pr-preview rollout status deployment/hub --timeout=180s + kubectl -n pr-preview rollout status deployment/proxy --timeout=180s + + - name: Report NebariApp reconcile status + if: always() + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + run: | + kubectl -n pr-preview get nebariapp -o wide || true + kubectl -n pr-preview describe nebariapp preview-nebari-data-science-pack || true + + - name: Port-forward JupyterHub proxy + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + run: | + kubectl -n pr-preview port-forward svc/proxy-public 8000:80 \ + > /tmp/port-forward.log 2>&1 & + echo "PORT_FORWARD_PID=$!" >> "$GITHUB_ENV" + + - name: Install cloudflared + run: | + curl -fsSL -o /tmp/cloudflared \ + "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64" + echo "${CLOUDFLARED_SHA256} /tmp/cloudflared" | sha256sum -c - + chmod +x /tmp/cloudflared + + # A per-run named Tunnel (not the anonymous quick-tunnel) so: + # (a) it can sit behind an Access application (quick tunnels have no + # account/zone attached, so no policy can be bound to them), and + # (b) each PR gets its own tunnel + hostname, so two PRs previewing + # at once don't share one route and cross-talk. + - name: Create Cloudflare Tunnel for this PR + id: cf_tunnel + env: + CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} + CF_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_TUNNEL_ACCOUNT_ID }} + PREVIEW_HOSTNAME: pr-${{ github.event.pull_request.number }}.${{ env.PREVIEW_DOMAIN }} + run: | + tunnel_secret=$(openssl rand -base64 32) + echo "::add-mask::${tunnel_secret}" + + create_resp=$(curl -fsS -X POST \ + "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "$(jq -n --arg name "pr-${{ github.event.pull_request.number }}-${{ github.run_id }}" --arg secret "$tunnel_secret" \ + '{name: $name, config_src: "cloudflare", tunnel_secret: $secret}')") + tunnel_id=$(jq -r '.result.id' <<< "$create_resp") + if [ -z "$tunnel_id" ] || [ "$tunnel_id" = "null" ]; then + echo "::error::Tunnel creation failed: $create_resp" + exit 1 + fi + echo "tunnel_id=${tunnel_id}" >> "$GITHUB_OUTPUT" + echo "TUNNEL_ID=${tunnel_id}" >> "$GITHUB_ENV" + + token_resp=$(curl -fsS \ + "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel/${tunnel_id}/token" \ + -H "Authorization: Bearer ${CF_API_TOKEN}") + tunnel_token=$(jq -r '.result' <<< "$token_resp") + echo "::add-mask::${tunnel_token}" + echo "TUNNEL_TOKEN=${tunnel_token}" >> "$GITHUB_ENV" + + curl -fsS -X PUT \ + "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel/${tunnel_id}/configurations" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "$(jq -n --arg host "$PREVIEW_HOSTNAME" \ + '{config: {ingress: [{hostname: $host, service: "http://localhost:8000"}, {service: "http_status:404"}]}}')" \ + > /dev/null + + - name: Point DNS at the tunnel + id: cf_dns + env: + CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} + PREVIEW_HOSTNAME: pr-${{ github.event.pull_request.number }}.${{ env.PREVIEW_DOMAIN }} + run: | + zone_id=$(curl -fsS "https://api.cloudflare.com/client/v4/zones?name=iakte.ch" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" | jq -r '.result[0].id') + if [ -z "$zone_id" ] || [ "$zone_id" = "null" ]; then + echo "::error::Could not resolve zone id for iakte.ch" + exit 1 + fi + echo "ZONE_ID=${zone_id}" >> "$GITHUB_ENV" + + record_resp=$(curl -fsS -X POST "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "$(jq -n --arg host "$PREVIEW_HOSTNAME" --arg target "${TUNNEL_ID}.cfargotunnel.com" \ + '{type: "CNAME", name: $host, content: $target, proxied: true}')") + record_id=$(jq -r '.result.id' <<< "$record_resp") + if [ -z "$record_id" ] || [ "$record_id" = "null" ]; then + echo "::error::DNS record creation failed: $record_resp" + exit 1 + fi + echo "DNS_RECORD_ID=${record_id}" >> "$GITHUB_ENV" + echo "url=https://${PREVIEW_HOSTNAME}" >> "$GITHUB_OUTPUT" + + - name: Comment preview link on PR + uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 + with: + header: k8s-preview + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + message: | + **K8s stack preview** for `${{ github.event.pull_request.head.ref }}`: + ${{ steps.cf_dns.outputs.url }} + + ${{ github.event.pull_request.head.repo.fork && '⚠️ **This PR is from a fork** — the code running in this preview is not from a trusted maintainer branch.' || '' }} + + You'll be asked to sign in via Cloudflare Access (GitHub SSO) before + reaching JupyterHub — only members of this GitHub org get through. + **Then JupyterHub login:** dummy authenticator, any username + any password. + + This goes straight to JupyterHub's proxy and skips the operator-provisioned + OIDC/Keycloak route (unrelated realm/redirect URIs to Access's GitHub SSO). + The `NebariApp` resource is still deployed alongside it so the operator/ + Keycloak reconcile itself is exercised — see the "Report NebariApp reconcile + status" step in this run for its `Ready` condition. + + Live only for this job's run (up to 90 min), or until the `deploy-preview` + label is removed. Push a new commit or re-add the label to redeploy. + + - name: Run tunnel until the job times out + run: /tmp/cloudflared tunnel run --token "${TUNNEL_TOKEN}" --no-autoupdate + + - name: Delete DNS record + if: always() + env: + CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} + run: | + [ -n "${ZONE_ID:-}" ] && [ -n "${DNS_RECORD_ID:-}" ] || exit 0 + curl -fsS -X DELETE \ + "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records/${DNS_RECORD_ID}" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" || true + + - name: Delete Cloudflare Tunnel + if: always() + env: + CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} + CF_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_TUNNEL_ACCOUNT_ID }} + run: | + [ -n "${TUNNEL_ID:-}" ] || exit 0 + curl -fsS -X DELETE \ + "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel/${TUNNEL_ID}" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" || true + + - name: Cleanup cluster + if: always() + run: k3d cluster delete ${{ steps.sandbox.outputs.cluster-name }} || true + + cleanup-preview: + if: github.event.action == 'unlabeled' && github.event.label.name == 'deploy-preview' + name: Stop preview + runs-on: ubuntu-latest + permissions: + pull-requests: write + actions: write + steps: + - name: Cancel the in-flight preview run for this PR + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + run_id=$(gh api "repos/${{ github.repository }}/actions/runs?event=pull_request&status=in_progress" \ + --jq '.workflow_runs[] | select(.name == "K8s Stack Preview") | select(.pull_requests[]?.number == ${{ github.event.pull_request.number }}) | .id' \ + | head -1) + if [ -n "$run_id" ]; then + gh run cancel "$run_id" --repo "${{ github.repository }}" + fi + + - name: Comment that the preview stopped + uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 + with: + header: k8s-preview + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + message: | + **K8s stack preview** stopped — the `deploy-preview` label was removed. + + Add it again to redeploy. From 72dab4607f8aa7a54d6032f1d1527273f3f82127 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 3 Aug 2026 14:47:01 +0100 Subject: [PATCH 02/54] ci: temporarily trigger smoketest on this PR for iteration pull_request/synchronize on this one file, scoped narrowly, so the tunnel-only smoketest can run without waiting for main to see the workflow_dispatch trigger. Remove before merging. --- .github/workflows/k8s-preview-tunnel-smoketest.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/k8s-preview-tunnel-smoketest.yaml b/.github/workflows/k8s-preview-tunnel-smoketest.yaml index 4392d0d..64cef48 100644 --- a/.github/workflows/k8s-preview-tunnel-smoketest.yaml +++ b/.github/workflows/k8s-preview-tunnel-smoketest.yaml @@ -12,6 +12,14 @@ name: K8s Preview Tunnel Smoketest on: workflow_dispatch: + # TEMPORARY, for iterating on this PR before workflow_dispatch is + # reachable (it needs the file on the default branch first). Scoped to + # this file only so it can't fire on unrelated PRs. Remove this + # `pull_request` block before merging. + pull_request: + types: [synchronize] + paths: + - .github/workflows/k8s-preview-tunnel-smoketest.yaml env: PREVIEW_DOMAIN: dspack.iakte.ch From 52d9f86e8cd1c95f739c48e5a0f617e2878761e4 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 3 Aug 2026 14:52:25 +0100 Subject: [PATCH 03/54] fix: move cloudflared --no-autoupdate flag before the run subcommand --no-autoupdate is a tunnel-level flag, not a run-subcommand flag; placed after `run` it errored with "flag provided but not defined" and the tunnel process exited immediately without ever connecting. --- .github/workflows/k8s-preview-tunnel-smoketest.yaml | 2 +- .github/workflows/k8s-preview.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/k8s-preview-tunnel-smoketest.yaml b/.github/workflows/k8s-preview-tunnel-smoketest.yaml index 64cef48..e639af0 100644 --- a/.github/workflows/k8s-preview-tunnel-smoketest.yaml +++ b/.github/workflows/k8s-preview-tunnel-smoketest.yaml @@ -122,7 +122,7 @@ jobs: echo "Live for up to 15 minutes (this job's timeout)." >> "$GITHUB_STEP_SUMMARY" - name: Run tunnel until the job times out - run: /tmp/cloudflared tunnel run --token "${TUNNEL_TOKEN}" --no-autoupdate + run: /tmp/cloudflared tunnel --no-autoupdate run --token "${TUNNEL_TOKEN}" - name: Delete DNS record if: always() diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index cbf510f..5abe013 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -252,7 +252,7 @@ jobs: label is removed. Push a new commit or re-add the label to redeploy. - name: Run tunnel until the job times out - run: /tmp/cloudflared tunnel run --token "${TUNNEL_TOKEN}" --no-autoupdate + run: /tmp/cloudflared tunnel --no-autoupdate run --token "${TUNNEL_TOKEN}" - name: Delete DNS record if: always() From 2818f4844b18388fca56546452a5b347b161d4f4 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 3 Aug 2026 15:00:12 +0100 Subject: [PATCH 04/54] ci: use single-level github.fyi hostnames for the preview tunnel Two-level hostnames (pr-.dspack.iakte.ch) aren't covered by Cloudflare's default Universal SSL, which only auto-issues a cert for the zone apex plus one wildcard level. Switching to a dedicated single-level domain (pr-.github.fyi) avoids needing the paid Advanced Certificate Manager add-on. --- .../k8s-preview-tunnel-smoketest.yaml | 6 ++-- .github/workflows/k8s-preview.yaml | 34 +++++++++++-------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/.github/workflows/k8s-preview-tunnel-smoketest.yaml b/.github/workflows/k8s-preview-tunnel-smoketest.yaml index e639af0..cc7b601 100644 --- a/.github/workflows/k8s-preview-tunnel-smoketest.yaml +++ b/.github/workflows/k8s-preview-tunnel-smoketest.yaml @@ -22,7 +22,7 @@ on: - .github/workflows/k8s-preview-tunnel-smoketest.yaml env: - PREVIEW_DOMAIN: dspack.iakte.ch + PREVIEW_DOMAIN: github.fyi CLOUDFLARED_VERSION: "2026.7.3" CLOUDFLARED_SHA256: "9d71c677db00134c1bd4144b7783486b654ad281b1ea62b4972098d19f770f17" @@ -95,10 +95,10 @@ jobs: CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} PREVIEW_HOSTNAME: smoketest-${{ github.run_id }}.${{ env.PREVIEW_DOMAIN }} run: | - zone_id=$(curl -fsS "https://api.cloudflare.com/client/v4/zones?name=iakte.ch" \ + zone_id=$(curl -fsS "https://api.cloudflare.com/client/v4/zones?name=github.fyi" \ -H "Authorization: Bearer ${CF_API_TOKEN}" | jq -r '.result[0].id') if [ -z "$zone_id" ] || [ "$zone_id" = "null" ]; then - echo "::error::Could not resolve zone id for iakte.ch" + echo "::error::Could not resolve zone id for github.fyi" exit 1 fi echo "ZONE_ID=${zone_id}" >> "$GITHUB_ENV" diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 5abe013..4d48889 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -9,14 +9,15 @@ name: K8s Stack Preview # (a quick-tunnel URL, a basic-auth password) is readable by anyone who # opens the PR, not just intended reviewers. Access closes that gap by # authenticating the *person*, not a string in the comment: Cloudflare -# challenges every request to *.dspack.iakte.ch with a GitHub SSO -# login and only lets it through to cloudflared if the signed-in account -# is a member of this GitHub org. The PR comment only ever contains a URL. +# challenges every request to *.github.fyi with a GitHub SSO login and +# only lets it through to cloudflared if the signed-in account is a +# member of this GitHub org. The PR comment only ever contains a URL. # -# iakte.ch is a placeholder domain used temporarily because it lives in -# the same Cloudflare account as the Tunnel and Zero Trust setup; swap -# PREVIEW_DOMAIN below once a permanent domain is available in that -# account. +# Hostnames are single-level (pr-.github.fyi, not pr-.dspack.github.fyi) +# deliberately: Cloudflare's free Universal SSL only auto-covers the zone +# apex plus one wildcard level (github.fyi + *.github.fyi); a second level +# needs the paid Advanced Certificate Manager add-on, which this setup +# doesn't use. # # Scope, deliberately: the tunnel points straight at the `proxy-public` # service (dummy-authenticator login, same as local Tilt dev), not the @@ -33,12 +34,15 @@ name: K8s Stack Preview # # One-time setup this workflow assumes already exists (Cloudflare Zero # Trust dashboard, done by a repo admin, not scripted here). The tunnel, -# the `iakte.ch` zone, and Zero Trust/Access all live in ONE Cloudflare +# the `github.fyi` zone, and Zero Trust/Access all live in ONE Cloudflare # account (aktechlabs) — not the account behind CLOUDFLARE_API_TOKEN / # CLOUDFLARE_ACCOUNT_ID, which docs.yml uses for Pages: -# - Zone `iakte.ch` in that account, with an Access self-hosted -# application for `*.dspack.iakte.ch`, GitHub as identity provider, -# policy scoped to this org. +# - Zone `github.fyi` in that account, with an Access self-hosted +# application for `*.github.fyi`, GitHub as identity provider, +# policy scoped to this org. Note this wildcard covers ANY +# single-label subdomain of github.fyi, not just previews — fine as +# long as github.fyi isn't also hosting unrelated services outside +# this org's control. # - Secret CLOUDFLARE_TUNNEL_ACCOUNT_ID: that account's id (the # `cfd_tunnel` API is account-scoped; can't be derived from the # token alone). @@ -47,7 +51,7 @@ name: K8s Stack Preview # * Account -> Cloudflare Tunnel -> Edit # * Zone -> Zone -> Read (to resolve the zone id by name) # * Zone -> DNS -> Edit (to create/delete the CNAME record) -# Zone Resources: Include -> Specific zone -> iakte.ch. +# Zone Resources: Include -> Specific zone -> github.fyi. # Account Resources: Include -> Specific account -> aktechlabs. # # Only runs when a maintainer/collaborator adds the `deploy-preview` label @@ -74,7 +78,7 @@ concurrency: env: PREVIEW_LABEL: deploy-preview - PREVIEW_DOMAIN: dspack.iakte.ch + PREVIEW_DOMAIN: github.fyi CLOUDFLARED_VERSION: "2026.7.3" # sha256 of cloudflared-linux-amd64 for the pinned version above, # computed from the official release asset at @@ -206,10 +210,10 @@ jobs: CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} PREVIEW_HOSTNAME: pr-${{ github.event.pull_request.number }}.${{ env.PREVIEW_DOMAIN }} run: | - zone_id=$(curl -fsS "https://api.cloudflare.com/client/v4/zones?name=iakte.ch" \ + zone_id=$(curl -fsS "https://api.cloudflare.com/client/v4/zones?name=github.fyi" \ -H "Authorization: Bearer ${CF_API_TOKEN}" | jq -r '.result[0].id') if [ -z "$zone_id" ] || [ "$zone_id" = "null" ]; then - echo "::error::Could not resolve zone id for iakte.ch" + echo "::error::Could not resolve zone id for github.fyi" exit 1 fi echo "ZONE_ID=${zone_id}" >> "$GITHUB_ENV" From fedb68f84d84002722c74b6f32cb685e5cd0a7a0 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 3 Aug 2026 15:12:40 +0100 Subject: [PATCH 05/54] ci: drop temporary pull_request trigger from smoketest workflow workflow_dispatch is now registered and reachable; the iteration workaround is no longer needed. --- .github/workflows/k8s-preview-tunnel-smoketest.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/k8s-preview-tunnel-smoketest.yaml b/.github/workflows/k8s-preview-tunnel-smoketest.yaml index cc7b601..61786f0 100644 --- a/.github/workflows/k8s-preview-tunnel-smoketest.yaml +++ b/.github/workflows/k8s-preview-tunnel-smoketest.yaml @@ -12,14 +12,6 @@ name: K8s Preview Tunnel Smoketest on: workflow_dispatch: - # TEMPORARY, for iterating on this PR before workflow_dispatch is - # reachable (it needs the file on the default branch first). Scoped to - # this file only so it can't fire on unrelated PRs. Remove this - # `pull_request` block before merging. - pull_request: - types: [synchronize] - paths: - - .github/workflows/k8s-preview-tunnel-smoketest.yaml env: PREVIEW_DOMAIN: github.fyi From b22f710106a2bb39ff78744806696ddbb8ea5930 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 3 Aug 2026 15:18:56 +0100 Subject: [PATCH 06/54] fix: fetch chart dependencies before helm install in preview workflow charts/ is gitignored, so the vendored jupyterhub-4.3.2.tgz seen in local dev never reaches a fresh CI checkout. helm dependency build resolves it against the already-committed Chart.lock digest. --- .github/workflows/k8s-preview.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 4d48889..3dde5ea 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -116,6 +116,13 @@ jobs: - name: Side-load hub image into the sandbox cluster run: k3d image import nebari-data-science-pack-jupyterhub:preview -c ${{ steps.sandbox.outputs.cluster-name }} + # charts/ is gitignored (dependency .tgz files aren't committed), so a + # fresh checkout needs this before `helm upgrade --install` can find + # the jupyterhub subchart. Resolves against the version/digest already + # pinned in the committed Chart.lock, not a new or bumped dependency. + - name: Fetch chart dependencies + run: helm dependency build . + - name: Deploy chart id: deploy env: From 1c7618ad43de5534e0c74eb522eda3e5a467abb0 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 3 Aug 2026 15:21:03 +0100 Subject: [PATCH 07/54] ci: post smoketest URL as a PR comment via optional pr_number input workflow_dispatch has no PR context to auto-detect, so the comment step uses number_force (sticky-pull-request-comment's any-event PR number override, distinct from number which is push-event only). --- .../k8s-preview-tunnel-smoketest.yaml | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/k8s-preview-tunnel-smoketest.yaml b/.github/workflows/k8s-preview-tunnel-smoketest.yaml index 61786f0..ac08747 100644 --- a/.github/workflows/k8s-preview-tunnel-smoketest.yaml +++ b/.github/workflows/k8s-preview-tunnel-smoketest.yaml @@ -12,6 +12,10 @@ name: K8s Preview Tunnel Smoketest on: workflow_dispatch: + inputs: + pr_number: + description: 'PR number to post the smoketest URL to (optional; skips the comment if blank)' + required: false env: PREVIEW_DOMAIN: github.fyi @@ -25,6 +29,7 @@ jobs: timeout-minutes: 15 permissions: contents: read + pull-requests: write steps: - name: Serve a trivial static page run: | @@ -112,6 +117,21 @@ jobs: echo "Visiting it should challenge you with Cloudflare Access GitHub SSO," >> "$GITHUB_STEP_SUMMARY" echo "then show 'Tunnel smoketest OK' once you're through." >> "$GITHUB_STEP_SUMMARY" echo "Live for up to 15 minutes (this job's timeout)." >> "$GITHUB_STEP_SUMMARY" + echo "url=https://${PREVIEW_HOSTNAME}" >> "$GITHUB_OUTPUT" + + - name: Comment smoketest link on PR + if: github.event.inputs.pr_number != '' + uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 + with: + header: k8s-preview-smoketest + number_force: ${{ github.event.inputs.pr_number }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + message: | + **Tunnel smoketest** (static page, not the real stack): + ${{ steps.cf_dns.outputs.url }} + + Sign-in via Cloudflare Access (GitHub SSO), then should show "Tunnel smoketest OK". + Live for up to 15 minutes from this run. - name: Run tunnel until the job times out run: /tmp/cloudflared tunnel --no-autoupdate run --token "${TUNNEL_TOKEN}" From 0a75683f6c906071d52b2b806d865e0cd5e69336 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 3 Aug 2026 15:28:51 +0100 Subject: [PATCH 08/54] fix: register the jupyterhub chart repo before dependency build helm dependency build resolves against locally-registered repos, not just the Chart.lock digest; a fresh runner has none configured. Added helm repo add before the build step. Verified against a clean clone. --- .github/workflows/k8s-preview.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 3dde5ea..23c8428 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -121,7 +121,9 @@ jobs: # the jupyterhub subchart. Resolves against the version/digest already # pinned in the committed Chart.lock, not a new or bumped dependency. - name: Fetch chart dependencies - run: helm dependency build . + run: | + helm repo add jupyterhub https://hub.jupyter.org/helm-chart/ + helm dependency build . - name: Deploy chart id: deploy From 1eec127f9e1f77806be0b3e1592c9cb1700d7cc6 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 3 Aug 2026 15:43:30 +0100 Subject: [PATCH 09/54] fix: opt preview namespace into nic-operator management, add failure debugging nic-operator ignores any namespace without nebari.dev/managed=true, so the NebariApp CRD never reconciled. Also adds a static pod/job/event dump and an interactive tmate SSH session (actor-restricted, 20min cap) on deploy failure, since the cluster is deleted right after and static logs alone weren't enough to diagnose the last two failures. --- .github/workflows/k8s-preview.yaml | 43 +++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 23c8428..4d9e6bd 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -125,13 +125,23 @@ jobs: helm repo add jupyterhub https://hub.jupyter.org/helm-chart/ helm dependency build . + # nic-operator's CoreReconciler ignores any namespace without this + # label (NamespaceNotOptedIn) — without it the NebariApp never + # reconciles regardless of what the chart deploys. + - name: Create and opt-in the preview namespace + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + run: | + kubectl create namespace pr-preview + kubectl label namespace pr-preview nebari.dev/managed=true + - name: Deploy chart id: deploy env: KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} run: | helm upgrade --install preview . \ - --namespace pr-preview --create-namespace \ + --namespace pr-preview \ --set jupyterhub.hub.image.name=nebari-data-science-pack-jupyterhub \ --set jupyterhub.hub.image.tag=preview \ --set nebariapp.enabled=true \ @@ -154,6 +164,37 @@ jobs: kubectl -n pr-preview get nebariapp -o wide || true kubectl -n pr-preview describe nebariapp preview-nebari-data-science-pack || true + # Cleanup deletes the whole cluster next, so this is the only chance + # to see why a pod/job didn't reach Ready if `helm --wait` timed out. + - name: Debug pod/job status on deploy failure + if: failure() + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + run: | + kubectl -n pr-preview get pods -o wide || true + kubectl -n pr-preview get jobs || true + kubectl -n pr-preview get events --sort-by=.lastTimestamp || true + for pod in $(kubectl -n pr-preview get pods -o name 2>/dev/null); do + echo "--- describe $pod ---" + kubectl -n pr-preview describe "$pod" || true + echo "--- logs $pod ---" + kubectl -n pr-preview logs "$pod" --all-containers --tail=100 || true + done + + # Interactive SSH debug session into the live runner (cluster still + # up, KUBECONFIG still valid) instead of guessing blind from static + # logs. limit-access-to-actor restricts the SSH session to whoever + # triggered this run — required on a public repo. Bounded to 20min + # so a forgotten session doesn't eat the whole 90min job timeout. + - name: Debug via tmate SSH on deploy failure + if: failure() + uses: mxschmitt/action-tmate@35b54afac29c97fb54faba5b513f8fbd1882f113 # v3.24 + timeout-minutes: 20 + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + with: + limit-access-to-actor: true + - name: Port-forward JupyterHub proxy env: KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} From 17f723b74dd9240479290a59cc7f4b9c8e6da493 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 3 Aug 2026 16:03:16 +0100 Subject: [PATCH 10/54] fix: disable nebariapp for preview deploy, hub crash-loops otherwise 00-gateway-auth.py reads /etc/oauth/issuer-url unconditionally at import time. That file only exists once the operator's async Keycloak client provisioning finishes, which Helm doesn't wait for, so the hub pod crash-loops immediately when nebariapp.auth.enabled=true. Confirmed via kubectl logs during a failed run. Dropping nebariapp entirely for this preview; operator/OIDC reconcile isn't exercised here. --- .github/workflows/k8s-preview.yaml | 45 ++++++++---------------------- 1 file changed, 12 insertions(+), 33 deletions(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 4d9e6bd..e780243 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -20,12 +20,13 @@ name: K8s Stack Preview # doesn't use. # # Scope, deliberately: the tunnel points straight at the `proxy-public` -# service (dummy-authenticator login, same as local Tilt dev), not the -# operator-provisioned NebariApp/OIDC route — that route exists for its -# own Keycloak realm and redirect URIs, unrelated to Access's GitHub SSO. -# The NebariApp CRD is still installed (auth.enabled=true) purely so the -# operator/Keycloak reconcile is exercised and its Ready condition -# reported in the run; it is not what the reviewer clicks through. +# service (dummy-authenticator login, same as local Tilt dev). The chart +# deploys with nebariapp.enabled=false — nebariapp.auth.enabled=true +# was tried to additionally exercise the operator/Keycloak reconcile, +# but 00-gateway-auth.py reads /etc/oauth/issuer-url unconditionally at +# import time, before the operator's async client provisioning can ever +# populate it, so the hub pod crash-loops. Not worth chasing for a +# preview link; the operator/OIDC path stays untested here. # # The link only lives for the run's duration (bounded by timeout-minutes # below) — it is not a persistent per-PR environment. Each run creates its @@ -125,28 +126,16 @@ jobs: helm repo add jupyterhub https://hub.jupyter.org/helm-chart/ helm dependency build . - # nic-operator's CoreReconciler ignores any namespace without this - # label (NamespaceNotOptedIn) — without it the NebariApp never - # reconciles regardless of what the chart deploys. - - name: Create and opt-in the preview namespace - env: - KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} - run: | - kubectl create namespace pr-preview - kubectl label namespace pr-preview nebari.dev/managed=true - - name: Deploy chart id: deploy env: KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} run: | helm upgrade --install preview . \ - --namespace pr-preview \ + --namespace pr-preview --create-namespace \ --set jupyterhub.hub.image.name=nebari-data-science-pack-jupyterhub \ --set jupyterhub.hub.image.tag=preview \ - --set nebariapp.enabled=true \ - --set nebariapp.hostname="pr-${{ github.event.pull_request.number }}.${{ steps.sandbox.outputs.domain }}" \ - --set nebariapp.auth.enabled=true \ + --set nebariapp.enabled=false \ --wait --timeout 5m - name: Wait for hub + proxy @@ -156,14 +145,6 @@ jobs: kubectl -n pr-preview rollout status deployment/hub --timeout=180s kubectl -n pr-preview rollout status deployment/proxy --timeout=180s - - name: Report NebariApp reconcile status - if: always() - env: - KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} - run: | - kubectl -n pr-preview get nebariapp -o wide || true - kubectl -n pr-preview describe nebariapp preview-nebari-data-science-pack || true - # Cleanup deletes the whole cluster next, so this is the only chance # to see why a pod/job didn't reach Ready if `helm --wait` timed out. - name: Debug pod/job status on deploy failure @@ -296,11 +277,9 @@ jobs: reaching JupyterHub — only members of this GitHub org get through. **Then JupyterHub login:** dummy authenticator, any username + any password. - This goes straight to JupyterHub's proxy and skips the operator-provisioned - OIDC/Keycloak route (unrelated realm/redirect URIs to Access's GitHub SSO). - The `NebariApp` resource is still deployed alongside it so the operator/ - Keycloak reconcile itself is exercised — see the "Report NebariApp reconcile - status" step in this run for its `Ready` condition. + This goes straight to JupyterHub's proxy; the operator-provisioned + NebariApp/OIDC route isn't deployed here (see workflow header comment + for why) — this preview doesn't exercise operator/Keycloak reconcile. Live only for this job's run (up to 90 min), or until the `deploy-preview` label is removed. Push a new commit or re-add the label to redeploy. From a43dfb707f0741d07e9931683a53eb5842dcb0a3 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 3 Aug 2026 16:49:59 +0100 Subject: [PATCH 11/54] fix: set jupyterhub.custom.external-url to the tunnel hostname Without it, 02-jhub-apps.py falls back to bind_url=http://0.0.0.0:8000, which JupyterHub then bakes into browser-facing OAuth redirect URLs for the jhub-apps service (client_id=service-japps), breaking the login flow after dummy-auth. Confirmed via a real login attempt on the deployed preview. --- .github/workflows/k8s-preview.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index e780243..ab28466 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -136,6 +136,7 @@ jobs: --set jupyterhub.hub.image.name=nebari-data-science-pack-jupyterhub \ --set jupyterhub.hub.image.tag=preview \ --set nebariapp.enabled=false \ + --set jupyterhub.custom.external-url="pr-${{ github.event.pull_request.number }}.${{ env.PREVIEW_DOMAIN }}" \ --wait --timeout 5m - name: Wait for hub + proxy From a14661698c645e168b2b027e0380e7527006a490 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Wed, 5 Aug 2026 13:32:44 +0100 Subject: [PATCH 12/54] ci: show deploy/expiry timestamps in preview PR comments The preview URL is stable per-PR, so redeploys posted byte-identical comment text and looked like they never updated. Adding deployed-at and expires-at timestamps makes every redeploy visibly change the comment and tells reviewers exactly when the link goes dead. --- .../workflows/k8s-preview-tunnel-smoketest.yaml | 9 ++++++++- .github/workflows/k8s-preview.yaml | 15 +++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/.github/workflows/k8s-preview-tunnel-smoketest.yaml b/.github/workflows/k8s-preview-tunnel-smoketest.yaml index ac08747..c2e0c04 100644 --- a/.github/workflows/k8s-preview-tunnel-smoketest.yaml +++ b/.github/workflows/k8s-preview-tunnel-smoketest.yaml @@ -119,6 +119,12 @@ jobs: echo "Live for up to 15 minutes (this job's timeout)." >> "$GITHUB_STEP_SUMMARY" echo "url=https://${PREVIEW_HOSTNAME}" >> "$GITHUB_OUTPUT" + - name: Compute deployment timestamps + id: timestamps + run: | + echo "deployed_at=$(date -u +'%Y-%m-%d %H:%M UTC')" >> "$GITHUB_OUTPUT" + echo "expires_at=$(date -u -d '+15 minutes' +'%Y-%m-%d %H:%M UTC')" >> "$GITHUB_OUTPUT" + - name: Comment smoketest link on PR if: github.event.inputs.pr_number != '' uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 @@ -130,8 +136,9 @@ jobs: **Tunnel smoketest** (static page, not the real stack): ${{ steps.cf_dns.outputs.url }} + Deployed: ${{ steps.timestamps.outputs.deployed_at }} · Expires: ${{ steps.timestamps.outputs.expires_at }} + Sign-in via Cloudflare Access (GitHub SSO), then should show "Tunnel smoketest OK". - Live for up to 15 minutes from this run. - name: Run tunnel until the job times out run: /tmp/cloudflared tunnel --no-autoupdate run --token "${TUNNEL_TOKEN}" diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index ab28466..d99e51a 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -263,6 +263,15 @@ jobs: echo "DNS_RECORD_ID=${record_id}" >> "$GITHUB_ENV" echo "url=https://${PREVIEW_HOSTNAME}" >> "$GITHUB_OUTPUT" + # The URL itself (pr-.github.fyi) is identical on every run, so + # without a timestamp the sticky comment would post byte-identical + # text each redeploy and look like it never updated. + - name: Compute deployment timestamps + id: timestamps + run: | + echo "deployed_at=$(date -u +'%Y-%m-%d %H:%M UTC')" >> "$GITHUB_OUTPUT" + echo "expires_at=$(date -u -d '+90 minutes' +'%Y-%m-%d %H:%M UTC')" >> "$GITHUB_OUTPUT" + - name: Comment preview link on PR uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 with: @@ -272,6 +281,8 @@ jobs: **K8s stack preview** for `${{ github.event.pull_request.head.ref }}`: ${{ steps.cf_dns.outputs.url }} + Deployed: ${{ steps.timestamps.outputs.deployed_at }} · Expires: ${{ steps.timestamps.outputs.expires_at }} + ${{ github.event.pull_request.head.repo.fork && '⚠️ **This PR is from a fork** — the code running in this preview is not from a trusted maintainer branch.' || '' }} You'll be asked to sign in via Cloudflare Access (GitHub SSO) before @@ -282,8 +293,8 @@ jobs: NebariApp/OIDC route isn't deployed here (see workflow header comment for why) — this preview doesn't exercise operator/Keycloak reconcile. - Live only for this job's run (up to 90 min), or until the `deploy-preview` - label is removed. Push a new commit or re-add the label to redeploy. + Live until the expiry time above, or until the `deploy-preview` label + is removed. Push a new commit or re-add the label to redeploy. - name: Run tunnel until the job times out run: /tmp/cloudflared tunnel --no-autoupdate run --token "${TUNNEL_TOKEN}" From 65651b453af49420ff3922c712ee8b620449f2d8 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Wed, 5 Aug 2026 15:14:41 +0100 Subject: [PATCH 13/54] fix: make tunnel creation idempotent, retries reuse the run_id A workflow retry keeps the same github.run_id (only run_attempt changes), so a re-run after an attempt already created the tunnel but didn't reach its cleanup step hit a 409 name conflict. Now falls back to looking up and reusing the existing tunnel by name instead of failing outright. Confirmed via a real run_attempt=2 failure. --- .../k8s-preview-tunnel-smoketest.yaml | 19 +++++++++++---- .github/workflows/k8s-preview.yaml | 23 +++++++++++++++---- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/.github/workflows/k8s-preview-tunnel-smoketest.yaml b/.github/workflows/k8s-preview-tunnel-smoketest.yaml index c2e0c04..be92bb7 100644 --- a/.github/workflows/k8s-preview-tunnel-smoketest.yaml +++ b/.github/workflows/k8s-preview-tunnel-smoketest.yaml @@ -58,15 +58,24 @@ jobs: tunnel_secret=$(openssl rand -base64 32) echo "::add-mask::${tunnel_secret}" - create_resp=$(curl -fsS -X POST \ + tunnel_name="smoketest-${{ github.run_id }}" + create_resp=$(curl -sS -X POST \ "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel" \ -H "Authorization: Bearer ${CF_API_TOKEN}" \ -H "Content-Type: application/json" \ - -d "$(jq -n --arg name "smoketest-${{ github.run_id }}" --arg secret "$tunnel_secret" \ + -d "$(jq -n --arg name "$tunnel_name" --arg secret "$tunnel_secret" \ '{name: $name, config_src: "cloudflare", tunnel_secret: $secret}')") - tunnel_id=$(jq -r '.result.id' <<< "$create_resp") - if [ -z "$tunnel_id" ] || [ "$tunnel_id" = "null" ]; then - echo "::error::Tunnel creation failed: $create_resp" + tunnel_id=$(jq -r '.result.id // empty' <<< "$create_resp") + + # Same retry-safety as the real workflow: a retry reuses run_id, + # so reuse the existing tunnel by name on a 409 name conflict. + if [ -z "$tunnel_id" ]; then + echo "::warning::Tunnel create failed (likely a name conflict from a retry), looking up existing tunnel named ${tunnel_name}: $create_resp" + tunnel_id=$(curl -fsS "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel?name=${tunnel_name}&is_deleted=false" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" | jq -r '.result[0].id // empty') + fi + if [ -z "$tunnel_id" ]; then + echo "::error::Tunnel creation failed and no existing tunnel named ${tunnel_name} found: $create_resp" exit 1 fi echo "TUNNEL_ID=${tunnel_id}" >> "$GITHUB_ENV" diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index d99e51a..3347767 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -207,15 +207,28 @@ jobs: tunnel_secret=$(openssl rand -base64 32) echo "::add-mask::${tunnel_secret}" - create_resp=$(curl -fsS -X POST \ + tunnel_name="pr-${{ github.event.pull_request.number }}-${{ github.run_id }}" + create_resp=$(curl -sS -X POST \ "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel" \ -H "Authorization: Bearer ${CF_API_TOKEN}" \ -H "Content-Type: application/json" \ - -d "$(jq -n --arg name "pr-${{ github.event.pull_request.number }}-${{ github.run_id }}" --arg secret "$tunnel_secret" \ + -d "$(jq -n --arg name "$tunnel_name" --arg secret "$tunnel_secret" \ '{name: $name, config_src: "cloudflare", tunnel_secret: $secret}')") - tunnel_id=$(jq -r '.result.id' <<< "$create_resp") - if [ -z "$tunnel_id" ] || [ "$tunnel_id" = "null" ]; then - echo "::error::Tunnel creation failed: $create_resp" + tunnel_id=$(jq -r '.result.id // empty' <<< "$create_resp") + + # A GitHub Actions retry reuses the same run_id (only run_attempt + # changes), so a re-run after the first attempt already created + # this tunnel (and didn't get to clean it up) hits a 409 name + # conflict here. Reuse the existing tunnel by name instead of + # failing — it doesn't need the original tunnel_secret, just a + # fresh --token from the /token endpoint below. + if [ -z "$tunnel_id" ]; then + echo "::warning::Tunnel create failed (likely a name conflict from a retry), looking up existing tunnel named ${tunnel_name}: $create_resp" + tunnel_id=$(curl -fsS "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel?name=${tunnel_name}&is_deleted=false" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" | jq -r '.result[0].id // empty') + fi + if [ -z "$tunnel_id" ]; then + echo "::error::Tunnel creation failed and no existing tunnel named ${tunnel_name} found: $create_resp" exit 1 fi echo "tunnel_id=${tunnel_id}" >> "$GITHUB_OUTPUT" From 74d3d49815dcaf811977d3367c8830ae2bebdf13 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 31 Aug 2026 12:46:35 +0100 Subject: [PATCH 14/54] ci: source preview domain from a repo Variable instead of hardcoding it PREVIEW_DOMAIN now reads from vars.PREVIEW_DOMAIN (repo Settings -> Actions -> Variables), set to github.fyi, in both the real preview workflow and the smoketest. Repointing to a different domain no longer needs a code change. --- .../k8s-preview-tunnel-smoketest.yaml | 8 ++-- .github/workflows/k8s-preview.yaml | 44 +++++++++++-------- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/.github/workflows/k8s-preview-tunnel-smoketest.yaml b/.github/workflows/k8s-preview-tunnel-smoketest.yaml index be92bb7..30daeb4 100644 --- a/.github/workflows/k8s-preview-tunnel-smoketest.yaml +++ b/.github/workflows/k8s-preview-tunnel-smoketest.yaml @@ -18,7 +18,9 @@ on: required: false env: - PREVIEW_DOMAIN: github.fyi + # Same repo Variable k8s-preview.yaml uses (Settings -> Actions -> + # Variables -> PREVIEW_DOMAIN) so the two workflows can't drift apart. + PREVIEW_DOMAIN: ${{ vars.PREVIEW_DOMAIN }} CLOUDFLARED_VERSION: "2026.7.3" CLOUDFLARED_SHA256: "9d71c677db00134c1bd4144b7783486b654ad281b1ea62b4972098d19f770f17" @@ -101,10 +103,10 @@ jobs: CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} PREVIEW_HOSTNAME: smoketest-${{ github.run_id }}.${{ env.PREVIEW_DOMAIN }} run: | - zone_id=$(curl -fsS "https://api.cloudflare.com/client/v4/zones?name=github.fyi" \ + zone_id=$(curl -fsS "https://api.cloudflare.com/client/v4/zones?name=${PREVIEW_DOMAIN}" \ -H "Authorization: Bearer ${CF_API_TOKEN}" | jq -r '.result[0].id') if [ -z "$zone_id" ] || [ "$zone_id" = "null" ]; then - echo "::error::Could not resolve zone id for github.fyi" + echo "::error::Could not resolve zone id for ${PREVIEW_DOMAIN}" exit 1 fi echo "ZONE_ID=${zone_id}" >> "$GITHUB_ENV" diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 3347767..73cd061 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -9,15 +9,18 @@ name: K8s Stack Preview # (a quick-tunnel URL, a basic-auth password) is readable by anyone who # opens the PR, not just intended reviewers. Access closes that gap by # authenticating the *person*, not a string in the comment: Cloudflare -# challenges every request to *.github.fyi with a GitHub SSO login and -# only lets it through to cloudflared if the signed-in account is a -# member of this GitHub org. The PR comment only ever contains a URL. +# challenges every request to *. with a GitHub SSO +# login and only lets it through to cloudflared if the signed-in account +# is a member of this GitHub org. The PR comment only ever contains a URL. # -# Hostnames are single-level (pr-.github.fyi, not pr-.dspack.github.fyi) -# deliberately: Cloudflare's free Universal SSL only auto-covers the zone -# apex plus one wildcard level (github.fyi + *.github.fyi); a second level -# needs the paid Advanced Certificate Manager add-on, which this setup -# doesn't use. +# The domain is a repo Variable (Settings -> Secrets and variables -> +# Actions -> Variables -> PREVIEW_DOMAIN, currently github.fyi), not +# hardcoded, so it can be repointed without editing this file. Hostnames +# built from it stay single-level (pr-., never +# pr-.dspack.) deliberately: Cloudflare's free Universal SSL +# only auto-covers the zone apex plus one wildcard level ( + +# *.); a second level needs the paid Advanced Certificate +# Manager add-on, which this setup doesn't use. # # Scope, deliberately: the tunnel points straight at the `proxy-public` # service (dummy-authenticator login, same as local Tilt dev). The chart @@ -35,14 +38,17 @@ name: K8s Stack Preview # # One-time setup this workflow assumes already exists (Cloudflare Zero # Trust dashboard, done by a repo admin, not scripted here). The tunnel, -# the `github.fyi` zone, and Zero Trust/Access all live in ONE Cloudflare -# account (aktechlabs) — not the account behind CLOUDFLARE_API_TOKEN / -# CLOUDFLARE_ACCOUNT_ID, which docs.yml uses for Pages: -# - Zone `github.fyi` in that account, with an Access self-hosted -# application for `*.github.fyi`, GitHub as identity provider, +# the PREVIEW_DOMAIN zone, and Zero Trust/Access all live in ONE +# Cloudflare account (aktechlabs) — not the account behind +# CLOUDFLARE_API_TOKEN / CLOUDFLARE_ACCOUNT_ID, which docs.yml uses for +# Pages: +# - Variable PREVIEW_DOMAIN (repo Settings -> Actions -> Variables): +# the zone name, e.g. github.fyi. +# - That zone in the Cloudflare account, with an Access self-hosted +# application for `*.`, GitHub as identity provider, # policy scoped to this org. Note this wildcard covers ANY -# single-label subdomain of github.fyi, not just previews — fine as -# long as github.fyi isn't also hosting unrelated services outside +# single-label subdomain of the zone, not just previews — fine as +# long as the zone isn't also hosting unrelated services outside # this org's control. # - Secret CLOUDFLARE_TUNNEL_ACCOUNT_ID: that account's id (the # `cfd_tunnel` API is account-scoped; can't be derived from the @@ -52,7 +58,7 @@ name: K8s Stack Preview # * Account -> Cloudflare Tunnel -> Edit # * Zone -> Zone -> Read (to resolve the zone id by name) # * Zone -> DNS -> Edit (to create/delete the CNAME record) -# Zone Resources: Include -> Specific zone -> github.fyi. +# Zone Resources: Include -> Specific zone -> the PREVIEW_DOMAIN zone. # Account Resources: Include -> Specific account -> aktechlabs. # # Only runs when a maintainer/collaborator adds the `deploy-preview` label @@ -79,7 +85,7 @@ concurrency: env: PREVIEW_LABEL: deploy-preview - PREVIEW_DOMAIN: github.fyi + PREVIEW_DOMAIN: ${{ vars.PREVIEW_DOMAIN }} CLOUDFLARED_VERSION: "2026.7.3" # sha256 of cloudflared-linux-amd64 for the pinned version above, # computed from the official release asset at @@ -255,10 +261,10 @@ jobs: CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} PREVIEW_HOSTNAME: pr-${{ github.event.pull_request.number }}.${{ env.PREVIEW_DOMAIN }} run: | - zone_id=$(curl -fsS "https://api.cloudflare.com/client/v4/zones?name=github.fyi" \ + zone_id=$(curl -fsS "https://api.cloudflare.com/client/v4/zones?name=${PREVIEW_DOMAIN}" \ -H "Authorization: Bearer ${CF_API_TOKEN}" | jq -r '.result[0].id') if [ -z "$zone_id" ] || [ "$zone_id" = "null" ]; then - echo "::error::Could not resolve zone id for github.fyi" + echo "::error::Could not resolve zone id for ${PREVIEW_DOMAIN}" exit 1 fi echo "ZONE_ID=${zone_id}" >> "$GITHUB_ENV" From 71817f5e31df36f43afbda4ad1e00f30bc9007ec Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 31 Aug 2026 13:07:28 +0100 Subject: [PATCH 15/54] ci: switch preview domain to openteams.app, flatten per-PR hostname Same account already holds both the tunnel (Cloudflare Tunnel:Edit) and the openteams.app zone (Zone:Read, DNS:Edit) on the existing CLOUDFLARE_TUNNEL_API_TOKEN, so no new token needed. Hostnames flatten to pr--data-science-pack.openteams.app (single label) to stay within Cloudflare's free Universal SSL one-wildcard-level coverage. --- .github/workflows/k8s-preview.yaml | 32 +++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 73cd061..2564fdd 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -14,13 +14,13 @@ name: K8s Stack Preview # is a member of this GitHub org. The PR comment only ever contains a URL. # # The domain is a repo Variable (Settings -> Secrets and variables -> -# Actions -> Variables -> PREVIEW_DOMAIN, currently github.fyi), not +# Actions -> Variables -> PREVIEW_DOMAIN, currently openteams.app), not # hardcoded, so it can be repointed without editing this file. Hostnames -# built from it stay single-level (pr-., never -# pr-.dspack.) deliberately: Cloudflare's free Universal SSL -# only auto-covers the zone apex plus one wildcard level ( + -# *.); a second level needs the paid Advanced Certificate -# Manager add-on, which this setup doesn't use. +# built from it stay single-level (pr--data-science-pack., +# never pr-.data-science-pack.) deliberately: Cloudflare's +# free Universal SSL only auto-covers the zone apex plus one wildcard +# level ( + *.); a second level needs the paid Advanced +# Certificate Manager add-on, which this setup doesn't use. # # Scope, deliberately: the tunnel points straight at the `proxy-public` # service (dummy-authenticator login, same as local Tilt dev). The chart @@ -39,16 +39,16 @@ name: K8s Stack Preview # One-time setup this workflow assumes already exists (Cloudflare Zero # Trust dashboard, done by a repo admin, not scripted here). The tunnel, # the PREVIEW_DOMAIN zone, and Zero Trust/Access all live in ONE -# Cloudflare account (aktechlabs) — not the account behind +# Cloudflare account (OpenTeams Account) — not the account behind # CLOUDFLARE_API_TOKEN / CLOUDFLARE_ACCOUNT_ID, which docs.yml uses for # Pages: # - Variable PREVIEW_DOMAIN (repo Settings -> Actions -> Variables): -# the zone name, e.g. github.fyi. +# the zone name, e.g. openteams.app. # - That zone in the Cloudflare account, with an Access self-hosted # application for `*.`, GitHub as identity provider, -# policy scoped to this org. Note this wildcard covers ANY -# single-label subdomain of the zone, not just previews — fine as -# long as the zone isn't also hosting unrelated services outside +# policy scoped to this org (nebari-dev). Note this wildcard covers +# ANY single-label subdomain of the zone, not just previews — fine +# as long as the zone isn't also hosting unrelated services outside # this org's control. # - Secret CLOUDFLARE_TUNNEL_ACCOUNT_ID: that account's id (the # `cfd_tunnel` API is account-scoped; can't be derived from the @@ -59,7 +59,7 @@ name: K8s Stack Preview # * Zone -> Zone -> Read (to resolve the zone id by name) # * Zone -> DNS -> Edit (to create/delete the CNAME record) # Zone Resources: Include -> Specific zone -> the PREVIEW_DOMAIN zone. -# Account Resources: Include -> Specific account -> aktechlabs. +# Account Resources: Include -> Specific account -> OpenTeams Account. # # Only runs when a maintainer/collaborator adds the `deploy-preview` label # (GitHub restricts who can label a PR) — arbitrary PR authors, including @@ -142,7 +142,7 @@ jobs: --set jupyterhub.hub.image.name=nebari-data-science-pack-jupyterhub \ --set jupyterhub.hub.image.tag=preview \ --set nebariapp.enabled=false \ - --set jupyterhub.custom.external-url="pr-${{ github.event.pull_request.number }}.${{ env.PREVIEW_DOMAIN }}" \ + --set jupyterhub.custom.external-url="pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }}" \ --wait --timeout 5m - name: Wait for hub + proxy @@ -208,7 +208,7 @@ jobs: env: CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} CF_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_TUNNEL_ACCOUNT_ID }} - PREVIEW_HOSTNAME: pr-${{ github.event.pull_request.number }}.${{ env.PREVIEW_DOMAIN }} + PREVIEW_HOSTNAME: pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }} run: | tunnel_secret=$(openssl rand -base64 32) echo "::add-mask::${tunnel_secret}" @@ -259,7 +259,7 @@ jobs: id: cf_dns env: CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} - PREVIEW_HOSTNAME: pr-${{ github.event.pull_request.number }}.${{ env.PREVIEW_DOMAIN }} + PREVIEW_HOSTNAME: pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }} run: | zone_id=$(curl -fsS "https://api.cloudflare.com/client/v4/zones?name=${PREVIEW_DOMAIN}" \ -H "Authorization: Bearer ${CF_API_TOKEN}" | jq -r '.result[0].id') @@ -282,7 +282,7 @@ jobs: echo "DNS_RECORD_ID=${record_id}" >> "$GITHUB_ENV" echo "url=https://${PREVIEW_HOSTNAME}" >> "$GITHUB_OUTPUT" - # The URL itself (pr-.github.fyi) is identical on every run, so + # The URL itself (pr--data-science-pack.) is identical on every run, so # without a timestamp the sticky comment would post byte-identical # text each redeploy and look like it never updated. - name: Compute deployment timestamps From 4919f731fcf0def87e484128d31776e6a17d747c Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 31 Aug 2026 13:37:37 +0100 Subject: [PATCH 16/54] ci: migrate k8s-preview to action-nebari-sandbox v3 (kind, not k3d) Drops the removed profile/k3d-version/k8s-version inputs, installs kind via helm/kind-action, swaps k3d image import + manual cluster delete for kind load docker-image (v3 auto-tears-down the cluster on its own). --- .github/workflows/k8s-preview.yaml | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 2564fdd..cee76a5 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -1,7 +1,7 @@ name: K8s Stack Preview # Deploys the full Nebari platform stack (Keycloak + nic-operator + Envoy -# Gateway, via nebari-dev/action-nebari-sandbox's `platform` profile) plus -# this PR's chart into an ephemeral k3d cluster on the runner, then exposes +# Gateway, via nebari-dev/action-nebari-sandbox) plus this PR's chart into +# an ephemeral kind cluster on the runner, then exposes # JupyterHub through a per-PR Cloudflare Tunnel behind Cloudflare Access # (GitHub-org SSO) so a reviewer can click a link and use it. # @@ -67,8 +67,8 @@ name: K8s Stack Preview # attacker-authored code once labeled; the comment flags this so whoever # labels it is doing so knowingly. # -# Residual risk not covered here: k3d pods share the runner's Docker daemon -# rather than being hardware-isolated, and k3d's default CNI (flannel) +# Residual risk not covered here: kind nodes share the runner's Docker daemon +# rather than being hardware-isolated, and kind's default CNI (kindnet) # does not enforce NetworkPolicy, so a container escape or outbound abuse # from inside a spawned notebook pod is not blocked at the network layer. # Per-job GITHUB_TOKEN permissions are scoped to the minimum each job @@ -105,23 +105,23 @@ jobs: - name: Checkout uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - name: Provision sandbox (k3d + full NIC platform stack) + - name: Install kind + uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0 + with: + version: v0.27.0 + install_only: true + + - name: Provision sandbox (kind + full NIC platform stack) id: sandbox - uses: nebari-dev/action-nebari-sandbox@dbdb054c16efee3b29e26aee406c12fb9639bfd0 # v2.2.0 + uses: nebari-dev/action-nebari-sandbox@9ac369ebf87ac2ae217504dcbf824c77f70e429a # v3.0.0 with: - profile: platform cluster-name: pr-preview-${{ github.event.pull_request.number }} - name: Build hub image from this PR run: docker build --target jupyterhub -t nebari-data-science-pack-jupyterhub:preview images/ - - name: Install k3d CLI (for image import) - run: | - curl -fsSL https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | \ - TAG=v5.8.3 bash - - name: Side-load hub image into the sandbox cluster - run: k3d image import nebari-data-science-pack-jupyterhub:preview -c ${{ steps.sandbox.outputs.cluster-name }} + run: kind load docker-image nebari-data-science-pack-jupyterhub:preview --name ${{ steps.sandbox.outputs.cluster-name }} # charts/ is gitignored (dependency .tgz files aren't committed), so a # fresh checkout needs this before `helm upgrade --install` can find @@ -339,10 +339,6 @@ jobs: "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel/${TUNNEL_ID}" \ -H "Authorization: Bearer ${CF_API_TOKEN}" || true - - name: Cleanup cluster - if: always() - run: k3d cluster delete ${{ steps.sandbox.outputs.cluster-name }} || true - cleanup-preview: if: github.event.action == 'unlabeled' && github.event.label.name == 'deploy-preview' name: Stop preview From c849c85f55e13feab10254a450a404cc177c9f1d Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 31 Aug 2026 13:44:37 +0100 Subject: [PATCH 17/54] ci: bump kind CLI to v0.33.0 in k8s-preview for containerd config v4 kind load docker-image failed with "unknown containerd config version: 4" against the sandbox's kind node image. Support for that format landed in kind v0.32.0; v0.27.0 (matching test.yaml, whose own kind-created cluster doesn't hit this) predates it. --- .github/workflows/k8s-preview.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index cee76a5..84afa56 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -105,10 +105,13 @@ jobs: - name: Checkout uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + # v0.32.0+ required: the sandbox's kind cluster uses containerd's + # config v4 format, which `kind load` on older CLI releases can't + # parse ("ERROR: unknown containerd config version: 4"). - name: Install kind uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0 with: - version: v0.27.0 + version: v0.33.0 install_only: true - name: Provision sandbox (kind + full NIC platform stack) From cec54bee7635b4dd7f74f37279ae2fd5f5decef9 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 31 Aug 2026 13:54:43 +0100 Subject: [PATCH 18/54] ci: always dump hub logs in k8s-preview to diagnose jhub-apps startup A live browser login through the preview link 502s on /services/japps/oauth_callback. jhub-apps runs as a managed-service subprocess inside the hub pod, so a crash there doesn't fail helm --wait or the rollout status checks -- add an unconditional hub log dump so the cause shows up in the run's own output. --- .github/workflows/k8s-preview.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 84afa56..70d6140 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -155,6 +155,18 @@ jobs: kubectl -n pr-preview rollout status deployment/hub --timeout=180s kubectl -n pr-preview rollout status deployment/proxy --timeout=180s + # jhub-apps runs as a managed service subprocess inside the hub pod, + # not its own Deployment, so a crash there doesn't fail `helm --wait` + # or the rollout checks above -- it only shows up as a 502 on + # /services/japps/* once someone hits it. Always dump the hub log + # (not gated on failure) so a broken jhub-apps subprocess is visible + # in this run's own output instead of only from a live login attempt. + - name: Dump hub logs (jhub-apps startup) + if: always() + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + run: kubectl -n pr-preview logs deployment/hub --tail=500 + # Cleanup deletes the whole cluster next, so this is the only chance # to see why a pod/job didn't reach Ready if `helm --wait` timed out. - name: Debug pod/job status on deploy failure From f577f6a8ac63d50632aea3699d2c0bce96db6124 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 31 Aug 2026 14:05:07 +0100 Subject: [PATCH 19/54] ci: smoke-test jhub-apps directly in k8s-preview before opening the tunnel The prior hub-log dump ran right after startup, before any request had ever reached jhub-apps, so it captured nothing useful. Move the port-forward earlier and curl /services/japps/ straight through it (bypassing Cloudflare) so a crashed or never-bound uvicorn process shows up in the run's own log instead of only surfacing as a 502 on a live login. --- .github/workflows/k8s-preview.yaml | 35 ++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 70d6140..076be02 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -155,12 +155,33 @@ jobs: kubectl -n pr-preview rollout status deployment/hub --timeout=180s kubectl -n pr-preview rollout status deployment/proxy --timeout=180s + - name: Port-forward JupyterHub proxy + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + run: | + kubectl -n pr-preview port-forward svc/proxy-public 8000:80 \ + > /tmp/port-forward.log 2>&1 & + echo "PORT_FORWARD_PID=$!" >> "$GITHUB_ENV" + sleep 3 + # jhub-apps runs as a managed service subprocess inside the hub pod, # not its own Deployment, so a crash there doesn't fail `helm --wait` # or the rollout checks above -- it only shows up as a 502 on - # /services/japps/* once someone hits it. Always dump the hub log - # (not gated on failure) so a broken jhub-apps subprocess is visible - # in this run's own output instead of only from a live login attempt. + # /services/japps/* once someone hits it. Hit its root path directly + # (bypassing Cloudflare, straight to CHP) so a crashed/never-bound + # uvicorn process shows up here instead of only from a live login. + - name: Smoke-test jhub-apps service + if: always() + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + run: | + echo "--- GET /services/japps/ ---" + curl -sS -o /tmp/japps-root.html -w 'HTTP %{http_code}\n' http://localhost:8000/services/japps/ || true + head -c 2000 /tmp/japps-root.html || true + echo + echo "--- japps process in hub pod ---" + kubectl -n pr-preview exec deploy/hub -- ps aux | grep -i "uvicorn\|japps" || true + - name: Dump hub logs (jhub-apps startup) if: always() env: @@ -198,14 +219,6 @@ jobs: with: limit-access-to-actor: true - - name: Port-forward JupyterHub proxy - env: - KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} - run: | - kubectl -n pr-preview port-forward svc/proxy-public 8000:80 \ - > /tmp/port-forward.log 2>&1 & - echo "PORT_FORWARD_PID=$!" >> "$GITHUB_ENV" - - name: Install cloudflared run: | curl -fsSL -o /tmp/cloudflared \ From 10fa3710e2277f3df0accf36b98523d536aaa568 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 31 Aug 2026 14:13:53 +0100 Subject: [PATCH 20/54] ci: capture hub logs after the tunnel closes, not just at startup The startup-time log dump ran before any request ever reached the hub, so a request-triggered failure (e.g. jhub-apps' oauth callback) never showed up in it. Add a second dump gated on the tunnel step ending, so a manual cancel after exercising the preview captures what actually happened. --- .github/workflows/k8s-preview.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 076be02..e55c3cc 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -346,6 +346,15 @@ jobs: - name: Run tunnel until the job times out run: /tmp/cloudflared tunnel --no-autoupdate run --token "${TUNNEL_TOKEN}" + # Runs once the tunnel step above ends (timeout or a manual cancel), + # so this captures anything logged in response to real traffic during + # the tunnel's lifetime -- unlike the earlier startup-time log dump. + - name: Dump hub logs after tunnel closes + if: always() + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + run: kubectl -n pr-preview logs deployment/hub --tail=1000 || true + - name: Delete DNS record if: always() env: From 26f7b65b62b8164819f7ce6f87d3dcc8b6323186 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 31 Aug 2026 14:27:54 +0100 Subject: [PATCH 21/54] ci: cache docker build, enable Keycloak auth, always start tmate debug - Docker layer cache via GH Actions cache (type=gha) so an unchanged pixi.lock/pixi.toml reuses the previous run's apt/pixi install instead of paying it on every fresh runner. - Enable nebariapp + Keycloak auth (was forced off after an earlier crash-loop investigation) so the preview goes through the operator's Keycloak client provisioning instead of the dummy authenticator. - tmate SSH step now always runs (was failure-gated) with a longer timeout, for live debugging on the runner instead of push/cancel/log iteration cycles. --- .github/workflows/k8s-preview.yaml | 31 +++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index e55c3cc..1948dd2 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -120,8 +120,23 @@ jobs: with: cluster-name: pr-preview-${{ github.event.pull_request.number }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + + # Every run gets a fresh runner (empty Docker daemon), so a plain + # `docker build` re-runs every layer -- including the apt/pixi + # installs -- from scratch every time. GitHub Actions cache (type=gha) + # persists layers across runs so an unchanged pixi.lock/pixi.toml + # reuses the previous run's install instead of redoing it. - name: Build hub image from this PR - run: docker build --target jupyterhub -t nebari-data-science-pack-jupyterhub:preview images/ + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: images/ + target: jupyterhub + tags: nebari-data-science-pack-jupyterhub:preview + load: true + cache-from: type=gha,scope=k8s-preview-jupyterhub + cache-to: type=gha,scope=k8s-preview-jupyterhub,mode=max - name: Side-load hub image into the sandbox cluster run: kind load docker-image nebari-data-science-pack-jupyterhub:preview --name ${{ steps.sandbox.outputs.cluster-name }} @@ -144,7 +159,8 @@ jobs: --namespace pr-preview --create-namespace \ --set jupyterhub.hub.image.name=nebari-data-science-pack-jupyterhub \ --set jupyterhub.hub.image.tag=preview \ - --set nebariapp.enabled=false \ + --set nebariapp.enabled=true \ + --set nebariapp.hostname="pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }}" \ --set jupyterhub.custom.external-url="pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }}" \ --wait --timeout 5m @@ -208,12 +224,13 @@ jobs: # Interactive SSH debug session into the live runner (cluster still # up, KUBECONFIG still valid) instead of guessing blind from static # logs. limit-access-to-actor restricts the SSH session to whoever - # triggered this run — required on a public repo. Bounded to 20min - # so a forgotten session doesn't eat the whole 90min job timeout. - - name: Debug via tmate SSH on deploy failure - if: failure() + # triggered this run — required on a public repo. Temporarily + # unconditional (not gated to failure()) while actively debugging the + # nebariapp/Keycloak auth path live instead of iterating full CI runs. + - name: Debug via tmate SSH + if: always() uses: mxschmitt/action-tmate@35b54afac29c97fb54faba5b513f8fbd1882f113 # v3.24 - timeout-minutes: 20 + timeout-minutes: 45 env: KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} with: From d15efbc835986365ebe2851c104d66139ddfa8d2 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 31 Aug 2026 14:50:24 +0100 Subject: [PATCH 22/54] ci: wait for the operator's Keycloak secret instead of blocking helm --wait Root cause of the nebariapp/Keycloak crash-loop: the hub pod reads the operator-provisioned OIDC client Secret at import time (00-gateway-auth.py), but nic-operator provisions it asynchronously after the NebariApp CRD is created, so a fresh install always starts hub before the Secret exists (FileNotFoundError: /etc/oauth/issuer-url). The Secret volume is mounted optional:true so the pod itself comes up fine -- only the hub process crash-loops -- and helm --wait just timed out waiting on a rollout that could never complete before the operator finished. Drop --wait from the initial install, poll for the operator's Secret directly, then force a hub restart so it doesn't have to wait out CrashLoopBackOff's own backoff schedule. Also revert the tmate step back to failure-gated -- left unconditional for live debugging this session, but it never actually connected in ~7 minutes, so leaving it always-on would stall every future run. --- .github/workflows/k8s-preview.yaml | 52 +++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 1948dd2..f645e8d 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -150,6 +150,15 @@ jobs: helm repo add jupyterhub https://hub.jupyter.org/helm-chart/ helm dependency build . + # No --wait here: with nebariapp.auth.enabled=true, the hub pod reads + # the operator-provisioned Keycloak client Secret at import time + # (config/jupyterhub/00-gateway-auth.py) and crashes with + # FileNotFoundError if it starts before the operator has created it -- + # the Secret volume is mounted `optional: true` so the pod itself + # comes up fine, only the hub container's Python process crash-loops. + # That race is inherent (operator reconciliation runs concurrently + # with the chart install, not before it), so we wait for the Secret + # separately below instead of blocking helm on hub's rollout status. - name: Deploy chart id: deploy env: @@ -161,8 +170,36 @@ jobs: --set jupyterhub.hub.image.tag=preview \ --set nebariapp.enabled=true \ --set nebariapp.hostname="pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }}" \ - --set jupyterhub.custom.external-url="pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }}" \ - --wait --timeout 5m + --set jupyterhub.custom.external-url="pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }}" + + # Secret name convention: {Release.Name}-{Chart.Name}-oidc-client + # (see values.yaml, jupyterhub.hub.extraVolumes comment). Poll by + # label-free name match rather than assuming a fixed provisioning + # time -- the operator's reconcile loop has no SLA. + - name: Wait for operator to provision the Keycloak client secret + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + run: | + for i in $(seq 1 36); do + if kubectl -n pr-preview get secret preview-nebari-data-science-pack-oidc-client >/dev/null 2>&1; then + echo "operator secret present after $(( i * 5 ))s" + exit 0 + fi + sleep 5 + done + echo "::error::operator never created the Keycloak client secret within 3m" + kubectl -n pr-preview get nebariapp -o yaml || true + exit 1 + + # The hub container already crash-looped at least once (reading the + # Secret volume before the operator populated it) and Kubernetes' + # optional-secret volume sync can lag kubelet's resync period, so + # force an immediate restart now that the secret is confirmed present + # instead of waiting out CrashLoopBackOff's own backoff schedule. + - name: Restart hub to pick up the operator secret + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + run: kubectl -n pr-preview rollout restart deployment/hub - name: Wait for hub + proxy env: @@ -224,13 +261,12 @@ jobs: # Interactive SSH debug session into the live runner (cluster still # up, KUBECONFIG still valid) instead of guessing blind from static # logs. limit-access-to-actor restricts the SSH session to whoever - # triggered this run — required on a public repo. Temporarily - # unconditional (not gated to failure()) while actively debugging the - # nebariapp/Keycloak auth path live instead of iterating full CI runs. - - name: Debug via tmate SSH - if: always() + # triggered this run — required on a public repo. Bounded to 20min + # so a forgotten session doesn't eat the whole 90min job timeout. + - name: Debug via tmate SSH on deploy failure + if: failure() uses: mxschmitt/action-tmate@35b54afac29c97fb54faba5b513f8fbd1882f113 # v3.24 - timeout-minutes: 45 + timeout-minutes: 20 env: KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} with: From 2b6f1a928c31782345c77b1bc165f4582ea1a4a8 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 31 Aug 2026 15:07:36 +0100 Subject: [PATCH 23/54] ci: cover the keycloak namespace in the deploy-failure debug dump Deploy chart's second failure was a Helm post-install hook timeout (the RBAC bootstrap Job, which now activates now that nebariapp.enabled is true), not the hub rollout. That Job runs in the keycloak namespace by default, which the debug step never looked at -- extend it to dump pods/jobs/secrets/events there too, plus the NebariApp resource, so the actual failure is visible instead of only pr-preview's already-healthy state. --- .github/workflows/k8s-preview.yaml | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index f645e8d..5d4450a 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -248,15 +248,20 @@ jobs: env: KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} run: | - kubectl -n pr-preview get pods -o wide || true - kubectl -n pr-preview get jobs || true - kubectl -n pr-preview get events --sort-by=.lastTimestamp || true - for pod in $(kubectl -n pr-preview get pods -o name 2>/dev/null); do - echo "--- describe $pod ---" - kubectl -n pr-preview describe "$pod" || true - echo "--- logs $pod ---" - kubectl -n pr-preview logs "$pod" --all-containers --tail=100 || true + for ns in pr-preview keycloak; do + echo "=== namespace: $ns ===" + kubectl -n "$ns" get pods -o wide || true + kubectl -n "$ns" get jobs || true + kubectl -n "$ns" get secrets || true + kubectl -n "$ns" get events --sort-by=.lastTimestamp || true + for pod in $(kubectl -n "$ns" get pods -o name 2>/dev/null); do + echo "--- describe $pod ($ns) ---" + kubectl -n "$ns" describe "$pod" || true + echo "--- logs $pod ($ns) ---" + kubectl -n "$ns" logs "$pod" --all-containers --tail=100 || true + done done + kubectl -n pr-preview get nebariapp -o yaml || true # Interactive SSH debug session into the live runner (cluster still # up, KUBECONFIG still valid) instead of guessing blind from static From 494eee4d076fa4d90f0df9bad7aabf314f41e7ea Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 31 Aug 2026 15:21:29 +0100 Subject: [PATCH 24/54] ci: label the preview namespace so nic-operator actually reconciles it Found the real root cause via kubectl get nebariapp -o yaml on a failed run: nic-operator sets condition NamespaceNotOptedIn and never touches the NebariApp at all when its namespace lacks nebari.dev/managed=true. This wasn't a provisioning-speed race -- the operator's Keycloak client secret was never going to appear no matter how long anything waited, which is why both the RBAC bootstrap hook's internal 5min retry and the previous "wait for operator secret" step were doomed regardless of timeout length. Create + label the namespace before the chart install so the operator picks up the NebariApp immediately. --- .github/workflows/k8s-preview.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 5d4450a..91e85c3 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -150,6 +150,19 @@ jobs: helm repo add jupyterhub https://hub.jupyter.org/helm-chart/ helm dependency build . + # nic-operator only reconciles NebariApps in namespaces opted into + # Nebari management; without this label it sets condition + # NamespaceNotOptedIn and never provisions the Keycloak client at + # all (confirmed via `kubectl get nebariapp -o yaml` after a deploy + # failure -- not a slow-provisioning race, a permanent gate that + # would never resolve on its own). + - name: Create + label the preview namespace for the operator + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + run: | + kubectl create namespace pr-preview --dry-run=client -o yaml | kubectl apply -f - + kubectl label namespace pr-preview nebari.dev/managed=true --overwrite + # No --wait here: with nebariapp.auth.enabled=true, the hub pod reads # the operator-provisioned Keycloak client Secret at import time # (config/jupyterhub/00-gateway-auth.py) and crashes with From be49af83501e4ffbdc7af879ed1328c31e16ea15 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 31 Aug 2026 15:32:19 +0100 Subject: [PATCH 25/54] docs: update k8s-preview comments to match the current auth setup The header comment and PR sticky-comment text still described the earlier state: GitHub-org-restricted Access (actually an email allow-list now) and dummy-authenticator login (nebariapp is enabled by default now, so login goes through the operator-provisioned Keycloak client). No behavior change, just correcting stale documentation. --- .github/workflows/k8s-preview.yaml | 38 +++++++++++++++++------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 91e85c3..6660361 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -3,7 +3,7 @@ name: K8s Stack Preview # Gateway, via nebari-dev/action-nebari-sandbox) plus this PR's chart into # an ephemeral kind cluster on the runner, then exposes # JupyterHub through a per-PR Cloudflare Tunnel behind Cloudflare Access -# (GitHub-org SSO) so a reviewer can click a link and use it. +# (GitHub SSO) so a reviewer can click a link and use it. # # This repo is public, so a plain shared secret posted in the PR comment # (a quick-tunnel URL, a basic-auth password) is readable by anyone who @@ -11,7 +11,10 @@ name: K8s Stack Preview # authenticating the *person*, not a string in the comment: Cloudflare # challenges every request to *. with a GitHub SSO # login and only lets it through to cloudflared if the signed-in account -# is a member of this GitHub org. The PR comment only ever contains a URL. +# matches the Access application's policy (currently an explicit email +# allow-list, not org membership -- see the Cloudflare Access application +# for this domain for who's currently allowed). The PR comment only ever +# contains a URL. # # The domain is a repo Variable (Settings -> Secrets and variables -> # Actions -> Variables -> PREVIEW_DOMAIN, currently openteams.app), not @@ -22,14 +25,18 @@ name: K8s Stack Preview # level ( + *.); a second level needs the paid Advanced # Certificate Manager add-on, which this setup doesn't use. # -# Scope, deliberately: the tunnel points straight at the `proxy-public` -# service (dummy-authenticator login, same as local Tilt dev). The chart -# deploys with nebariapp.enabled=false — nebariapp.auth.enabled=true -# was tried to additionally exercise the operator/Keycloak reconcile, -# but 00-gateway-auth.py reads /etc/oauth/issuer-url unconditionally at -# import time, before the operator's async client provisioning can ever -# populate it, so the hub pod crash-loops. Not worth chasing for a -# preview link; the operator/OIDC path stays untested here. +# Scope: the tunnel points straight at the `proxy-public` service. The +# chart deploys with nebariapp.enabled=true (nebariapp.auth.enabled=true +# by chart default), so login goes through the real operator-provisioned +# Keycloak OIDC client, exercising the same auth path production deploys +# use. Getting this working required: (1) not blocking `helm upgrade +# --install` with `--wait`, since the hub pod crash-loops on +# FileNotFoundError until the operator's async client-provisioning +# Secret exists -- a separate step now polls for that Secret and force- +# restarts hub once it appears; (2) labelling the release namespace +# `nebari.dev/managed=true` before install, since nic-operator otherwise +# never reconciles the NebariApp at all (condition NamespaceNotOptedIn), +# which is a permanent gate, not a slow-provisioning race. # # The link only lives for the run's duration (bounded by timeout-minutes # below) — it is not a persistent per-PR environment. Each run creates its @@ -403,13 +410,10 @@ jobs: ${{ github.event.pull_request.head.repo.fork && '⚠️ **This PR is from a fork** — the code running in this preview is not from a trusted maintainer branch.' || '' }} - You'll be asked to sign in via Cloudflare Access (GitHub SSO) before - reaching JupyterHub — only members of this GitHub org get through. - **Then JupyterHub login:** dummy authenticator, any username + any password. - - This goes straight to JupyterHub's proxy; the operator-provisioned - NebariApp/OIDC route isn't deployed here (see workflow header comment - for why) — this preview doesn't exercise operator/Keycloak reconcile. + You'll be asked to sign in via Cloudflare Access (GitHub SSO) first — + only accounts on the Access application's allow-list get through. + **Then JupyterHub login:** goes through Keycloak (the operator-provisioned + OIDC client for this preview's NebariApp). Live until the expiry time above, or until the `deploy-preview` label is removed. Push a new commit or re-add the label to redeploy. From daba57327288d56d1f95951f9ead41c92561dcbc Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 31 Aug 2026 15:35:50 +0100 Subject: [PATCH 26/54] ci: diagnose why the operator's issuer-url secret key comes back empty Hub still crash-loops on FileNotFoundError even after the operator's Secret exists and hub is restarted -- nebari-operator's own source (GetExternalIssuerURL in providers/keycloak.go) writes issuer-url empty whenever its KEYCLOAK_EXTERNAL_URL env var isn't set, which is templated from the NIC config's top-level domain field (pkg/argocd/templates/manifests/nebari-operator/deployment-patch.yaml in nebari-infrastructure-core). Dump the operator deployment's env, the secret's actual keys, and the NIC config's domain to confirm which of those is empty before guessing at a fix. --- .github/workflows/k8s-preview.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 6660361..ac0a06c 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -283,6 +283,21 @@ jobs: done kubectl -n pr-preview get nebariapp -o yaml || true + echo "=== nebari-operator deployment env ===" + for d in $(kubectl get deploy -A -o json | jq -r '.items[] | select(.metadata.name | test("operator")) | "\(.metadata.namespace)/\(.metadata.name)"'); do + ns="${d%%/*}"; name="${d##*/}" + echo "--- $d ---" + kubectl -n "$ns" get deploy "$name" -o jsonpath='{.spec.template.spec.containers[0].env}' | jq . || true + done + + echo "=== oidc-client secret: keys + issuer-url presence (no values printed) ===" + kubectl -n pr-preview get secret preview-nebari-data-science-pack-oidc-client -o json 2>/dev/null | jq -r '.data | keys' || true + issuer_b64=$(kubectl -n pr-preview get secret preview-nebari-data-science-pack-oidc-client -o jsonpath='{.data.issuer-url}' 2>/dev/null) + echo "issuer-url key present: $([ -n "$issuer_b64" ] && echo yes || echo no); decoded byte length: $(echo -n "$issuer_b64" | base64 -d 2>/dev/null | wc -c)" + + echo "=== NIC config domain ===" + find /tmp -maxdepth 1 -iname "nic-config*.yaml" -exec grep -H "^domain:" {} \; || true + # Interactive SSH debug session into the live runner (cluster still # up, KUBECONFIG still valid) instead of guessing blind from static # logs. limit-access-to-actor restricts the SSH session to whoever From 47e6606895326425742af326d78cc71613e6cf8c Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 31 Aug 2026 15:47:33 +0100 Subject: [PATCH 27/54] ci: wait for issuer-url specifically, not just the secret's existence Confirmed via the operator-env diagnostic: KEYCLOAK_EXTERNAL_URL is correctly set (https://keycloak.nebari.local) and the Secret does end up with a populated issuer-url -- but the operator creates the Secret with client-id/client-secret on an earlier reconcile pass and patches in issuer-url on a later one. The previous wait step only checked the Secret object's existence, so a hub restart triggered between those two passes still hit FileNotFoundError on /etc/oauth/issuer-url specifically. Poll for that key's actual value instead. --- .github/workflows/k8s-preview.yaml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index ac0a06c..3f041f1 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -196,18 +196,25 @@ jobs: # (see values.yaml, jupyterhub.hub.extraVolumes comment). Poll by # label-free name match rather than assuming a fixed provisioning # time -- the operator's reconcile loop has no SLA. + # Checking the Secret merely EXISTS isn't enough: the operator creates + # it with client-id/client-secret on its first reconcile pass, then + # patches in issuer-url on a later pass (GetExternalIssuerURL). A hub + # restart triggered right after the Secret's first appearance still + # hits FileNotFoundError on /etc/oauth/issuer-url specifically -- wait + # for that key's actual (non-empty) value, not just the object. - name: Wait for operator to provision the Keycloak client secret env: KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} run: | for i in $(seq 1 36); do - if kubectl -n pr-preview get secret preview-nebari-data-science-pack-oidc-client >/dev/null 2>&1; then - echo "operator secret present after $(( i * 5 ))s" + issuer_b64=$(kubectl -n pr-preview get secret preview-nebari-data-science-pack-oidc-client -o jsonpath='{.data.issuer-url}' 2>/dev/null) + if [ -n "$issuer_b64" ]; then + echo "operator secret's issuer-url populated after $(( i * 5 ))s" exit 0 fi sleep 5 done - echo "::error::operator never created the Keycloak client secret within 3m" + echo "::error::operator never populated issuer-url on the Keycloak client secret within 3m" kubectl -n pr-preview get nebariapp -o yaml || true exit 1 From 612f07def06f0cfb62c91f9c28dcfe5da5892a57 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 31 Aug 2026 15:59:32 +0100 Subject: [PATCH 28/54] ci: retry the hub restart until it actually picks up the operator secret The API-server-confirmed issuer-url fix (previous commit) didn't hold: a restart issued 5s after issuer-url was confirmed present still hit the exact same FileNotFoundError. That points at kubelet's own Secret volume cache (node-local, ~1min TTL) serving the freshly-restarted pod the pre-population snapshot from its first crash-looped attempt, not an API-level race. Retry the restart (up to 5x, 90s rollout-status timeout each) until it actually succeeds, giving the cache time to expire, rather than assuming a single restart is enough. --- .github/workflows/k8s-preview.yaml | 34 ++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 3f041f1..da2e8ef 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -218,22 +218,34 @@ jobs: kubectl -n pr-preview get nebariapp -o yaml || true exit 1 - # The hub container already crash-looped at least once (reading the - # Secret volume before the operator populated it) and Kubernetes' - # optional-secret volume sync can lag kubelet's resync period, so - # force an immediate restart now that the secret is confirmed present - # instead of waiting out CrashLoopBackOff's own backoff schedule. - - name: Restart hub to pick up the operator secret + # A single restart isn't reliable here even though the API server + # confirms issuer-url is populated: kubelet's own Secret volume cache + # (node-local, ~1min TTL) can still hand a freshly-restarted pod the + # pre-population snapshot it fetched for the pod's first, crash-looped + # attempt -- confirmed by the same FileNotFoundError recurring on a + # restart issued 5s after the secret was already confirmed complete. + # Retry the restart until a rollout actually succeeds, giving the + # kubelet cache time to expire between attempts, instead of assuming + # one restart is enough. + - name: Restart hub until it picks up the operator secret env: KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} - run: kubectl -n pr-preview rollout restart deployment/hub + run: | + for attempt in 1 2 3 4 5; do + kubectl -n pr-preview rollout restart deployment/hub + if kubectl -n pr-preview rollout status deployment/hub --timeout=90s; then + echo "hub ready on attempt $attempt" + exit 0 + fi + echo "hub not ready on attempt $attempt, retrying..." + done + echo "::error::hub never became ready after 5 restart attempts" + exit 1 - - name: Wait for hub + proxy + - name: Wait for proxy env: KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} - run: | - kubectl -n pr-preview rollout status deployment/hub --timeout=180s - kubectl -n pr-preview rollout status deployment/proxy --timeout=180s + run: kubectl -n pr-preview rollout status deployment/proxy --timeout=180s - name: Port-forward JupyterHub proxy env: From b0c12f0ca5d5cadbed92d77779800d64409f30fd Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 31 Aug 2026 16:16:47 +0100 Subject: [PATCH 29/54] ci: fix the actual bug -- hub was mounting the wrong secret name values.yaml's jupyterhub.hub.extraVolumes[1] hardcodes secretName: data-science-pack-nebari-data-science-pack-oidc-client as a literal string, not templated off .Release.Name. This release installs as "preview", so the hub pod was mounting a secret that never existed -- the operator's real, correctly-populated secret (preview-nebari-data-science-pack-oidc-client) was never what the pod looked at. Every prior "fix" in this chain (namespace label, waiting for issuer-url specifically, retrying restarts across kubelet's cache window) was real and necessary for other reasons, but none of them could have worked while the pod was pointed at a nonexistent secret. Override the secretName explicitly to match this release's actual name. --- .github/workflows/k8s-preview.yaml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index da2e8ef..c5fb862 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -179,6 +179,16 @@ jobs: # That race is inherent (operator reconciliation runs concurrently # with the chart install, not before it), so we wait for the Secret # separately below instead of blocking helm on hub's rollout status. + # + # jupyterhub.hub.extraVolumes[1] (values.yaml) hardcodes + # secretName: data-science-pack-nebari-data-science-pack-oidc-client + # -- a literal string baked in for a release named "data-science-pack", + # not templated off .Release.Name. Every restart/retry against the + # real (correctly-populated) operator secret still 404'd on the mount + # because the pod was mounting a DIFFERENT, nonexistent secret name + # the whole time (release name here is "preview") -- confirmed only + # after ruling out timing races and kubelet caching across 5 retries. + # Override it explicitly to match this release's actual name. - name: Deploy chart id: deploy env: @@ -190,7 +200,8 @@ jobs: --set jupyterhub.hub.image.tag=preview \ --set nebariapp.enabled=true \ --set nebariapp.hostname="pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }}" \ - --set jupyterhub.custom.external-url="pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }}" + --set jupyterhub.custom.external-url="pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }}" \ + --set jupyterhub.hub.extraVolumes[1].secret.secretName=preview-nebari-data-science-pack-oidc-client # Secret name convention: {Release.Name}-{Chart.Name}-oidc-client # (see values.yaml, jupyterhub.hub.extraVolumes comment). Poll by From 0d041ce44203522e35e688b037f5d10c7bb2bc44 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 31 Aug 2026 16:25:15 +0100 Subject: [PATCH 30/54] ci: use --set-json for extraVolumes, not indexed --set The prior fix's --set jupyterhub.hub.extraVolumes[1].secret.secretName=... corrupted the rendered Deployment: Helm's --set doesn't deep-merge an indexed path into an existing values.yaml-defined list, so the other elements lost their name field entirely ("volumes[2].name: Required value, volumes[3].name: Required value", volumeMounts referencing names that no longer existed). Verified locally with `helm template` before pushing this time: --set-json with the full four-element array (only secretName changed) renders a correct Deployment. --- .github/workflows/k8s-preview.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index c5fb862..5130708 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -188,7 +188,13 @@ jobs: # because the pod was mounting a DIFFERENT, nonexistent secret name # the whole time (release name here is "preview") -- confirmed only # after ruling out timing races and kubelet caching across 5 retries. - # Override it explicitly to match this release's actual name. + # Override it explicitly to match this release's actual name. Using + # --set with list-index syntax (extraVolumes[1].secret.secretName=...) + # against a values.yaml-defined list corrupts the OTHER elements' + # `name` fields too (Helm's --set doesn't deep-merge per-element into + # an existing default list) -- confirmed by a real failure: + # "volumes[2].name: Required value, volumes[3].name: Required value". + # --set-json with the full, correct array sidesteps that. - name: Deploy chart id: deploy env: @@ -201,7 +207,7 @@ jobs: --set nebariapp.enabled=true \ --set nebariapp.hostname="pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }}" \ --set jupyterhub.custom.external-url="pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }}" \ - --set jupyterhub.hub.extraVolumes[1].secret.secretName=preview-nebari-data-science-pack-oidc-client + --set-json 'jupyterhub.hub.extraVolumes=[{"name":"custom-config","configMap":{"name":"nebari-data-science-pack-hub-config"}},{"name":"oauth-client","secret":{"secretName":"preview-nebari-data-science-pack-oidc-client","optional":true}},{"name":"org-ca","configMap":{"name":"nebari-trust-bundle","optional":true}},{"name":"ca-merged","emptyDir":{}}]' # Secret name convention: {Release.Name}-{Chart.Name}-oidc-client # (see values.yaml, jupyterhub.hub.extraVolumes comment). Poll by From 0b2e76e1c3449c51faca6c229e878e8f98ce3535 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 31 Aug 2026 16:53:31 +0100 Subject: [PATCH 31/54] ci: expose Keycloak's own login page through a second tunnel route Adds a second Cloudflare Tunnel ingress rule + DNS record for keycloak-pr--data-science-pack., port-forwarding svc/keycloak-keycloakx-http alongside the existing proxy-public forward. Keycloak's issuer/hostname stay pinned to NIC's internal keycloak.nebari.local (that's what the operator baked into hub's OIDC client secret; changing it would break the hub login flow just fixed), so this new hostname rewrites the Host header back to keycloak.nebari.local at the tunnel via originRequest.httpHostHeader instead of reconfiguring Keycloak. A reviewer can sign in at the Keycloak URL directly and land on JupyterHub already authenticated via the shared SSO session. --- .github/workflows/k8s-preview.yaml | 49 ++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 5130708..9fad217 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -273,6 +273,23 @@ jobs: echo "PORT_FORWARD_PID=$!" >> "$GITHUB_ENV" sleep 3 + # Keycloak itself, so a reviewer can sign in there directly and land + # on JupyterHub already authenticated (same Keycloak SSO cookie the + # hub's own OAuth redirect relies on). Keycloak's issuer/hostname + # stay pinned to NIC's internal keycloak.nebari.local (that's what + # the operator baked into hub's OIDC client secret -- changing it + # would break the hub login flow) so this is exposed under a + # DIFFERENT public hostname with the Host header rewritten back to + # keycloak.nebari.local at the tunnel, not by reconfiguring Keycloak. + - name: Port-forward Keycloak + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + run: | + kubectl -n keycloak port-forward svc/keycloak-keycloakx-http 8001:8080 \ + > /tmp/port-forward-keycloak.log 2>&1 & + echo "KEYCLOAK_PORT_FORWARD_PID=$!" >> "$GITHUB_ENV" + sleep 3 + # jhub-apps runs as a managed service subprocess inside the hub pod, # not its own Deployment, so a crash there doesn't fail `helm --wait` # or the rollout checks above -- it only shows up as a 502 on @@ -366,6 +383,7 @@ jobs: CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} CF_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_TUNNEL_ACCOUNT_ID }} PREVIEW_HOSTNAME: pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }} + KEYCLOAK_HOSTNAME: keycloak-pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }} run: | tunnel_secret=$(openssl rand -base64 32) echo "::add-mask::${tunnel_secret}" @@ -408,8 +426,12 @@ jobs: "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel/${tunnel_id}/configurations" \ -H "Authorization: Bearer ${CF_API_TOKEN}" \ -H "Content-Type: application/json" \ - -d "$(jq -n --arg host "$PREVIEW_HOSTNAME" \ - '{config: {ingress: [{hostname: $host, service: "http://localhost:8000"}, {service: "http_status:404"}]}}')" \ + -d "$(jq -n --arg host "$PREVIEW_HOSTNAME" --arg kchost "$KEYCLOAK_HOSTNAME" \ + '{config: {ingress: [ + {hostname: $host, service: "http://localhost:8000"}, + {hostname: $kchost, service: "http://localhost:8001", originRequest: {httpHostHeader: "keycloak.nebari.local"}}, + {service: "http_status:404"} + ]}}')" \ > /dev/null - name: Point DNS at the tunnel @@ -417,6 +439,7 @@ jobs: env: CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} PREVIEW_HOSTNAME: pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }} + KEYCLOAK_HOSTNAME: keycloak-pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }} run: | zone_id=$(curl -fsS "https://api.cloudflare.com/client/v4/zones?name=${PREVIEW_DOMAIN}" \ -H "Authorization: Bearer ${CF_API_TOKEN}" | jq -r '.result[0].id') @@ -439,6 +462,19 @@ jobs: echo "DNS_RECORD_ID=${record_id}" >> "$GITHUB_ENV" echo "url=https://${PREVIEW_HOSTNAME}" >> "$GITHUB_OUTPUT" + kc_record_resp=$(curl -fsS -X POST "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "$(jq -n --arg host "$KEYCLOAK_HOSTNAME" --arg target "${TUNNEL_ID}.cfargotunnel.com" \ + '{type: "CNAME", name: $host, content: $target, proxied: true}')") + kc_record_id=$(jq -r '.result.id' <<< "$kc_record_resp") + if [ -z "$kc_record_id" ] || [ "$kc_record_id" = "null" ]; then + echo "::error::Keycloak DNS record creation failed: $kc_record_resp" + exit 1 + fi + echo "KEYCLOAK_DNS_RECORD_ID=${kc_record_id}" >> "$GITHUB_ENV" + echo "keycloak_url=https://${KEYCLOAK_HOSTNAME}" >> "$GITHUB_OUTPUT" + # The URL itself (pr--data-science-pack.) is identical on every run, so # without a timestamp the sticky comment would post byte-identical # text each redeploy and look like it never updated. @@ -466,6 +502,11 @@ jobs: **Then JupyterHub login:** goes through Keycloak (the operator-provisioned OIDC client for this preview's NebariApp). + Keycloak's own login page is also reachable directly: + ${{ steps.cf_dns.outputs.keycloak_url }} + Signing in there first establishes the SSO session, so visiting + JupyterHub afterward skips straight past the login prompt. + Live until the expiry time above, or until the `deploy-preview` label is removed. Push a new commit or re-add the label to redeploy. @@ -490,6 +531,10 @@ jobs: curl -fsS -X DELETE \ "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records/${DNS_RECORD_ID}" \ -H "Authorization: Bearer ${CF_API_TOKEN}" || true + [ -n "${KEYCLOAK_DNS_RECORD_ID:-}" ] || exit 0 + curl -fsS -X DELETE \ + "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records/${KEYCLOAK_DNS_RECORD_ID}" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" || true - name: Delete Cloudflare Tunnel if: always() From f925800a016d664b36216c1b465075ef02afedd9 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 31 Aug 2026 17:22:32 +0100 Subject: [PATCH 32/54] ci: point Keycloak's own hostname at the public route instead of rewriting headers The Host-header-rewrite trick only fixed the initial GET: Keycloak's own KC_HOSTNAME (fixed to NIC's internal keycloak.nebari.local) still bakes that hostname into every self-referencing URL it renders -- confirmed live, the login form's own action attribute pointed at keycloak.nebari.local regardless of the incoming Host header, so credentials could never actually be submitted through any public route. This also means the ORIGINAL hub-redirect flow was never actually completable by a real browser either, not just the new direct-Keycloak route. Fix: patch Keycloak's KC_HOSTNAME and the operator's KEYCLOAK_EXTERNAL_URL to both point at the same public, per-PR, single-label hostname the tunnel already routes. No DNS/cert changes (stays within the free-tier single wildcard level), no Cloudflare Worker, no cost, no collision risk between concurrent PR previews -- every URL Keycloak emits (form actions, issuer, redirects) becomes directly reachable, and the operator writes a matching issuer into hub's OIDC client secret so the whole chain stays consistent. Drops the now-unneeded httpHostHeader rewrite on the tunnel ingress rule. --- .github/workflows/k8s-preview.yaml | 44 +++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 9fad217..1347350 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -170,6 +170,38 @@ jobs: kubectl create namespace pr-preview --dry-run=client -o yaml | kubectl apply -f - kubectl label namespace pr-preview nebari.dev/managed=true --overwrite + # Keycloak's own KC_HOSTNAME (bitnami/keycloakx) is fixed to NIC's + # internal keycloak.nebari.local by default -- every self-referencing + # URL Keycloak renders (login form action, issuer, redirects) is + # absolute and uses that value regardless of the incoming Host + # header, confirmed live: a request proxied to Keycloak with the Host + # header rewritten to keycloak.nebari.local still rendered a login + # form whose action="https://keycloak.nebari.local/..." -- unreachable + # from a real browser, since .local is never publicly resolvable. + # + # Point Keycloak's own hostname AND the operator's KEYCLOAK_EXTERNAL_URL + # at the same public, per-PR, single-label hostname the Cloudflare + # Tunnel already routes (no DNS/cert changes, no Worker, no collision + # risk) so Keycloak's self-generated URLs are actually reachable and + # the operator writes a matching issuer into hub's OIDC client secret. + - name: Point Keycloak's own hostname at the public tunnel route + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + run: | + echo "--- current keycloak-keycloakx env (KC_HOSTNAME*) ---" + kubectl -n keycloak get statefulset keycloak-keycloakx \ + -o jsonpath='{.spec.template.spec.containers[0].env}' | jq '[.[] | select(.name | test("HOSTNAME"))]' + + kc_public_url="https://keycloak-pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }}" + kubectl -n keycloak set env statefulset/keycloak-keycloakx \ + KC_HOSTNAME="$kc_public_url" KC_HOSTNAME_STRICT=true + kubectl -n keycloak rollout status statefulset/keycloak-keycloakx --timeout=180s + + operator_deploy=$(kubectl get deploy -A -o json | jq -r '.items[] | select(.metadata.name | test("operator")) | "\(.metadata.namespace)/\(.metadata.name)"' | head -1) + kubectl -n "${operator_deploy%%/*}" set env deployment/"${operator_deploy##*/}" \ + KEYCLOAK_EXTERNAL_URL="$kc_public_url" + kubectl -n "${operator_deploy%%/*}" rollout status deployment/"${operator_deploy##*/}" --timeout=120s + # No --wait here: with nebariapp.auth.enabled=true, the hub pod reads # the operator-provisioned Keycloak client Secret at import time # (config/jupyterhub/00-gateway-auth.py) and crashes with @@ -275,12 +307,10 @@ jobs: # Keycloak itself, so a reviewer can sign in there directly and land # on JupyterHub already authenticated (same Keycloak SSO cookie the - # hub's own OAuth redirect relies on). Keycloak's issuer/hostname - # stay pinned to NIC's internal keycloak.nebari.local (that's what - # the operator baked into hub's OIDC client secret -- changing it - # would break the hub login flow) so this is exposed under a - # DIFFERENT public hostname with the Host header rewritten back to - # keycloak.nebari.local at the tunnel, not by reconfiguring Keycloak. + # hub's own OAuth redirect relies on). Keycloak's own hostname was + # repointed at this same public route earlier (see "Point Keycloak's + # own hostname..."), so no Host-header rewrite is needed here -- the + # request just flows straight through with its real Host header. - name: Port-forward Keycloak env: KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} @@ -429,7 +459,7 @@ jobs: -d "$(jq -n --arg host "$PREVIEW_HOSTNAME" --arg kchost "$KEYCLOAK_HOSTNAME" \ '{config: {ingress: [ {hostname: $host, service: "http://localhost:8000"}, - {hostname: $kchost, service: "http://localhost:8001", originRequest: {httpHostHeader: "keycloak.nebari.local"}}, + {hostname: $kchost, service: "http://localhost:8001"}, {service: "http_status:404"} ]}}')" \ > /dev/null From c85f9b4f853fcb3368e210693b0038e17f0eff8d Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 31 Aug 2026 17:38:36 +0100 Subject: [PATCH 33/54] ci: fix Keycloak hostname via GitOps instead of a direct kubectl patch Both Keycloak and nebari-operator are ArgoCD Applications with selfHeal: true, reconciled against NIC's auto-created local GitOps repo. A direct kubectl set env DID apply (confirmed live: both workloads genuinely rolled to new pods), but ArgoCD's reconcile loop silently reverted it within a couple minutes once it noticed the drift from the GitOps repo -- hub's redirect still landed on the unreachable keycloak.nebari.local well after the "successful" patch. Edit the two source files in the GitOps repo directly (values/keycloak/base.yaml, manifests/nebari-operator/deployment-patch.yaml), commit, and force an ArgoCD hard refresh so selfHeal applies our change instead of fighting it -- the same pattern already used for this project's other ArgoCD-managed clusters. Poll for the live env var to actually change before trusting rollout status, since a stale refresh can make rollout status return instantly without ArgoCD having applied anything yet. --- .github/workflows/k8s-preview.yaml | 79 +++++++++++++++++++++--------- 1 file changed, 57 insertions(+), 22 deletions(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 1347350..90b1add 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -170,37 +170,72 @@ jobs: kubectl create namespace pr-preview --dry-run=client -o yaml | kubectl apply -f - kubectl label namespace pr-preview nebari.dev/managed=true --overwrite - # Keycloak's own KC_HOSTNAME (bitnami/keycloakx) is fixed to NIC's - # internal keycloak.nebari.local by default -- every self-referencing - # URL Keycloak renders (login form action, issuer, redirects) is + # Keycloak's own KC_HOSTNAME (codecentric/keycloakx, via NIC's + # values/keycloak/base.yaml) is fixed to NIC's internal + # keycloak.nebari.local by default -- every self-referencing URL + # Keycloak renders (login form action, issuer, redirects) is # absolute and uses that value regardless of the incoming Host - # header, confirmed live: a request proxied to Keycloak with the Host - # header rewritten to keycloak.nebari.local still rendered a login - # form whose action="https://keycloak.nebari.local/..." -- unreachable - # from a real browser, since .local is never publicly resolvable. + # header, confirmed live: a request reaching Keycloak still rendered + # a login form whose action="https://keycloak.nebari.local/..." -- + # unreachable from a real browser, since .local is never publicly + # resolvable. # - # Point Keycloak's own hostname AND the operator's KEYCLOAK_EXTERNAL_URL - # at the same public, per-PR, single-label hostname the Cloudflare - # Tunnel already routes (no DNS/cert changes, no Worker, no collision - # risk) so Keycloak's self-generated URLs are actually reachable and - # the operator writes a matching issuer into hub's OIDC client secret. + # A direct `kubectl set env` patch onto the live StatefulSet/ + # Deployment DOES apply immediately but gets silently reverted: + # both Keycloak and nebari-operator are ArgoCD Applications with + # selfHeal: true (pkg/argocd/templates/apps/{keycloak,nebari- + # operator}.yaml in nebari-infrastructure-core), continuously + # reconciled against NIC's auto-created local GitOps repo + # (~/.nic/gitops/, repository.local: {} in NIC's + # config) -- confirmed live: the patch step's own logs showed the + # StatefulSet/Deployment genuinely rolling to new pods, yet hub's + # redirect still landed on keycloak.nebari.local a couple minutes + # later once ArgoCD's reconcile loop caught the drift and reverted + # it. Editing the GitOps repo itself and forcing a hard refresh + # (the same pattern used for this project's other ArgoCD-managed + # clusters) lets selfHeal work for this change instead of against it. - name: Point Keycloak's own hostname at the public tunnel route env: KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} run: | - echo "--- current keycloak-keycloakx env (KC_HOSTNAME*) ---" - kubectl -n keycloak get statefulset keycloak-keycloakx \ - -o jsonpath='{.spec.template.spec.containers[0].env}' | jq '[.[] | select(.name | test("HOSTNAME"))]' - kc_public_url="https://keycloak-pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }}" - kubectl -n keycloak set env statefulset/keycloak-keycloakx \ - KC_HOSTNAME="$kc_public_url" KC_HOSTNAME_STRICT=true - kubectl -n keycloak rollout status statefulset/keycloak-keycloakx --timeout=180s + gitops_dir="$HOME/.nic/gitops/${{ steps.sandbox.outputs.cluster-name }}" + + echo "--- current rendered values (pre-patch) ---" + grep -n "keycloak.nebari.local" "$gitops_dir/values/keycloak/base.yaml" "$gitops_dir/manifests/nebari-operator/deployment-patch.yaml" + + sed -i "s#https://keycloak\.nebari\.local#${kc_public_url}#g" \ + "$gitops_dir/values/keycloak/base.yaml" \ + "$gitops_dir/manifests/nebari-operator/deployment-patch.yaml" + + git -C "$gitops_dir" -c user.email="ci@example.com" -c user.name="k8s-preview-ci" \ + commit -am "Point Keycloak hostname at the public preview tunnel route" + + kubectl -n argocd annotate application/keycloak application/nebari-operator \ + argocd.argoproj.io/refresh=hard --overwrite operator_deploy=$(kubectl get deploy -A -o json | jq -r '.items[] | select(.metadata.name | test("operator")) | "\(.metadata.namespace)/\(.metadata.name)"' | head -1) - kubectl -n "${operator_deploy%%/*}" set env deployment/"${operator_deploy##*/}" \ - KEYCLOAK_EXTERNAL_URL="$kc_public_url" - kubectl -n "${operator_deploy%%/*}" rollout status deployment/"${operator_deploy##*/}" --timeout=120s + + # `rollout status` alone can return instantly if ArgoCD hasn't + # actually applied the refreshed spec yet (nothing new requested + # from kubectl's point of view) -- poll for the live env var to + # actually change before trusting rollout status to mean anything. + for i in $(seq 1 24); do + kc_live=$(kubectl -n keycloak get statefulset keycloak-keycloakx -o jsonpath='{.spec.template.spec.containers[0].env}' | jq -r '.[] | select(.name=="KC_HOSTNAME") | .value') + op_live=$(kubectl -n "${operator_deploy%%/*}" get deploy "${operator_deploy##*/}" -o jsonpath='{.spec.template.spec.containers[0].env}' | jq -r '.[] | select(.name=="KEYCLOAK_EXTERNAL_URL") | .value') + if [ "$kc_live" = "$kc_public_url" ] && [ "$op_live" = "$kc_public_url" ]; then + echo "ArgoCD applied the GitOps change after $(( i * 5 ))s" + break + fi + sleep 5 + done + if [ "$kc_live" != "$kc_public_url" ] || [ "$op_live" != "$kc_public_url" ]; then + echo "::error::ArgoCD never applied the GitOps hostname change within 2m (keycloak=$kc_live, operator=$op_live)" + exit 1 + fi + + kubectl -n keycloak rollout status statefulset/keycloak-keycloakx --timeout=180s + kubectl -n "${operator_deploy%%/*}" rollout status deployment/"${operator_deploy##*/}" --timeout=180s # No --wait here: with nebariapp.auth.enabled=true, the hub pod reads # the operator-provisioned Keycloak client Secret at import time From 78b008634ed15b98e8c7e27675e63946e1610de8 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 31 Aug 2026 18:00:56 +0100 Subject: [PATCH 34/54] ci: create a simple reviewer/admin test user in Keycloak Cloudflare Access is the real security boundary for this preview -- only allow-listed accounts reach the tunnel at all -- so a simple, known Keycloak password is fine here rather than making reviewers hunt for real credentials on a throwaway cluster. Creates the user via Keycloak's admin API, piping the admin secret straight into the token request without ever echoing or logging it. PR comment now states the credentials directly. --- .github/workflows/k8s-preview.yaml | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 90b1add..f2ee236 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -355,6 +355,29 @@ jobs: echo "KEYCLOAK_PORT_FORWARD_PID=$!" >> "$GITHUB_ENV" sleep 3 + # Cloudflare Access is the real security boundary for this preview + # (only allow-listed accounts reach the tunnel at all) -- a simple, + # known password for the Keycloak-side login is fine here, so + # reviewers don't need to hunt for real credentials on a throwaway + # cluster. The admin password is piped straight from the Secret into + # the token request and never echoed or logged. + - name: Create a test login user in Keycloak + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + run: | + admin_token=$(curl -sS -X POST "http://localhost:8001/realms/master/protocol/openid-connect/token" \ + -d "grant_type=password" -d "client_id=admin-cli" -d "username=admin" \ + --data-urlencode "password=$(kubectl -n keycloak get secret keycloak-admin-credentials -o jsonpath='{.data.admin-password}' | base64 -d)" \ + | jq -r '.access_token') + if [ -z "$admin_token" ] || [ "$admin_token" = "null" ]; then + echo "::error::Failed to obtain a Keycloak admin token" + exit 1 + fi + + curl -sS -o /dev/null -w 'create user: HTTP %{http_code}\n' -X POST "http://localhost:8001/admin/realms/nebari/users" \ + -H "Authorization: Bearer $admin_token" -H "Content-Type: application/json" \ + -d '{"username":"reviewer","enabled":true,"emailVerified":true,"credentials":[{"type":"password","value":"admin","temporary":false}]}' + # jhub-apps runs as a managed service subprocess inside the hub pod, # not its own Deployment, so a crash there doesn't fail `helm --wait` # or the rollout checks above -- it only shows up as a 502 on @@ -565,7 +588,9 @@ jobs: You'll be asked to sign in via Cloudflare Access (GitHub SSO) first — only accounts on the Access application's allow-list get through. **Then JupyterHub login:** goes through Keycloak (the operator-provisioned - OIDC client for this preview's NebariApp). + OIDC client for this preview's NebariApp). **Username:** `reviewer` + **Password:** `admin` — Cloudflare Access is the real gate here, so + this account is intentionally simple. Keycloak's own login page is also reachable directly: ${{ steps.cf_dns.outputs.keycloak_url }} From 6c96912416627f057f15e74de130a6eab79a6b17 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 31 Aug 2026 18:14:24 +0100 Subject: [PATCH 35/54] ci: route hub's Keycloak backchannel calls in-cluster, not through Access Confirmed live: after a real login, hub's OAuth callback hit json.decoder.JSONDecodeError: Expecting value (an empty/non-JSON body) on the token exchange. Keycloak's public hostname sits behind Cloudflare Access same as the JupyterHub hostname -- fine for the browser-facing authorize/login legs since the reviewer already has an Access session, but hub's own server-to-server token/userinfo calls have no such session and get blocked by Access instead of a JSON response. Set keycloak.backchannelURL to Keycloak's in-cluster Service so only token_url/userdata_url move there -- authorize_url and end_session_url stay on the public issuer, which is what the browser actually needs to reach. This is exactly the split-horizon path config/jupyterhub/00-gateway-auth.py already supported; just needed the value set. --- .github/workflows/k8s-preview.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index f2ee236..a782846 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -262,6 +262,18 @@ jobs: # an existing default list) -- confirmed by a real failure: # "volumes[2].name: Required value, volumes[3].name: Required value". # --set-json with the full, correct array sidesteps that. + # + # keycloak.backchannelURL: Keycloak's public hostname sits behind + # Cloudflare Access (same as the JupyterHub hostname), which is fine + # for the browser-facing authorize/login legs -- the reviewer already + # has an Access session -- but hub's OWN server-to-server token + # exchange has no such session and gets blocked/redirected by + # Access instead of getting a JSON response back, confirmed live: + # json.decoder.JSONDecodeError: Expecting value: line 1 column 1 + # (an empty/non-JSON body from the token endpoint). Point the + # backchannel (token_url, userdata_url only -- authorize_url stays + # on the public issuer) straight at Keycloak's in-cluster Service, + # bypassing Cloudflare/Access entirely for that leg. - name: Deploy chart id: deploy env: @@ -274,6 +286,7 @@ jobs: --set nebariapp.enabled=true \ --set nebariapp.hostname="pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }}" \ --set jupyterhub.custom.external-url="pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }}" \ + --set keycloak.backchannelURL="http://keycloak-keycloakx-http.keycloak.svc.cluster.local:8080" \ --set-json 'jupyterhub.hub.extraVolumes=[{"name":"custom-config","configMap":{"name":"nebari-data-science-pack-hub-config"}},{"name":"oauth-client","secret":{"secretName":"preview-nebari-data-science-pack-oidc-client","optional":true}},{"name":"org-ca","configMap":{"name":"nebari-trust-bundle","optional":true}},{"name":"ca-merged","emptyDir":{}}]' # Secret name convention: {Release.Name}-{Chart.Name}-oidc-client From d5c27c7eb64758a27295621d46ef0da4edbdf16f Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 31 Aug 2026 20:45:30 +0100 Subject: [PATCH 36/54] fix: point jhub-apps' hub-API client at localhost, not the hub Service jhub-apps runs as a managed-service subprocess inside the same pod as hub, but z2jh injects JUPYTERHUB_API_URL pointing at the `hub` Service (a ClusterIP self-reference). Routing same-pod traffic through a Service depends on the CNI supporting hairpin NAT for a pod reaching its own Service -- confirmed via a live preview to time out (httpcore.ConnectTimeout) on a kind/kindnet cluster, breaking /services/japps/oauth_callback with a 500 right after a successful Keycloak login. Rewrite the host to localhost (same port/path) before forwarding it into the japps service's subprocess environment. This is a general fix, not preview-specific: same-pod loopback is always reliable regardless of hairpin NAT support on whatever CNI a real deployment happens to run. --- config/jupyterhub/02-jhub-apps.py | 29 ++++++++ tests/unit/test_jhub_apps_backend_url.py | 85 ++++++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 tests/unit/test_jhub_apps_backend_url.py diff --git a/config/jupyterhub/02-jhub-apps.py b/config/jupyterhub/02-jhub-apps.py index e559204..adb27bd 100644 --- a/config/jupyterhub/02-jhub-apps.py +++ b/config/jupyterhub/02-jhub-apps.py @@ -2,12 +2,32 @@ # ruff: noqa: F821 - `c` is a magic global provided by JupyterHub import os +from urllib.parse import urlsplit, urlunsplit from jhub_apps import theme_template_paths, themes from jhub_apps.configuration import install_jhub_apps from kubespawner import KubeSpawner from z2jh import get_config + +def _localhost_hub_api_url(url: str) -> str: + """Rewrite a hub API URL's host to localhost, keeping port and path. + + jhub-apps runs as a managed-service subprocess inside the SAME pod as + hub, but z2jh's JUPYTERHUB_API_URL points at the `hub` Service + (ClusterIP self-reference). Routing same-pod traffic through a Service + depends on the CNI supporting hairpin NAT for a pod reaching its own + Service -- observed to time out (httpcore.ConnectTimeout) on a + kind/kindnet cluster. localhost is always reliable for same-pod + traffic regardless of hairpin NAT support. + """ + if not url: + return url + parsed = urlsplit(url) + netloc = f"localhost:{parsed.port}" if parsed.port else "localhost" + return urlunsplit(parsed._replace(netloc=netloc)) + + # Configure jhub-apps # bind_url must include the real external hostname so JupyterHub constructs # correct OAuth redirect URLs for internal services like jhub-apps. @@ -104,3 +124,12 @@ if svc.get("name") == "japps": svc.setdefault("environment", {})["JUPYTERHUB_OIDC_CLIENT_SECRET"] = _oidc_secret break + +# Point jhub-apps' own hub-API client at localhost instead of the `hub` +# Service it inherits from z2jh -- see _localhost_hub_api_url's docstring. +_hub_api_url = _localhost_hub_api_url(os.environ.get("JUPYTERHUB_API_URL", "")) +if _hub_api_url: + for svc in c.JupyterHub.services: + if svc.get("name") == "japps": + svc.setdefault("environment", {})["JUPYTERHUB_API_URL"] = _hub_api_url + break diff --git a/tests/unit/test_jhub_apps_backend_url.py b/tests/unit/test_jhub_apps_backend_url.py new file mode 100644 index 0000000..d08219b --- /dev/null +++ b/tests/unit/test_jhub_apps_backend_url.py @@ -0,0 +1,85 @@ +"""jhub-apps' own hub-API base URL wiring in 02-jhub-apps.py. + +jhub-apps runs as a managed-service subprocess inside the SAME pod as +hub, but z2jh injects JUPYTERHUB_API_URL pointing at the `hub` Service +(ClusterIP self-reference). Routing same-pod traffic through a Service +depends on the CNI supporting hairpin NAT for a pod reaching its own +Service via that Service's ClusterIP -- confirmed to time out +(httpcore.ConnectTimeout) on a kind/kindnet cluster. Rewriting the host +to localhost (same port/path) is always reliable for same-pod traffic +and doesn't depend on hairpin NAT support. + +02-jhub-apps.py isn't independently importable: it needs `jhub_apps` +and `z2jh` (real chart-image dependencies, not installed in the unit +test venv) at module load time for things unrelated to this fix. Stub +both minimally so the real file executes end-to-end and this test +exercises the actual production code path, not a reimplementation of +it elsewhere. +""" + +from __future__ import annotations + +import sys +import types + +from conftest import FakeConfig, load_config_module + + +def _install_stub_dependencies(monkeypatch): + """Stub jhub_apps + z2jh just enough for 02-jhub-apps.py to load.""" + jhub_apps_mod = types.ModuleType("jhub_apps") + jhub_apps_mod.theme_template_paths = [] + jhub_apps_mod.themes = types.SimpleNamespace(DEFAULT_THEME={}) + + def _fake_install_jhub_apps(c, spawner_to_subclass=None): + c.JupyterHub.services = [ + {"name": "japps", "oauth_client_id": "service-japps"} + ] + c.JupyterHub.load_roles = [{"name": "user", "scopes": []}] + return c + + configuration_mod = types.ModuleType("jhub_apps.configuration") + configuration_mod.install_jhub_apps = _fake_install_jhub_apps + + def _fake_get_config(key, default=None): + return default + + z2jh_mod = types.ModuleType("z2jh") + z2jh_mod.get_config = _fake_get_config + + monkeypatch.setitem(sys.modules, "jhub_apps", jhub_apps_mod) + monkeypatch.setitem(sys.modules, "jhub_apps.configuration", configuration_mod) + monkeypatch.setitem(sys.modules, "z2jh", z2jh_mod) + + +def _load(monkeypatch, **env): + """Set env vars, then load 02-jhub-apps.py (which reads them at exec time).""" + _install_stub_dependencies(monkeypatch) + for key, value in env.items(): + monkeypatch.setenv(key, value) + c = FakeConfig() + mod = load_config_module("02-jhub-apps.py", inject_c=c) + return c, mod + + +def _japps_service(c): + return next(svc for svc in c.JupyterHub.services if svc.get("name") == "japps") + + +def test_japps_service_env_points_hub_api_at_localhost(monkeypatch): + c, _ = _load(monkeypatch, JUPYTERHUB_API_URL="http://hub:8081/hub/api") + japps = _japps_service(c) + assert japps["environment"]["JUPYTERHUB_API_URL"] == "http://localhost:8081/hub/api" + + +def test_preserves_a_nonstandard_port(monkeypatch): + c, _ = _load(monkeypatch, JUPYTERHUB_API_URL="http://hub:9999/hub/api") + japps = _japps_service(c) + assert japps["environment"]["JUPYTERHUB_API_URL"] == "http://localhost:9999/hub/api" + + +def test_no_jupyterhub_api_url_set_leaves_japps_env_untouched(monkeypatch): + monkeypatch.delenv("JUPYTERHUB_API_URL", raising=False) + c, _ = _load(monkeypatch) + japps = _japps_service(c) + assert "JUPYTERHUB_API_URL" not in japps.get("environment", {}) From fdec180fd8cf1ac11788bd746cfb9cfe717121a7 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 31 Aug 2026 21:02:42 +0100 Subject: [PATCH 37/54] fix: the env-dict override for JUPYTERHUB_API_URL doesn't survive spawn Confirmed live: setting svc["environment"]["JUPYTERHUB_API_URL"] (same pattern as the existing OIDC secret forwarding) had no effect -- the subprocess still saw the `hub` Service URL and still timed out. JupyterHub's own Spawner.get_env() computes env['JUPYTERHUB_API_URL'] = hub_api_url from self.hub.api_url *after* merging self.environment, unconditionally overwriting whatever a service config sets. Only something applied after JupyterHub finishes building the subprocess's env can actually win, so wrap the service's own command with a shell-level `env VAR=value` assignment instead -- that's scoped to jhub-apps' own process and can't be overridden by the parent. --- config/jupyterhub/02-jhub-apps.py | 20 +++++++++++-- tests/unit/test_jhub_apps_backend_url.py | 38 ++++++++++++++++++++---- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/config/jupyterhub/02-jhub-apps.py b/config/jupyterhub/02-jhub-apps.py index adb27bd..d22b0ab 100644 --- a/config/jupyterhub/02-jhub-apps.py +++ b/config/jupyterhub/02-jhub-apps.py @@ -2,6 +2,7 @@ # ruff: noqa: F821 - `c` is a magic global provided by JupyterHub import os +import shlex from urllib.parse import urlsplit, urlunsplit from jhub_apps import theme_template_paths, themes @@ -127,9 +128,24 @@ def _localhost_hub_api_url(url: str) -> str: # Point jhub-apps' own hub-API client at localhost instead of the `hub` # Service it inherits from z2jh -- see _localhost_hub_api_url's docstring. +# +# Setting it via svc["environment"] (like the OIDC secret above) does NOT +# work here: JupyterHub's Spawner.get_env() computes +# env['JUPYTERHUB_API_URL'] = hub_api_url from self.hub.api_url AFTER +# merging self.environment, unconditionally overwriting whatever we set +# -- confirmed live (the resulting subprocess still saw the `hub` Service +# URL). Wrapping the service's own command with a shell-level `env +# VAR=value` assignment is the only thing that can win: it sets the +# variable for jhub-apps' uvicorn process specifically, after JupyterHub +# has already finished building the parent env. _hub_api_url = _localhost_hub_api_url(os.environ.get("JUPYTERHUB_API_URL", "")) if _hub_api_url: for svc in c.JupyterHub.services: - if svc.get("name") == "japps": - svc.setdefault("environment", {})["JUPYTERHUB_API_URL"] = _hub_api_url + if svc.get("name") == "japps" and svc.get("command"): + quoted_cmd = " ".join(shlex.quote(part) for part in svc["command"]) + svc["command"] = [ + "sh", + "-c", + f"exec env JUPYTERHUB_API_URL={shlex.quote(_hub_api_url)} {quoted_cmd}", + ] break diff --git a/tests/unit/test_jhub_apps_backend_url.py b/tests/unit/test_jhub_apps_backend_url.py index d08219b..97e3736 100644 --- a/tests/unit/test_jhub_apps_backend_url.py +++ b/tests/unit/test_jhub_apps_backend_url.py @@ -9,6 +9,14 @@ to localhost (same port/path) is always reliable for same-pod traffic and doesn't depend on hairpin NAT support. +Setting svc["environment"] does NOT work here (confirmed live): +JupyterHub's Spawner.get_env() computes +env['JUPYTERHUB_API_URL'] = hub_api_url from self.hub.api_url AFTER +merging self.environment, unconditionally overwriting it. Only a +shell-level `env VAR=value` wrapped around the service's own command +can win, since that's applied after JupyterHub has already built the +parent env. + 02-jhub-apps.py isn't independently importable: it needs `jhub_apps` and `z2jh` (real chart-image dependencies, not installed in the unit test venv) at module load time for things unrelated to this fix. Stub @@ -24,6 +32,16 @@ from conftest import FakeConfig, load_config_module +ORIGINAL_COMMAND = [ + "python", + "-m", + "uvicorn", + "jhub_apps.service.app:app", + "--port=10202", + "--host=0.0.0.0", + "--workers=1", +] + def _install_stub_dependencies(monkeypatch): """Stub jhub_apps + z2jh just enough for 02-jhub-apps.py to load.""" @@ -33,7 +51,11 @@ def _install_stub_dependencies(monkeypatch): def _fake_install_jhub_apps(c, spawner_to_subclass=None): c.JupyterHub.services = [ - {"name": "japps", "oauth_client_id": "service-japps"} + { + "name": "japps", + "oauth_client_id": "service-japps", + "command": list(ORIGINAL_COMMAND), + } ] c.JupyterHub.load_roles = [{"name": "user", "scopes": []}] return c @@ -66,20 +88,24 @@ def _japps_service(c): return next(svc for svc in c.JupyterHub.services if svc.get("name") == "japps") -def test_japps_service_env_points_hub_api_at_localhost(monkeypatch): +def test_japps_command_wrapped_with_localhost_hub_api_url(monkeypatch): c, _ = _load(monkeypatch, JUPYTERHUB_API_URL="http://hub:8081/hub/api") japps = _japps_service(c) - assert japps["environment"]["JUPYTERHUB_API_URL"] == "http://localhost:8081/hub/api" + assert japps["command"][:2] == ["sh", "-c"] + shell_line = japps["command"][2] + assert "JUPYTERHUB_API_URL=http://localhost:8081/hub/api" in shell_line + # The original argv is still there, just appended after the env assignment. + assert "uvicorn jhub_apps.service.app:app" in shell_line def test_preserves_a_nonstandard_port(monkeypatch): c, _ = _load(monkeypatch, JUPYTERHUB_API_URL="http://hub:9999/hub/api") japps = _japps_service(c) - assert japps["environment"]["JUPYTERHUB_API_URL"] == "http://localhost:9999/hub/api" + assert "JUPYTERHUB_API_URL=http://localhost:9999/hub/api" in japps["command"][2] -def test_no_jupyterhub_api_url_set_leaves_japps_env_untouched(monkeypatch): +def test_no_jupyterhub_api_url_set_leaves_japps_command_untouched(monkeypatch): monkeypatch.delenv("JUPYTERHUB_API_URL", raising=False) c, _ = _load(monkeypatch) japps = _japps_service(c) - assert "JUPYTERHUB_API_URL" not in japps.get("environment", {}) + assert japps["command"] == ORIGINAL_COMMAND From 3ca96efde6f42aa2156b1c7b0890a2003336c48e Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 31 Aug 2026 21:15:13 +0100 Subject: [PATCH 38/54] ci: pre-fill email/name on the reviewer test user Keycloak's default realm profile requires email/firstName/lastName, so creating the user without them triggered a first-login "Update Account Information" prompt every time. Set them at creation instead so the reviewer account logs straight through. --- .github/workflows/k8s-preview.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index a782846..254b38b 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -389,7 +389,7 @@ jobs: curl -sS -o /dev/null -w 'create user: HTTP %{http_code}\n' -X POST "http://localhost:8001/admin/realms/nebari/users" \ -H "Authorization: Bearer $admin_token" -H "Content-Type: application/json" \ - -d '{"username":"reviewer","enabled":true,"emailVerified":true,"credentials":[{"type":"password","value":"admin","temporary":false}]}' + -d '{"username":"reviewer","enabled":true,"email":"reviewer@example.com","emailVerified":true,"firstName":"Preview","lastName":"Reviewer","credentials":[{"type":"password","value":"admin","temporary":false}]}' # jhub-apps runs as a managed service subprocess inside the hub pod, # not its own Deployment, so a crash there doesn't fail `helm --wait` From 928adec161a51adc8f1d4511c46ea44210308718 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 31 Aug 2026 21:38:46 +0100 Subject: [PATCH 39/54] fix: the JUPYTERHUB_API_URL rewrite for jhub-apps never actually ran The previous fix read os.environ.get("JUPYTERHUB_API_URL") in the hub process's own config-load-time environment to precompute a localhost URL to inject. That variable is empty there -- JupyterHub only computes and injects it into a service's own environment at spawn time -- so the rewrite was silently a no-op and jhub-apps kept timing out against the `hub` Service. Wrap the service's command with a shell snippet that rewrites $JUPYTERHUB_API_URL to localhost (via sed, keeping port/path) at the moment the subprocess execs, using whatever JupyterHub has actually put in its environment by then. --- config/jupyterhub/02-jhub-apps.py | 74 ++++++++++++------------ tests/unit/test_jhub_apps_backend_url.py | 70 ++++++++++++++-------- 2 files changed, 83 insertions(+), 61 deletions(-) diff --git a/config/jupyterhub/02-jhub-apps.py b/config/jupyterhub/02-jhub-apps.py index d22b0ab..f42f485 100644 --- a/config/jupyterhub/02-jhub-apps.py +++ b/config/jupyterhub/02-jhub-apps.py @@ -3,30 +3,35 @@ # ruff: noqa: F821 - `c` is a magic global provided by JupyterHub import os import shlex -from urllib.parse import urlsplit, urlunsplit from jhub_apps import theme_template_paths, themes from jhub_apps.configuration import install_jhub_apps from kubespawner import KubeSpawner from z2jh import get_config - -def _localhost_hub_api_url(url: str) -> str: - """Rewrite a hub API URL's host to localhost, keeping port and path. - - jhub-apps runs as a managed-service subprocess inside the SAME pod as - hub, but z2jh's JUPYTERHUB_API_URL points at the `hub` Service - (ClusterIP self-reference). Routing same-pod traffic through a Service - depends on the CNI supporting hairpin NAT for a pod reaching its own - Service -- observed to time out (httpcore.ConnectTimeout) on a - kind/kindnet cluster. localhost is always reliable for same-pod - traffic regardless of hairpin NAT support. - """ - if not url: - return url - parsed = urlsplit(url) - netloc = f"localhost:{parsed.port}" if parsed.port else "localhost" - return urlunsplit(parsed._replace(netloc=netloc)) +# jhub-apps runs as a managed-service subprocess inside the SAME pod as +# hub, but z2jh's hub_connect_ip="hub" (needed so KubeSpawner pods on other +# nodes can reach the hub) makes JupyterHub inject JUPYTERHUB_API_URL +# pointing at the `hub` Service (ClusterIP self-reference) for every +# service, same-pod ones included. Routing same-pod traffic through a +# Service depends on the CNI supporting hairpin NAT for a pod reaching its +# own Service -- confirmed to time out (httpcore.ConnectTimeout) on a +# kind/kindnet cluster. +# +# This can't be fixed by precomputing the target URL here in Python: +# os.environ has no JUPYTERHUB_API_URL at config-load time in the hub +# container's own process -- JupyterHub only computes and injects that +# value into a service's environment at spawn time (confirmed live: an +# earlier version of this fix read os.environ.get("JUPYTERHUB_API_URL") +# here and it was always empty, so the rewrite never applied). Instead, +# wrap the service's own command with a shell snippet that rewrites +# $JUPYTERHUB_API_URL to localhost (keeping port/path) at the one point +# where the real value exists: the moment the subprocess itself execs, +# using whatever JupyterHub just put in its environment. +_REWRITE_HUB_API_URL_TO_LOCALHOST = ( + 'export JUPYTERHUB_API_URL="$(printf %s "$JUPYTERHUB_API_URL" | ' + "sed -E 's#^(https?://)[^/:]+#\\1localhost#')\"" +) # Configure jhub-apps @@ -127,25 +132,18 @@ def _localhost_hub_api_url(url: str) -> str: break # Point jhub-apps' own hub-API client at localhost instead of the `hub` -# Service it inherits from z2jh -- see _localhost_hub_api_url's docstring. -# -# Setting it via svc["environment"] (like the OIDC secret above) does NOT -# work here: JupyterHub's Spawner.get_env() computes +# Service it inherits from z2jh -- see _REWRITE_HUB_API_URL_TO_LOCALHOST's +# comment above. Setting svc["environment"] (like the OIDC secret above) +# does NOT work here: JupyterHub's Spawner.get_env() computes # env['JUPYTERHUB_API_URL'] = hub_api_url from self.hub.api_url AFTER # merging self.environment, unconditionally overwriting whatever we set -# -- confirmed live (the resulting subprocess still saw the `hub` Service -# URL). Wrapping the service's own command with a shell-level `env -# VAR=value` assignment is the only thing that can win: it sets the -# variable for jhub-apps' uvicorn process specifically, after JupyterHub -# has already finished building the parent env. -_hub_api_url = _localhost_hub_api_url(os.environ.get("JUPYTERHUB_API_URL", "")) -if _hub_api_url: - for svc in c.JupyterHub.services: - if svc.get("name") == "japps" and svc.get("command"): - quoted_cmd = " ".join(shlex.quote(part) for part in svc["command"]) - svc["command"] = [ - "sh", - "-c", - f"exec env JUPYTERHUB_API_URL={shlex.quote(_hub_api_url)} {quoted_cmd}", - ] - break +# there -- confirmed live. +for svc in c.JupyterHub.services: + if svc.get("name") == "japps" and svc.get("command"): + quoted_cmd = " ".join(shlex.quote(part) for part in svc["command"]) + svc["command"] = [ + "sh", + "-c", + f"{_REWRITE_HUB_API_URL_TO_LOCALHOST}; exec {quoted_cmd}", + ] + break diff --git a/tests/unit/test_jhub_apps_backend_url.py b/tests/unit/test_jhub_apps_backend_url.py index 97e3736..25802dd 100644 --- a/tests/unit/test_jhub_apps_backend_url.py +++ b/tests/unit/test_jhub_apps_backend_url.py @@ -9,13 +9,15 @@ to localhost (same port/path) is always reliable for same-pod traffic and doesn't depend on hairpin NAT support. -Setting svc["environment"] does NOT work here (confirmed live): -JupyterHub's Spawner.get_env() computes -env['JUPYTERHUB_API_URL'] = hub_api_url from self.hub.api_url AFTER -merging self.environment, unconditionally overwriting it. Only a -shell-level `env VAR=value` wrapped around the service's own command -can win, since that's applied after JupyterHub has already built the -parent env. +The rewrite can't be precomputed in Python at config-load time: the hub +container's own os.environ has no JUPYTERHUB_API_URL there (JupyterHub +only computes and injects that value into a service's own environment +at spawn time) -- confirmed live, an earlier version of this fix that +read os.environ.get("JUPYTERHUB_API_URL") here always got "", so the +rewrite never applied and jhub-apps kept timing out against the `hub` +Service. So the command is wrapped with a shell snippet that rewrites +$JUPYTERHUB_API_URL to localhost at the moment the subprocess execs, +using whatever JupyterHub has actually put in its environment by then. 02-jhub-apps.py isn't independently importable: it needs `jhub_apps` and `z2jh` (real chart-image dependencies, not installed in the unit @@ -27,6 +29,7 @@ from __future__ import annotations +import subprocess import sys import types @@ -74,11 +77,8 @@ def _fake_get_config(key, default=None): monkeypatch.setitem(sys.modules, "z2jh", z2jh_mod) -def _load(monkeypatch, **env): - """Set env vars, then load 02-jhub-apps.py (which reads them at exec time).""" +def _load(monkeypatch): _install_stub_dependencies(monkeypatch) - for key, value in env.items(): - monkeypatch.setenv(key, value) c = FakeConfig() mod = load_config_module("02-jhub-apps.py", inject_c=c) return c, mod @@ -88,24 +88,48 @@ def _japps_service(c): return next(svc for svc in c.JupyterHub.services if svc.get("name") == "japps") -def test_japps_command_wrapped_with_localhost_hub_api_url(monkeypatch): - c, _ = _load(monkeypatch, JUPYTERHUB_API_URL="http://hub:8081/hub/api") +def _run_wrapped_command(command, jupyterhub_api_url): + """Actually run the wrapped shell command's rewrite in a real shell. + + The rewrite lives in a shell snippet, not Python, so the only honest + test is exec'ing it in a real /bin/sh with a fake JUPYTERHUB_API_URL + and observing the rewritten value -- a unit test of the Python string + construction alone couldn't catch a shell syntax error or a wrong sed + pattern. Splits the rewrite off the trailing `exec ` + (which would actually try to launch uvicorn) and prints the result + instead of exec'ing it. + """ + assert command[:2] == ["sh", "-c"] + rewrite = command[2].split("; exec ", 1)[0] + result = subprocess.run( + ["sh", "-c", f'{rewrite}; printf %s "$JUPYTERHUB_API_URL"'], + env={"JUPYTERHUB_API_URL": jupyterhub_api_url, "PATH": "/usr/bin:/bin"}, + capture_output=True, + text=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stderr + return result.stdout + + +def test_japps_command_is_wrapped_in_a_shell_rewrite(monkeypatch): + c, _ = _load(monkeypatch) japps = _japps_service(c) assert japps["command"][:2] == ["sh", "-c"] - shell_line = japps["command"][2] - assert "JUPYTERHUB_API_URL=http://localhost:8081/hub/api" in shell_line - # The original argv is still there, just appended after the env assignment. - assert "uvicorn jhub_apps.service.app:app" in shell_line + # The original argv is still there, just appended after the rewrite. + assert "exec python -m uvicorn jhub_apps.service.app:app" in japps["command"][2] -def test_preserves_a_nonstandard_port(monkeypatch): - c, _ = _load(monkeypatch, JUPYTERHUB_API_URL="http://hub:9999/hub/api") +def test_rewrite_replaces_host_with_localhost_keeping_port_and_path(monkeypatch): + c, _ = _load(monkeypatch) japps = _japps_service(c) - assert "JUPYTERHUB_API_URL=http://localhost:9999/hub/api" in japps["command"][2] + out = _run_wrapped_command(japps["command"], "http://hub:8081/hub/api") + assert out == "http://localhost:8081/hub/api" -def test_no_jupyterhub_api_url_set_leaves_japps_command_untouched(monkeypatch): - monkeypatch.delenv("JUPYTERHUB_API_URL", raising=False) +def test_rewrite_handles_a_url_with_no_explicit_port(monkeypatch): c, _ = _load(monkeypatch) japps = _japps_service(c) - assert japps["command"] == ORIGINAL_COMMAND + out = _run_wrapped_command(japps["command"], "http://hub/hub/api") + assert out == "http://localhost/hub/api" From ba3fd4540bc7ee27c2cb70b965a6b70df302329f Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Tue, 1 Sep 2026 11:26:24 +0100 Subject: [PATCH 40/54] ci: bound the tunnel step to 20min instead of the job timeout The tunnel step ran unbounded and relied on the job's timeout-minutes (90) to kill it, which makes GitHub mark the whole run "cancelled" with a "job has exceeded the maximum execution time" failure annotation -- looks like a real failure even though the preview is meant to expire. Bound the step itself to 20 minutes via `timeout`, treating its own deadline (exit 124) as success, so the job finishes normally well under the outer limit. Updates the reviewer-facing expiry timestamp and comments to match. --- .github/workflows/k8s-preview.yaml | 38 ++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 254b38b..31ea909 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -38,10 +38,10 @@ name: K8s Stack Preview # never reconciles the NebariApp at all (condition NamespaceNotOptedIn), # which is a permanent gate, not a slow-provisioning race. # -# The link only lives for the run's duration (bounded by timeout-minutes -# below) — it is not a persistent per-PR environment. Each run creates its -# own Cloudflare Tunnel + DNS record (so concurrent previews on different -# PRs don't collide on the same route) and deletes both on cleanup. +# The link only lives for 20 minutes (the tunnel step's own timeout) — it +# is not a persistent per-PR environment. Each run creates its own +# Cloudflare Tunnel + DNS record (so concurrent previews on different PRs +# don't collide on the same route) and deletes both on cleanup. # # One-time setup this workflow assumes already exists (Cloudflare Zero # Trust dashboard, done by a repo admin, not scripted here). The tunnel, @@ -79,7 +79,7 @@ name: K8s Stack Preview # does not enforce NetworkPolicy, so a container escape or outbound abuse # from inside a spawned notebook pod is not blocked at the network layer. # Per-job GITHUB_TOKEN permissions are scoped to the minimum each job -# needs so a compromised runner in the exposed 90-minute window can't use +# needs so a compromised runner in the exposed 20-minute window can't use # an ambient token to touch other workflows or repo state. on: @@ -583,7 +583,7 @@ jobs: id: timestamps run: | echo "deployed_at=$(date -u +'%Y-%m-%d %H:%M UTC')" >> "$GITHUB_OUTPUT" - echo "expires_at=$(date -u -d '+90 minutes' +'%Y-%m-%d %H:%M UTC')" >> "$GITHUB_OUTPUT" + echo "expires_at=$(date -u -d '+20 minutes' +'%Y-%m-%d %H:%M UTC')" >> "$GITHUB_OUTPUT" - name: Comment preview link on PR uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 @@ -613,12 +613,26 @@ jobs: Live until the expiry time above, or until the `deploy-preview` label is removed. Push a new commit or re-add the label to redeploy. - - name: Run tunnel until the job times out - run: /tmp/cloudflared tunnel --no-autoupdate run --token "${TUNNEL_TOKEN}" - - # Runs once the tunnel step above ends (timeout or a manual cancel), - # so this captures anything logged in response to real traffic during - # the tunnel's lifetime -- unlike the earlier startup-time log dump. + # Bounded to 20min so the live preview doesn't sit open (and burn CI + # minutes) indefinitely, and so this step ends itself well under the + # job's 90min timeout-minutes -- letting the job's own timeout kill + # this step instead makes GitHub mark the whole run "cancelled" with + # a "job has exceeded the maximum execution time" failure annotation, + # which looks like a real failure even though nothing broke -- the + # preview is *meant* to expire. `timeout`'s exit 124 (its own + # deadline hit) is the expected outcome and treated as success; any + # other nonzero exit (e.g. cloudflared itself crashing) still fails + # the step for real. + - name: Run tunnel until it times out + run: | + timeout 1200 /tmp/cloudflared tunnel --no-autoupdate run --token "${TUNNEL_TOKEN}" + ec=$? + [ "$ec" -eq 0 ] || [ "$ec" -eq 124 ] || exit "$ec" + + # Runs once the tunnel step above ends (its own timeout, cloudflared + # exiting, or a manual cancel), so this captures anything logged in + # response to real traffic during the tunnel's lifetime -- unlike the + # earlier startup-time log dump. - name: Dump hub logs after tunnel closes if: always() env: From d2845435da3c28fab12c63a2ed2dad021290cbd6 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Tue, 1 Sep 2026 11:51:53 +0100 Subject: [PATCH 41/54] ci: add a GitHub Deployment for the preview, not just a comment The sticky PR comment edits in place (it was created on the PR's first deploy), so it doesn't move in the thread and looked like it never updated even though its content was current. Add a GitHub Deployment/Environment (pr--preview) alongside it: GitHub renders that as its own status box pinned near the top of the PR with the live URL and timestamp, independent of the comment thread. Marked inactive once the tunnel closes or the deploy-preview label is removed, so it doesn't keep showing a dead link as green. --- .github/workflows/k8s-preview.yaml | 60 ++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 31ea909..8d2f2a7 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -108,6 +108,7 @@ jobs: permissions: contents: read pull-requests: write + deployments: write steps: - name: Checkout uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 @@ -585,6 +586,43 @@ jobs: echo "deployed_at=$(date -u +'%Y-%m-%d %H:%M UTC')" >> "$GITHUB_OUTPUT" echo "expires_at=$(date -u -d '+20 minutes' +'%Y-%m-%d %H:%M UTC')" >> "$GITHUB_OUTPUT" + # A GitHub Deployment/Environment, not just the comment below: GitHub + # renders this as its own "View deployment" box pinned near the top + # of the PR (like Vercel's bot), so the latest link + status is + # visible without scrolling into the comment thread -- unlike the + # sticky comment, which is an in-place edit of a comment created on + # the PR's first deploy, so it doesn't move and is easy to miss. + # `required_contexts: []` skips gating this on other checks for the + # same commit (e.g. lint/test workflows still running) -- this is a + # preview link, not a merge gate. auto_inactive (default true on the + # status call) marks any prior deployment to this same per-PR + # environment inactive, so redeploys don't leave stale green boxes. + - name: Create GitHub deployment + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + deployment_json=$(jq -n \ + --arg ref "${{ github.event.pull_request.head.sha }}" \ + --arg environment "pr-${{ github.event.pull_request.number }}-preview" \ + '{ref: $ref, environment: $environment, task: "deploy:preview", + auto_merge: false, transient_environment: true, + production_environment: false, required_contexts: [], + description: "K8s stack preview"}') + deployment_id=$(gh api "repos/${{ github.repository }}/deployments" \ + -X POST --input - --jq '.id' <<< "$deployment_json") + if [ -z "$deployment_id" ] || [ "$deployment_id" = "null" ]; then + echo "::error::Failed to create GitHub deployment" + exit 1 + fi + echo "DEPLOYMENT_ID=${deployment_id}" >> "$GITHUB_ENV" + + gh api "repos/${{ github.repository }}/deployments/${deployment_id}/statuses" \ + -X POST \ + -f state=success \ + -f environment_url="${{ steps.cf_dns.outputs.url }}" \ + -f log_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ + -f description="Live for 20 minutes" > /dev/null + - name: Comment preview link on PR uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 with: @@ -639,6 +677,18 @@ jobs: KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} run: kubectl -n pr-preview logs deployment/hub --tail=1000 || true + # The URL in the deployment box above is dead once the tunnel closes + # -- mark it inactive so the box reflects that instead of still + # showing green with a link that no longer resolves. + - name: Mark GitHub deployment inactive + if: always() + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + [ -n "${DEPLOYMENT_ID:-}" ] || exit 0 + gh api "repos/${{ github.repository }}/deployments/${DEPLOYMENT_ID}/statuses" \ + -X POST -f state=inactive -f description="Preview expired" > /dev/null || true + - name: Delete DNS record if: always() env: @@ -671,6 +721,7 @@ jobs: permissions: pull-requests: write actions: write + deployments: write steps: - name: Cancel the in-flight preview run for this PR env: @@ -683,6 +734,15 @@ jobs: gh run cancel "$run_id" --repo "${{ github.repository }}" fi + - name: Mark GitHub deployment inactive + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + deployment_id=$(gh api "repos/${{ github.repository }}/deployments?environment=pr-${{ github.event.pull_request.number }}-preview&per_page=1" --jq '.[0].id') + [ -n "$deployment_id" ] && [ "$deployment_id" != "null" ] || exit 0 + gh api "repos/${{ github.repository }}/deployments/${deployment_id}/statuses" \ + -X POST -f state=inactive -f description="Preview stopped (label removed)" > /dev/null || true + - name: Comment that the preview stopped uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 with: From 044a28ea96dac558f12d4ab2cb5ccf1437b5bbdc Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Tue, 1 Sep 2026 12:22:39 +0100 Subject: [PATCH 42/54] ci: fix the tunnel step's exit-124 handling under bash -e GitHub Actions runs run: steps as bash -e. A plain `timeout ...; ec=$?` doesn't work there -- -e aborts the step the instant timeout returns 124 (its own deadline hit), before ec=$? is ever reached, so the step showed as a real failure even on the expected timeout path (confirmed live on run 33499572113). Capture the exit code via `|| ec=$?` instead, which is a protected context -e doesn't abort on. --- .github/workflows/k8s-preview.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 8d2f2a7..50257f9 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -661,10 +661,16 @@ jobs: # deadline hit) is the expected outcome and treated as success; any # other nonzero exit (e.g. cloudflared itself crashing) still fails # the step for real. + # GitHub Actions runs `run:` steps as `bash -e`: a plain + # `cmd; ec=$?` doesn't work here because -e aborts the step the + # instant `timeout` returns 124, before `ec=$?` is ever reached. + # `|| ec=$?` catches the nonzero exit inside a protected context + # instead (the compound command's own status is that of the + # assignment, which always succeeds). - name: Run tunnel until it times out run: | - timeout 1200 /tmp/cloudflared tunnel --no-autoupdate run --token "${TUNNEL_TOKEN}" - ec=$? + ec=0 + timeout 1200 /tmp/cloudflared tunnel --no-autoupdate run --token "${TUNNEL_TOKEN}" || ec=$? [ "$ec" -eq 0 ] || [ "$ec" -eq 124 ] || exit "$ec" # Runs once the tunnel step above ends (its own timeout, cloudflared @@ -738,7 +744,7 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - deployment_id=$(gh api "repos/${{ github.repository }}/deployments?environment=pr-${{ github.event.pull_request.number }}-preview&per_page=1" --jq '.[0].id') + deployment_id=$(gh api "repos/${{ github.repository }}/deployments?environment=pr-${{ github.event.pull_request.number }}-preview&per_page=1" --jq '.[0].id' 2>/dev/null || true) [ -n "$deployment_id" ] && [ "$deployment_id" != "null" ] || exit 0 gh api "repos/${{ github.repository }}/deployments/${deployment_id}/statuses" \ -X POST -f state=inactive -f description="Preview stopped (label removed)" > /dev/null || true From 7371cc1c02c663c1ec0d25f00064d8e56a2cb6df Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Tue, 1 Sep 2026 12:47:51 +0100 Subject: [PATCH 43/54] ci: replace the preview comment's prose with a Vercel-style table Drops the login walkthrough (obvious to anyone opening the link) and lays out project/status/links/timestamp as a compact table instead, so the comment reads at a glance like a Vercel-for-GitHub bot comment. --- .github/workflows/k8s-preview.yaml | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 50257f9..017bdc5 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -629,27 +629,15 @@ jobs: header: k8s-preview GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} message: | - **K8s stack preview** for `${{ github.event.pull_request.head.ref }}`: - ${{ steps.cf_dns.outputs.url }} + The latest K8s stack preview for this PR. - Deployed: ${{ steps.timestamps.outputs.deployed_at }} · Expires: ${{ steps.timestamps.outputs.expires_at }} + | Project | Deployment | Actions | Updated (UTC) | + | --- | --- | --- | --- | + | `${{ github.event.pull_request.head.ref }}` | 🟢 [Ready](${{ steps.cf_dns.outputs.url }}) | [Preview](${{ steps.cf_dns.outputs.url }}) · [Keycloak](${{ steps.cf_dns.outputs.keycloak_url }}) | ${{ steps.timestamps.outputs.deployed_at }} | ${{ github.event.pull_request.head.repo.fork && '⚠️ **This PR is from a fork** — the code running in this preview is not from a trusted maintainer branch.' || '' }} - You'll be asked to sign in via Cloudflare Access (GitHub SSO) first — - only accounts on the Access application's allow-list get through. - **Then JupyterHub login:** goes through Keycloak (the operator-provisioned - OIDC client for this preview's NebariApp). **Username:** `reviewer` - **Password:** `admin` — Cloudflare Access is the real gate here, so - this account is intentionally simple. - - Keycloak's own login page is also reachable directly: - ${{ steps.cf_dns.outputs.keycloak_url }} - Signing in there first establishes the SSO session, so visiting - JupyterHub afterward skips straight past the login prompt. - - Live until the expiry time above, or until the `deploy-preview` label - is removed. Push a new commit or re-add the label to redeploy. + Expires ${{ steps.timestamps.outputs.expires_at }} — push a new commit or re-add the `deploy-preview` label to redeploy. # Bounded to 20min so the live preview doesn't sit open (and burn CI # minutes) indefinitely, and so this step ends itself well under the From f2896bcddeab5c4db90c02209f6acfde99dd8de4 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Tue, 1 Sep 2026 13:00:46 +0100 Subject: [PATCH 44/54] ci: fix the deployment table's Project column Was showing the PR's branch ref, not a project name -- doesn't match the Vercel-style table it's mimicking, where Project identifies the thing being deployed. Use the chart's actual name instead. --- .github/workflows/k8s-preview.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 017bdc5..8c1a917 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -633,7 +633,7 @@ jobs: | Project | Deployment | Actions | Updated (UTC) | | --- | --- | --- | --- | - | `${{ github.event.pull_request.head.ref }}` | 🟢 [Ready](${{ steps.cf_dns.outputs.url }}) | [Preview](${{ steps.cf_dns.outputs.url }}) · [Keycloak](${{ steps.cf_dns.outputs.keycloak_url }}) | ${{ steps.timestamps.outputs.deployed_at }} | + | `nebari-data-science-pack` | 🟢 [Ready](${{ steps.cf_dns.outputs.url }}) | [Preview](${{ steps.cf_dns.outputs.url }}) · [Keycloak](${{ steps.cf_dns.outputs.keycloak_url }}) | ${{ steps.timestamps.outputs.deployed_at }} | ${{ github.event.pull_request.head.repo.fork && '⚠️ **This PR is from a fork** — the code running in this preview is not from a trusted maintainer branch.' || '' }} From 7b1df6a752c9adf622525ebc8e25501fe7805708 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Tue, 1 Sep 2026 13:11:55 +0100 Subject: [PATCH 45/54] ci: show a live-updating expiry time instead of a static UTC string Reading "Expires: 2026-09-01 12:37 UTC" means doing timezone math by hand to know if the preview is about to die. Use GitHub's own web component (confirmed via gh api /markdown to survive comment sanitization) so it reads "in 17 minutes" and keeps ticking client-side, no re-editing the comment required. Falls back to the plain UTC string as its text content before the element hydrates. --- .github/workflows/k8s-preview.yaml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 8c1a917..eb7b5ca 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -580,11 +580,21 @@ jobs: # The URL itself (pr--data-science-pack.) is identical on every run, so # without a timestamp the sticky comment would post byte-identical # text each redeploy and look like it never updated. + # ISO timestamps feed in the comment below -- GitHub's + # own web component (used all over its UI for "3 minutes ago") that + # ticks live client-side once rendered, confirmed via `gh api /markdown` + # to survive comment sanitization unstripped. Beats a static UTC + # string the reader has to convert to their own timezone by hand, and + # unlike re-editing the comment every minute, needs no extra CI runs. + # The plain UTC strings are the fallback text shown before JS + # hydrates . - name: Compute deployment timestamps id: timestamps run: | echo "deployed_at=$(date -u +'%Y-%m-%d %H:%M UTC')" >> "$GITHUB_OUTPUT" echo "expires_at=$(date -u -d '+20 minutes' +'%Y-%m-%d %H:%M UTC')" >> "$GITHUB_OUTPUT" + echo "deployed_at_iso=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> "$GITHUB_OUTPUT" + echo "expires_at_iso=$(date -u -d '+20 minutes' +'%Y-%m-%dT%H:%M:%SZ')" >> "$GITHUB_OUTPUT" # A GitHub Deployment/Environment, not just the comment below: GitHub # renders this as its own "View deployment" box pinned near the top @@ -631,13 +641,13 @@ jobs: message: | The latest K8s stack preview for this PR. - | Project | Deployment | Actions | Updated (UTC) | + | Project | Deployment | Actions | Updated | | --- | --- | --- | --- | - | `nebari-data-science-pack` | 🟢 [Ready](${{ steps.cf_dns.outputs.url }}) | [Preview](${{ steps.cf_dns.outputs.url }}) · [Keycloak](${{ steps.cf_dns.outputs.keycloak_url }}) | ${{ steps.timestamps.outputs.deployed_at }} | + | `nebari-data-science-pack` | 🟢 [Ready](${{ steps.cf_dns.outputs.url }}) | [Preview](${{ steps.cf_dns.outputs.url }}) · [Keycloak](${{ steps.cf_dns.outputs.keycloak_url }}) | ${{ steps.timestamps.outputs.deployed_at }} | ${{ github.event.pull_request.head.repo.fork && '⚠️ **This PR is from a fork** — the code running in this preview is not from a trusted maintainer branch.' || '' }} - Expires ${{ steps.timestamps.outputs.expires_at }} — push a new commit or re-add the `deploy-preview` label to redeploy. + Expires ${{ steps.timestamps.outputs.expires_at }} — push a new commit or re-add the `deploy-preview` label to redeploy. # Bounded to 20min so the live preview doesn't sit open (and burn CI # minutes) indefinitely, and so this step ends itself well under the From 26bbfe108074377761bd2b750ae578cc0888cb7d Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Tue, 1 Sep 2026 14:24:25 +0100 Subject: [PATCH 46/54] ci: update the preview comment again when it expires The comment was only ever written once, at deploy time. kept the clock ticking but the surrounding wording and green status dot never changed, so an expired preview still read "Ready" next to "Expires 39 minutes ago". Edit the same comment again during teardown with the correct wording and an inactive status marker. Also strips stray em dashes from workflow comments and the PR-facing text. --- .github/workflows/k8s-preview.yaml | 43 +++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index eb7b5ca..81f94fc 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -38,7 +38,7 @@ name: K8s Stack Preview # never reconciles the NebariApp at all (condition NamespaceNotOptedIn), # which is a permanent gate, not a slow-provisioning race. # -# The link only lives for 20 minutes (the tunnel step's own timeout) — it +# The link only lives for 20 minutes (the tunnel step's own timeout); it # is not a persistent per-PR environment. Each run creates its own # Cloudflare Tunnel + DNS record (so concurrent previews on different PRs # don't collide on the same route) and deletes both on cleanup. @@ -46,7 +46,7 @@ name: K8s Stack Preview # One-time setup this workflow assumes already exists (Cloudflare Zero # Trust dashboard, done by a repo admin, not scripted here). The tunnel, # the PREVIEW_DOMAIN zone, and Zero Trust/Access all live in ONE -# Cloudflare account (OpenTeams Account) — not the account behind +# Cloudflare account (OpenTeams Account), not the account behind # CLOUDFLARE_API_TOKEN / CLOUDFLARE_ACCOUNT_ID, which docs.yml uses for # Pages: # - Variable PREVIEW_DOMAIN (repo Settings -> Actions -> Variables): @@ -54,7 +54,7 @@ name: K8s Stack Preview # - That zone in the Cloudflare account, with an Access self-hosted # application for `*.`, GitHub as identity provider, # policy scoped to this org (nebari-dev). Note this wildcard covers -# ANY single-label subdomain of the zone, not just previews — fine +# ANY single-label subdomain of the zone, not just previews, fine # as long as the zone isn't also hosting unrelated services outside # this org's control. # - Secret CLOUDFLARE_TUNNEL_ACCOUNT_ID: that account's id (the @@ -69,7 +69,7 @@ name: K8s Stack Preview # Account Resources: Include -> Specific account -> OpenTeams Account. # # Only runs when a maintainer/collaborator adds the `deploy-preview` label -# (GitHub restricts who can label a PR) — arbitrary PR authors, including +# (GitHub restricts who can label a PR): arbitrary PR authors, including # from forks, cannot trigger this themselves. Even so, a fork PR still runs # attacker-authored code once labeled; the comment flags this so whoever # labels it is doing so knowingly. @@ -456,7 +456,7 @@ jobs: # Interactive SSH debug session into the live runner (cluster still # up, KUBECONFIG still valid) instead of guessing blind from static # logs. limit-access-to-actor restricts the SSH session to whoever - # triggered this run — required on a public repo. Bounded to 20min + # triggered this run, required on a public repo. Bounded to 20min # so a forgotten session doesn't eat the whole 90min job timeout. - name: Debug via tmate SSH on deploy failure if: failure() @@ -503,7 +503,7 @@ jobs: # changes), so a re-run after the first attempt already created # this tunnel (and didn't get to clean it up) hits a 409 name # conflict here. Reuse the existing tunnel by name instead of - # failing — it doesn't need the original tunnel_secret, just a + # failing: it doesn't need the original tunnel_secret, just a # fresh --token from the /token endpoint below. if [ -z "$tunnel_id" ]; then echo "::warning::Tunnel create failed (likely a name conflict from a retry), looking up existing tunnel named ${tunnel_name}: $create_resp" @@ -634,6 +634,7 @@ jobs: -f description="Live for 20 minutes" > /dev/null - name: Comment preview link on PR + id: comment_preview uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 with: header: k8s-preview @@ -645,9 +646,9 @@ jobs: | --- | --- | --- | --- | | `nebari-data-science-pack` | 🟢 [Ready](${{ steps.cf_dns.outputs.url }}) | [Preview](${{ steps.cf_dns.outputs.url }}) · [Keycloak](${{ steps.cf_dns.outputs.keycloak_url }}) | ${{ steps.timestamps.outputs.deployed_at }} | - ${{ github.event.pull_request.head.repo.fork && '⚠️ **This PR is from a fork** — the code running in this preview is not from a trusted maintainer branch.' || '' }} + ${{ github.event.pull_request.head.repo.fork && '⚠️ **This PR is from a fork**: the code running in this preview is not from a trusted maintainer branch.' || '' }} - Expires ${{ steps.timestamps.outputs.expires_at }} — push a new commit or re-add the `deploy-preview` label to redeploy. + Expires ${{ steps.timestamps.outputs.expires_at }}. Push a new commit or re-add the `deploy-preview` label to redeploy. # Bounded to 20min so the live preview doesn't sit open (and burn CI # minutes) indefinitely, and so this step ends itself well under the @@ -693,6 +694,30 @@ jobs: gh api "repos/${{ github.repository }}/deployments/${DEPLOYMENT_ID}/statuses" \ -X POST -f state=inactive -f description="Preview expired" > /dev/null || true + # The comment above still says "Ready" / "Expires in N minutes" once + # the tunnel closes -- it was only ever written once, at deploy time, + # and only keeps the clock ticking, not the + # surrounding wording or status. Edit it again now that the run is + # actually tearing down, so a reader doesn't see a green "Ready" row + # next to a relative time that has flipped to the past. Only do this + # when the first post succeeded (skip if the run failed before ever + # reaching that step, since there is no live-preview comment to + # correct in that case). + - name: Comment that the preview expired + if: always() && steps.comment_preview.outcome == 'success' + uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 + with: + header: k8s-preview + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + message: | + The K8s stack preview for this PR has expired. + + | Project | Deployment | Actions | Updated | + | --- | --- | --- | --- | + | `nebari-data-science-pack` | ⚫ Expired | - | ${{ steps.timestamps.outputs.expires_at }} | + + Push a new commit or re-add the `deploy-preview` label to redeploy. + - name: Delete DNS record if: always() env: @@ -753,6 +778,6 @@ jobs: header: k8s-preview GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} message: | - **K8s stack preview** stopped — the `deploy-preview` label was removed. + **K8s stack preview** stopped: the `deploy-preview` label was removed. Add it again to redeploy. From f6c9f4a581d4fb923691fe9e1feae29da471556c Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Tue, 1 Sep 2026 14:56:01 +0100 Subject: [PATCH 47/54] ci: extend the preview's lifetime in place via an extend-preview label Adding extend-preview any time before the preview expires pushes the tunnel step's deadline back 20 minutes from that moment (a reset, not a cumulative add), without tearing down and rebuilding the cluster. The already-running tunnel step polls for the label every 15s and consumes it by deleting it, so it can be reused any number of times before the job's own 90min timeout-minutes catches up. Scopes the job's labeled/unlabeled trigger to the deploy-preview label specifically, since a labeled event fires for any label add -- without this, adding extend-preview would start a second run of the job, which concurrency: cancel-in-progress then uses to cancel the very run it was meant to extend. --- .github/workflows/k8s-preview.yaml | 100 ++++++++++++++++++++++------- 1 file changed, 78 insertions(+), 22 deletions(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 81f94fc..d5c72cd 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -100,19 +100,44 @@ env: CLOUDFLARED_SHA256: "9d71c677db00134c1bd4144b7783486b654ad281b1ea62b4972098d19f770f17" jobs: + # A `labeled` event fires the whole workflow for ANY label add, not just + # deploy-preview -- e.g. someone adding extend-preview (see the tunnel + # step below) would otherwise start a second run of this job, which + # `concurrency: cancel-in-progress` then uses to cancel the very run + # extend-preview was meant to extend. Scope labeled/unlabeled triggers to + # the deploy-preview label itself; extend-preview is read by the + # already-running tunnel step polling the API, not by starting a new run. deploy-preview: - if: contains(github.event.pull_request.labels.*.name, 'deploy-preview') && github.event.action != 'unlabeled' + if: >- + contains(github.event.pull_request.labels.*.name, 'deploy-preview') && + github.event.action != 'unlabeled' && + (github.event.action != 'labeled' || github.event.label.name == 'deploy-preview') name: Deploy preview runs-on: ubuntu-24.04 timeout-minutes: 90 permissions: contents: read pull-requests: write + issues: write deployments: write steps: - name: Checkout uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + # So it shows up as an existing, pickable label in the PR's label + # picker instead of requiring someone to freehand-type a brand-new + # name (which GitHub does support, but it's an easy thing to miss). + # `gh label create` errors if the label already exists; that's fine, + # this step is just making sure it's there. + - name: Ensure the extend-preview label exists + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh label create extend-preview --repo "${{ github.repository }}" \ + --color BFD4F2 \ + --description "Push this preview's expiry back 20 minutes" \ + 2>/dev/null || true + # v0.32.0+ required: the sandbox's kind cluster uses containerd's # config v4 format, which `kind load` on older CLI releases can't # parse ("ERROR: unknown containerd config version: 4"). @@ -648,29 +673,60 @@ jobs: ${{ github.event.pull_request.head.repo.fork && '⚠️ **This PR is from a fork**: the code running in this preview is not from a trusted maintainer branch.' || '' }} - Expires ${{ steps.timestamps.outputs.expires_at }}. Push a new commit or re-add the `deploy-preview` label to redeploy. - - # Bounded to 20min so the live preview doesn't sit open (and burn CI - # minutes) indefinitely, and so this step ends itself well under the - # job's 90min timeout-minutes -- letting the job's own timeout kill - # this step instead makes GitHub mark the whole run "cancelled" with - # a "job has exceeded the maximum execution time" failure annotation, - # which looks like a real failure even though nothing broke -- the - # preview is *meant* to expire. `timeout`'s exit 124 (its own - # deadline hit) is the expected outcome and treated as success; any - # other nonzero exit (e.g. cloudflared itself crashing) still fails - # the step for real. - # GitHub Actions runs `run:` steps as `bash -e`: a plain - # `cmd; ec=$?` doesn't work here because -e aborts the step the - # instant `timeout` returns 124, before `ec=$?` is ever reached. - # `|| ec=$?` catches the nonzero exit inside a protected context - # instead (the compound command's own status is that of the - # assignment, which always succeeds). - - name: Run tunnel until it times out + Expires ${{ steps.timestamps.outputs.expires_at }}. Add the `extend-preview` label any time before then for 20 more minutes, or push a new commit or re-add `deploy-preview` to redeploy from scratch. + + # Bounded to 20min by default so the live preview doesn't sit open + # (and burn CI minutes) indefinitely, and so this step ends itself + # well under the job's 90min timeout-minutes -- letting the job's own + # timeout kill this step instead makes GitHub mark the whole run + # "cancelled" with a "job has exceeded the maximum execution time" + # failure annotation, which looks like a real failure even though + # nothing broke -- the preview is *meant* to expire eventually. + # + # Runs cloudflared in the background instead of blocking on it, so + # this loop can poll for the `extend-preview` label between checks + # and push the deadline back 20 minutes from whenever it actually + # sees the label (not just tack 20 more onto whatever's left) -- + # each occurrence is a one-shot reset, consumed by removing the + # label, so it can be reused any number of times before expiry. + # Ultimately still bounded by the job's own 90min timeout-minutes + # regardless of how many times it's extended. + # + # `kill -0 $pid` checks liveness without signaling; if cloudflared + # exits on its own (a real crash, not us closing it), `wait` reaps + # its actual exit code so a genuine failure still fails the step -- + # `|| ec=$?` catches that under `bash -e` (GitHub Actions' default + # for run: steps) without the step aborting mid-check. + - name: Run tunnel until it times out (extend via the extend-preview label) + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | + /tmp/cloudflared tunnel --no-autoupdate run --token "${TUNNEL_TOKEN}" & + cloudflared_pid=$! + + deadline=$(( $(date +%s) + 1200 )) ec=0 - timeout 1200 /tmp/cloudflared tunnel --no-autoupdate run --token "${TUNNEL_TOKEN}" || ec=$? - [ "$ec" -eq 0 ] || [ "$ec" -eq 124 ] || exit "$ec" + while true; do + if ! kill -0 "$cloudflared_pid" 2>/dev/null; then + wait "$cloudflared_pid" || ec=$? + break + fi + now=$(date +%s) + if [ "$now" -ge "$deadline" ]; then + kill "$cloudflared_pid" 2>/dev/null || true + wait "$cloudflared_pid" 2>/dev/null || true + break + fi + if gh api "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels" --jq '.[].name' 2>/dev/null \ + | grep -qx "extend-preview"; then + deadline=$(( now + 1200 )) + gh api -X DELETE "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels/extend-preview" \ + > /dev/null 2>&1 || true + echo "extend-preview seen -- new deadline: $(date -u -d "@$deadline" +'%Y-%m-%d %H:%M UTC')" + fi + sleep 15 + done + [ "$ec" -eq 0 ] || exit "$ec" # Runs once the tunnel step above ends (its own timeout, cloudflared # exiting, or a manual cancel), so this captures anything logged in From 3693d08f0995e2074524a50fdd120407a423c698 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Tue, 1 Sep 2026 15:09:11 +0100 Subject: [PATCH 48/54] ci: don't let extend-preview cancel the run it's meant to extend The job-level if: condition added earlier only skips deploy-preview's job for a labeled event on any label but deploy-preview -- it doesn't stop concurrency: cancel-in-progress from cancelling the current run, since that operates on the workflow run itself regardless of whether the job inside it ends up running. Confirmed live: adding extend-preview queued a new run that was about to cancel the one it was supposed to extend. cancel-in-progress now evaluates to false specifically for a labeled event on extend-preview, so that new (skipped) run just queues behind the current one instead of cancelling it. --- .github/workflows/k8s-preview.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index d5c72cd..25a08e8 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -86,9 +86,18 @@ on: pull_request: types: [labeled, unlabeled, synchronize] +# cancel-in-progress is normally what we want (one live preview per PR, +# newest commit wins), but it operates on the whole workflow RUN, not on +# whether deploy-preview's job condition ends up true. Adding the +# extend-preview label (see the tunnel step) fires a `labeled` event just +# like any other label add, which would otherwise queue a new run here and +# cancel the very run extend-preview was meant to extend -- even though +# that new run's job condition skips it and does nothing. Excluding that +# one case lets the new (skipped) run queue harmlessly behind the current +# one instead of cancelling it. concurrency: group: k8s-preview-${{ github.event.pull_request.number }} - cancel-in-progress: true + cancel-in-progress: ${{ !(github.event.action == 'labeled' && github.event.label.name == 'extend-preview') }} env: PREVIEW_LABEL: deploy-preview From 9ab529d954f019566b65e16c382ac45883b95959 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Wed, 2 Sep 2026 15:19:49 +0100 Subject: [PATCH 49/54] refactor: add a testable Python foundation for k8s-preview.yaml's CI logic Starts extracting the business logic currently embedded as inline bash in .github/workflows/k8s-preview.yaml into scripts/preview/, following the existing scripts/bump_image_tags.py pattern (pure functions + a thin CLI entry, imported directly in tests rather than shelled out to). This increment adds the shared HTTP helper and GitHub REST wrappers (labels, deployments, workflow runs) with full test coverage; nothing in the workflow is wired to them yet. --- scripts/preview/github_api.py | 152 +++++++++++++++++++++ scripts/preview/http.py | 55 ++++++++ tests/unit/test_preview_github_api.py | 181 ++++++++++++++++++++++++++ tests/unit/test_preview_http.py | 112 ++++++++++++++++ 4 files changed, 500 insertions(+) create mode 100644 scripts/preview/github_api.py create mode 100644 scripts/preview/http.py create mode 100644 tests/unit/test_preview_github_api.py create mode 100644 tests/unit/test_preview_http.py diff --git a/scripts/preview/github_api.py b/scripts/preview/github_api.py new file mode 100644 index 0000000..60a5979 --- /dev/null +++ b/scripts/preview/github_api.py @@ -0,0 +1,152 @@ +"""GitHub REST API calls shared by the preview-deploy scripts. + +All calls go through ``http.request_json`` (stdlib ``urllib.request``, no new +dependency) rather than shelling out to the ``gh`` CLI, so the actual request +being made is a plain, testable function call. +""" + +from __future__ import annotations + +from .http import HTTPRequestError, request_json + +API_ROOT = "https://api.github.com" + + +def _headers(token: str) -> dict[str, str]: + return { + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + } + + +# --- labels ------------------------------------------------------------- + + +def list_labels(repo: str, pr_number: int, token: str) -> list[str]: + """Return the names of every label currently on the PR.""" + result = request_json( + "GET", f"{API_ROOT}/repos/{repo}/issues/{pr_number}/labels", headers=_headers(token) + ) + return [item["name"] for item in result] + + +def delete_label(repo: str, pr_number: int, label: str, token: str) -> None: + """Remove ``label`` from the PR. A no-op if it's already gone.""" + try: + request_json( + "DELETE", + f"{API_ROOT}/repos/{repo}/issues/{pr_number}/labels/{label}", + headers=_headers(token), + ) + except HTTPRequestError as exc: + if exc.status != 404: + raise + + +def ensure_label_exists(repo: str, name: str, color: str, description: str, token: str) -> None: + """Create the repo label ``name`` if it doesn't already exist.""" + try: + request_json( + "POST", + f"{API_ROOT}/repos/{repo}/labels", + headers=_headers(token), + body={"name": name, "color": color, "description": description}, + ) + except HTTPRequestError as exc: + if exc.status != 422: + raise + + +# --- deployments ---------------------------------------------------------- + + +def create_deployment( + repo: str, + ref: str, + environment: str, + token: str, + task: str = "deploy:preview", + description: str = "", +) -> int: + """Create a GitHub Deployment and return its id.""" + result = request_json( + "POST", + f"{API_ROOT}/repos/{repo}/deployments", + headers=_headers(token), + body={ + "ref": ref, + "environment": environment, + "task": task, + "auto_merge": False, + "transient_environment": True, + "production_environment": False, + "required_contexts": [], + "description": description, + }, + ) + return result["id"] + + +def set_deployment_status( + repo: str, + deployment_id: int, + state: str, + token: str, + environment_url: str | None = None, + log_url: str | None = None, + description: str | None = None, +) -> None: + """Post a new status onto a deployment. Optional fields are omitted, not sent empty.""" + body: dict[str, str] = {"state": state} + if environment_url is not None: + body["environment_url"] = environment_url + if log_url is not None: + body["log_url"] = log_url + if description is not None: + body["description"] = description + request_json( + "POST", + f"{API_ROOT}/repos/{repo}/deployments/{deployment_id}/statuses", + headers=_headers(token), + body=body, + ) + + +def mark_deployment_inactive(repo: str, deployment_id: int, token: str, description: str = "") -> None: + set_deployment_status(repo, deployment_id, "inactive", token, description=description) + + +def find_latest_deployment_id(repo: str, environment: str, token: str) -> int | None: + """Return the most recent deployment id for ``environment``, or None.""" + result = request_json( + "GET", + f"{API_ROOT}/repos/{repo}/deployments?environment={environment}&per_page=1", + headers=_headers(token), + ) + return result[0]["id"] if result else None + + +# --- workflow runs ---------------------------------------------------------- + + +def cancel_in_flight_run(repo: str, workflow_name: str, pr_number: int, token: str) -> int | None: + """Cancel the in-progress run of ``workflow_name`` for this PR, if any. + + Returns the cancelled run's id, or None if no matching run was running. + """ + result = request_json( + "GET", + f"{API_ROOT}/repos/{repo}/actions/runs?event=pull_request&status=in_progress", + headers=_headers(token), + ) + for run in result.get("workflow_runs", []): + if run["name"] != workflow_name: + continue + if any(pr["number"] == pr_number for pr in run.get("pull_requests", [])): + request_json( + "POST", + f"{API_ROOT}/repos/{repo}/actions/runs/{run['id']}/cancel", + headers=_headers(token), + ) + return run["id"] + return None diff --git a/scripts/preview/http.py b/scripts/preview/http.py new file mode 100644 index 0000000..821d4c0 --- /dev/null +++ b/scripts/preview/http.py @@ -0,0 +1,55 @@ +"""Shared JSON HTTP helper for scripts/preview/*.py. + +Thin wrapper over ``urllib.request`` (stdlib, no new dependency -- matches +config/jupyterhub/01-spawner.py's existing convention for this same kind of +Keycloak/GitHub/Cloudflare-style REST call). Raises HTTPRequestError with the +response body on a non-2xx status, so callers get an actual error message +instead of urllib's bare HTTPError. +""" + +from __future__ import annotations + +import json +import urllib.error +import urllib.request +from typing import Any + + +class HTTPRequestError(RuntimeError): + """Raised when a JSON HTTP request returns a non-2xx status.""" + + def __init__(self, method: str, url: str, status: int, body: str): + self.status = status + self.body = body + super().__init__(f"{method} {url} -> HTTP {status}: {body}") + + +def request_json( + method: str, + url: str, + headers: dict[str, str] | None = None, + body: Any = None, + timeout: float = 15, +) -> Any: + """Send an HTTP request, returning the parsed JSON response body. + + ``body``, if given, is JSON-encoded and sent with an + application/json Content-Type. Returns ``{}`` for an empty (e.g. 204) + response body. + """ + data = None + req_headers = dict(headers or {}) + if body is not None: + data = json.dumps(body).encode("utf-8") + req_headers.setdefault("Content-Type", "application/json") + + request = urllib.request.Request(url, data=data, headers=req_headers, method=method) + try: + with urllib.request.urlopen(request, timeout=timeout) as resp: + raw = resp.read() + except urllib.error.HTTPError as exc: + raise HTTPRequestError(method, url, exc.code, exc.read().decode("utf-8", errors="replace")) from exc + + if not raw: + return {} + return json.loads(raw) diff --git a/tests/unit/test_preview_github_api.py b/tests/unit/test_preview_github_api.py new file mode 100644 index 0000000..33a2b89 --- /dev/null +++ b/tests/unit/test_preview_github_api.py @@ -0,0 +1,181 @@ +"""Tests for scripts/preview/github_api.py's GitHub REST helpers.""" + +from __future__ import annotations + +import pytest + +from scripts.preview import github_api +from scripts.preview.http import HTTPRequestError + +REPO = "nebari-dev/data-science-pack" +TOKEN = "gh-token" + + +def _capture(monkeypatch): + calls = [] + + def fake_request_json(method, url, headers=None, body=None, timeout=15): + calls.append({"method": method, "url": url, "headers": headers, "body": body}) + return fake_request_json.next_result + + fake_request_json.next_result = {} + monkeypatch.setattr("scripts.preview.github_api.request_json", fake_request_json) + return calls, fake_request_json + + +# --- labels ----------------------------------------------------------------- + + +def test_list_labels_returns_names(monkeypatch): + calls, fake = _capture(monkeypatch) + fake.next_result = [{"name": "deploy-preview"}, {"name": "extend-preview"}] + + result = github_api.list_labels(REPO, 205, TOKEN) + + assert result == ["deploy-preview", "extend-preview"] + assert calls[0]["url"] == f"https://api.github.com/repos/{REPO}/issues/205/labels" + assert calls[0]["method"] == "GET" + assert calls[0]["headers"]["Authorization"] == f"Bearer {TOKEN}" + + +def test_delete_label_calls_delete_endpoint(monkeypatch): + calls, _ = _capture(monkeypatch) + + github_api.delete_label(REPO, 205, "extend-preview", TOKEN) + + assert calls[0]["method"] == "DELETE" + assert calls[0]["url"] == f"https://api.github.com/repos/{REPO}/issues/205/labels/extend-preview" + + +def test_delete_label_swallows_404_already_removed(monkeypatch): + def fake_request_json(method, url, headers=None, body=None, timeout=15): + raise HTTPRequestError(method, url, 404, "Not Found") + + monkeypatch.setattr("scripts.preview.github_api.request_json", fake_request_json) + + github_api.delete_label(REPO, 205, "extend-preview", TOKEN) # must not raise + + +def test_delete_label_reraises_other_errors(monkeypatch): + def fake_request_json(method, url, headers=None, body=None, timeout=15): + raise HTTPRequestError(method, url, 403, "Forbidden") + + monkeypatch.setattr("scripts.preview.github_api.request_json", fake_request_json) + + with pytest.raises(HTTPRequestError): + github_api.delete_label(REPO, 205, "extend-preview", TOKEN) + + +def test_ensure_label_exists_swallows_422_already_exists(monkeypatch): + def fake_request_json(method, url, headers=None, body=None, timeout=15): + raise HTTPRequestError(method, url, 422, "already_exists") + + monkeypatch.setattr("scripts.preview.github_api.request_json", fake_request_json) + + github_api.ensure_label_exists(REPO, "extend-preview", "BFD4F2", "desc", TOKEN) + + +def test_ensure_label_exists_reraises_other_errors(monkeypatch): + def fake_request_json(method, url, headers=None, body=None, timeout=15): + raise HTTPRequestError(method, url, 500, "boom") + + monkeypatch.setattr("scripts.preview.github_api.request_json", fake_request_json) + + with pytest.raises(HTTPRequestError): + github_api.ensure_label_exists(REPO, "extend-preview", "BFD4F2", "desc", TOKEN) + + +# --- deployments -------------------------------------------------------------- + + +def test_create_deployment_posts_expected_payload_and_returns_id(monkeypatch): + calls, fake = _capture(monkeypatch) + fake.next_result = {"id": 6199751384} + + deployment_id = github_api.create_deployment( + REPO, ref="abc123", environment="pr-205-preview", token=TOKEN, + task="deploy:preview", description="K8s stack preview", + ) + + assert deployment_id == 6199751384 + body = calls[0]["body"] + assert body["ref"] == "abc123" + assert body["environment"] == "pr-205-preview" + assert body["task"] == "deploy:preview" + assert body["auto_merge"] is False + assert body["transient_environment"] is True + assert body["production_environment"] is False + assert body["required_contexts"] == [] + + +def test_set_deployment_status_omits_unset_optional_fields(monkeypatch): + calls, _ = _capture(monkeypatch) + + github_api.set_deployment_status(REPO, 42, "success", TOKEN, environment_url="https://x") + + body = calls[0]["body"] + assert body["state"] == "success" + assert body["environment_url"] == "https://x" + assert "log_url" not in body + assert "description" not in body + + +def test_mark_deployment_inactive_sets_inactive_state(monkeypatch): + calls, _ = _capture(monkeypatch) + + github_api.mark_deployment_inactive(REPO, 42, TOKEN, description="Preview expired") + + assert calls[0]["url"] == f"https://api.github.com/repos/{REPO}/deployments/42/statuses" + assert calls[0]["body"]["state"] == "inactive" + assert calls[0]["body"]["description"] == "Preview expired" + + +def test_find_latest_deployment_id_returns_first_result(monkeypatch): + _, fake = _capture(monkeypatch) + fake.next_result = [{"id": 111}, {"id": 222}] + + assert github_api.find_latest_deployment_id(REPO, "pr-205-preview", TOKEN) == 111 + + +def test_find_latest_deployment_id_returns_none_when_empty(monkeypatch): + _, fake = _capture(monkeypatch) + fake.next_result = [] + + assert github_api.find_latest_deployment_id(REPO, "pr-205-preview", TOKEN) is None + + +# --- runs --------------------------------------------------------------------- + + +def test_cancel_in_flight_run_finds_and_cancels_matching_run(monkeypatch): + calls = [] + + def fake_request_json(method, url, headers=None, body=None, timeout=15): + calls.append({"method": method, "url": url, "headers": headers, "body": body}) + if "workflow_runs" in url or url.endswith("status=in_progress"): + return { + "workflow_runs": [ + {"id": 1, "name": "Other Workflow", "pull_requests": [{"number": 205}]}, + {"id": 2, "name": "K8s Stack Preview", "pull_requests": [{"number": 999}]}, + {"id": 3, "name": "K8s Stack Preview", "pull_requests": [{"number": 205}]}, + ] + } + return {} + + monkeypatch.setattr("scripts.preview.github_api.request_json", fake_request_json) + + cancelled = github_api.cancel_in_flight_run(REPO, "K8s Stack Preview", 205, TOKEN) + + assert cancelled == 3 + cancel_calls = [c for c in calls if c["url"].endswith("/runs/3/cancel")] + assert len(cancel_calls) == 1 + assert cancel_calls[0]["method"] == "POST" + + +def test_cancel_in_flight_run_returns_none_when_no_match(monkeypatch): + def fake_request_json(method, url, headers=None, body=None, timeout=15): + return {"workflow_runs": []} + + monkeypatch.setattr("scripts.preview.github_api.request_json", fake_request_json) + + assert github_api.cancel_in_flight_run(REPO, "K8s Stack Preview", 205, TOKEN) is None diff --git a/tests/unit/test_preview_http.py b/tests/unit/test_preview_http.py new file mode 100644 index 0000000..eb41e92 --- /dev/null +++ b/tests/unit/test_preview_http.py @@ -0,0 +1,112 @@ +"""Tests for scripts/preview/http.py's shared JSON HTTP helper.""" + +from __future__ import annotations + +import json +import urllib.error + +import pytest + +from scripts.preview.http import HTTPRequestError, request_json + + +class _FakeResponse: + def __init__(self, body: bytes): + self._body = body + + def read(self) -> bytes: + return self._body + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +def test_get_returns_parsed_json_body(monkeypatch): + captured = {} + + def fake_urlopen(request, timeout): + captured["url"] = request.full_url + captured["method"] = request.get_method() + captured["timeout"] = timeout + return _FakeResponse(b'{"id": 42}') + + monkeypatch.setattr("scripts.preview.http.urllib.request.urlopen", fake_urlopen) + + result = request_json("GET", "https://api.example.com/thing", timeout=5) + + assert result == {"id": 42} + assert captured["url"] == "https://api.example.com/thing" + assert captured["method"] == "GET" + assert captured["timeout"] == 5 + + +def test_post_sends_json_encoded_body_with_content_type(monkeypatch): + captured = {} + + def fake_urlopen(request, timeout): + captured["data"] = request.data + captured["content_type"] = request.get_header("Content-type") + return _FakeResponse(b"{}") + + monkeypatch.setattr("scripts.preview.http.urllib.request.urlopen", fake_urlopen) + + request_json("POST", "https://api.example.com/thing", body={"name": "pr-205"}) + + assert json.loads(captured["data"]) == {"name": "pr-205"} + assert captured["content_type"] == "application/json" + + +def test_custom_headers_are_forwarded(monkeypatch): + captured = {} + + def fake_urlopen(request, timeout): + captured["auth"] = request.get_header("Authorization") + return _FakeResponse(b"{}") + + monkeypatch.setattr("scripts.preview.http.urllib.request.urlopen", fake_urlopen) + + request_json( + "GET", + "https://api.example.com/thing", + headers={"Authorization": "Bearer tok"}, + ) + + assert captured["auth"] == "Bearer tok" + + +def test_empty_response_body_returns_empty_dict(monkeypatch): + monkeypatch.setattr( + "scripts.preview.http.urllib.request.urlopen", + lambda request, timeout: _FakeResponse(b""), + ) + + assert request_json("DELETE", "https://api.example.com/thing") == {} + + +def test_http_error_raises_with_status_and_body(monkeypatch): + def fake_urlopen(request, timeout): + raise urllib.error.HTTPError( + request.full_url, 409, "Conflict", hdrs=None, fp=None + ) + + monkeypatch.setattr("scripts.preview.http.urllib.request.urlopen", fake_urlopen) + # HTTPError.read() needs a real body stream; patch it directly on the + # instance urlopen raises rather than fighting urllib's fp plumbing. + real_error = urllib.error.HTTPError( + "https://api.example.com/thing", 409, "Conflict", hdrs=None, fp=None + ) + monkeypatch.setattr(real_error, "read", lambda: b"name already exists") + monkeypatch.setattr( + "scripts.preview.http.urllib.request.urlopen", + lambda request, timeout: (_ for _ in ()).throw(real_error), + ) + + with pytest.raises(HTTPRequestError) as exc_info: + request_json("POST", "https://api.example.com/thing", body={"name": "x"}) + + assert exc_info.value.status == 409 + assert "name already exists" in exc_info.value.body + assert "name already exists" in str(exc_info.value) From 748e6d56dd139c8e865b2591f97b21fc8e74647e Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Wed, 2 Sep 2026 15:29:13 +0100 Subject: [PATCH 50/54] refactor: move Cloudflare Tunnel/DNS logic into scripts/preview/cloudflare.py Replaces ~110 lines of inline bash (tunnel create with 409-retry-reuse handling, ingress config, DNS zone/record creation, both cleanup deletes) across 4 workflow steps with one-line CLI calls into a tested Python module. Same behavior: tunnel/zone/record ids and the tunnel token still flow through GITHUB_ENV/GITHUB_OUTPUT exactly as before, cleanup stays best-effort (never fails the job). Adds scripts/preview/gha.py, a shared helper for writing step outputs/env vars/masks from these scripts (multiline-safe), used by cloudflare.py's CLI layer and reusable by the rest of this refactor. --- .github/workflows/k8s-preview.yaml | 114 ++----------- scripts/preview/cloudflare.py | 232 +++++++++++++++++++++++++ scripts/preview/gha.py | 36 ++++ tests/unit/test_preview_cloudflare.py | 237 ++++++++++++++++++++++++++ tests/unit/test_preview_gha.py | 59 +++++++ 5 files changed, 581 insertions(+), 97 deletions(-) create mode 100644 scripts/preview/cloudflare.py create mode 100644 scripts/preview/gha.py create mode 100644 tests/unit/test_preview_cloudflare.py create mode 100644 tests/unit/test_preview_gha.py diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 25a08e8..bdd8bd0 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -518,98 +518,25 @@ jobs: env: CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} CF_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_TUNNEL_ACCOUNT_ID }} - PREVIEW_HOSTNAME: pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }} - KEYCLOAK_HOSTNAME: keycloak-pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }} run: | - tunnel_secret=$(openssl rand -base64 32) - echo "::add-mask::${tunnel_secret}" - - tunnel_name="pr-${{ github.event.pull_request.number }}-${{ github.run_id }}" - create_resp=$(curl -sS -X POST \ - "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel" \ - -H "Authorization: Bearer ${CF_API_TOKEN}" \ - -H "Content-Type: application/json" \ - -d "$(jq -n --arg name "$tunnel_name" --arg secret "$tunnel_secret" \ - '{name: $name, config_src: "cloudflare", tunnel_secret: $secret}')") - tunnel_id=$(jq -r '.result.id // empty' <<< "$create_resp") - - # A GitHub Actions retry reuses the same run_id (only run_attempt - # changes), so a re-run after the first attempt already created - # this tunnel (and didn't get to clean it up) hits a 409 name - # conflict here. Reuse the existing tunnel by name instead of - # failing: it doesn't need the original tunnel_secret, just a - # fresh --token from the /token endpoint below. - if [ -z "$tunnel_id" ]; then - echo "::warning::Tunnel create failed (likely a name conflict from a retry), looking up existing tunnel named ${tunnel_name}: $create_resp" - tunnel_id=$(curl -fsS "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel?name=${tunnel_name}&is_deleted=false" \ - -H "Authorization: Bearer ${CF_API_TOKEN}" | jq -r '.result[0].id // empty') - fi - if [ -z "$tunnel_id" ]; then - echo "::error::Tunnel creation failed and no existing tunnel named ${tunnel_name} found: $create_resp" - exit 1 - fi - echo "tunnel_id=${tunnel_id}" >> "$GITHUB_OUTPUT" - echo "TUNNEL_ID=${tunnel_id}" >> "$GITHUB_ENV" - - token_resp=$(curl -fsS \ - "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel/${tunnel_id}/token" \ - -H "Authorization: Bearer ${CF_API_TOKEN}") - tunnel_token=$(jq -r '.result' <<< "$token_resp") - echo "::add-mask::${tunnel_token}" - echo "TUNNEL_TOKEN=${tunnel_token}" >> "$GITHUB_ENV" - - curl -fsS -X PUT \ - "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel/${tunnel_id}/configurations" \ - -H "Authorization: Bearer ${CF_API_TOKEN}" \ - -H "Content-Type: application/json" \ - -d "$(jq -n --arg host "$PREVIEW_HOSTNAME" --arg kchost "$KEYCLOAK_HOSTNAME" \ - '{config: {ingress: [ - {hostname: $host, service: "http://localhost:8000"}, - {hostname: $kchost, service: "http://localhost:8001"}, - {service: "http_status:404"} - ]}}')" \ - > /dev/null + python -m scripts.preview.cloudflare create-tunnel \ + --account-id "$CF_ACCOUNT_ID" --api-token "$CF_API_TOKEN" \ + --name "pr-${{ github.event.pull_request.number }}-${{ github.run_id }}" \ + --preview-hostname "pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }}" \ + --preview-service "http://localhost:8000" \ + --keycloak-hostname "keycloak-pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }}" \ + --keycloak-service "http://localhost:8001" - name: Point DNS at the tunnel id: cf_dns env: CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} - PREVIEW_HOSTNAME: pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }} - KEYCLOAK_HOSTNAME: keycloak-pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }} run: | - zone_id=$(curl -fsS "https://api.cloudflare.com/client/v4/zones?name=${PREVIEW_DOMAIN}" \ - -H "Authorization: Bearer ${CF_API_TOKEN}" | jq -r '.result[0].id') - if [ -z "$zone_id" ] || [ "$zone_id" = "null" ]; then - echo "::error::Could not resolve zone id for ${PREVIEW_DOMAIN}" - exit 1 - fi - echo "ZONE_ID=${zone_id}" >> "$GITHUB_ENV" - - record_resp=$(curl -fsS -X POST "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records" \ - -H "Authorization: Bearer ${CF_API_TOKEN}" \ - -H "Content-Type: application/json" \ - -d "$(jq -n --arg host "$PREVIEW_HOSTNAME" --arg target "${TUNNEL_ID}.cfargotunnel.com" \ - '{type: "CNAME", name: $host, content: $target, proxied: true}')") - record_id=$(jq -r '.result.id' <<< "$record_resp") - if [ -z "$record_id" ] || [ "$record_id" = "null" ]; then - echo "::error::DNS record creation failed: $record_resp" - exit 1 - fi - echo "DNS_RECORD_ID=${record_id}" >> "$GITHUB_ENV" - echo "url=https://${PREVIEW_HOSTNAME}" >> "$GITHUB_OUTPUT" - - kc_record_resp=$(curl -fsS -X POST "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records" \ - -H "Authorization: Bearer ${CF_API_TOKEN}" \ - -H "Content-Type: application/json" \ - -d "$(jq -n --arg host "$KEYCLOAK_HOSTNAME" --arg target "${TUNNEL_ID}.cfargotunnel.com" \ - '{type: "CNAME", name: $host, content: $target, proxied: true}')") - kc_record_id=$(jq -r '.result.id' <<< "$kc_record_resp") - if [ -z "$kc_record_id" ] || [ "$kc_record_id" = "null" ]; then - echo "::error::Keycloak DNS record creation failed: $kc_record_resp" - exit 1 - fi - echo "KEYCLOAK_DNS_RECORD_ID=${kc_record_id}" >> "$GITHUB_ENV" - echo "keycloak_url=https://${KEYCLOAK_HOSTNAME}" >> "$GITHUB_OUTPUT" + python -m scripts.preview.cloudflare create-dns \ + --api-token "$CF_API_TOKEN" --domain "${{ env.PREVIEW_DOMAIN }}" \ + --preview-hostname "pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }}" \ + --keycloak-hostname "keycloak-pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }}" \ + --target "${TUNNEL_ID}.cfargotunnel.com" # The URL itself (pr--data-science-pack.) is identical on every run, so # without a timestamp the sticky comment would post byte-identical @@ -788,14 +715,9 @@ jobs: env: CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} run: | - [ -n "${ZONE_ID:-}" ] && [ -n "${DNS_RECORD_ID:-}" ] || exit 0 - curl -fsS -X DELETE \ - "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records/${DNS_RECORD_ID}" \ - -H "Authorization: Bearer ${CF_API_TOKEN}" || true - [ -n "${KEYCLOAK_DNS_RECORD_ID:-}" ] || exit 0 - curl -fsS -X DELETE \ - "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records/${KEYCLOAK_DNS_RECORD_ID}" \ - -H "Authorization: Bearer ${CF_API_TOKEN}" || true + python -m scripts.preview.cloudflare delete-dns \ + --api-token "$CF_API_TOKEN" --zone-id "${ZONE_ID:-}" \ + --record-id "${DNS_RECORD_ID:-}" --record-id "${KEYCLOAK_DNS_RECORD_ID:-}" - name: Delete Cloudflare Tunnel if: always() @@ -803,10 +725,8 @@ jobs: CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} CF_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_TUNNEL_ACCOUNT_ID }} run: | - [ -n "${TUNNEL_ID:-}" ] || exit 0 - curl -fsS -X DELETE \ - "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel/${TUNNEL_ID}" \ - -H "Authorization: Bearer ${CF_API_TOKEN}" || true + python -m scripts.preview.cloudflare delete-tunnel \ + --account-id "$CF_ACCOUNT_ID" --api-token "$CF_API_TOKEN" --tunnel-id "${TUNNEL_ID:-}" cleanup-preview: if: github.event.action == 'unlabeled' && github.event.label.name == 'deploy-preview' diff --git a/scripts/preview/cloudflare.py b/scripts/preview/cloudflare.py new file mode 100644 index 0000000..3e6f4b2 --- /dev/null +++ b/scripts/preview/cloudflare.py @@ -0,0 +1,232 @@ +"""Cloudflare Tunnel + DNS API calls for the k8s preview environment. + +One per-PR named Tunnel (not the anonymous quick-tunnel) so it can sit +behind a Cloudflare Access application and each PR gets its own hostname. + +Usage (each subcommand mirrors one k8s-preview.yaml workflow step): + python -m scripts.preview.cloudflare create-tunnel --account-id ID \\ + --api-token TOKEN --name NAME \\ + --preview-hostname HOST --preview-service URL \\ + --keycloak-hostname HOST --keycloak-service URL + python -m scripts.preview.cloudflare create-dns --api-token TOKEN \\ + --domain DOMAIN --preview-hostname HOST --keycloak-hostname HOST \\ + --target TUNNEL_ID.cfargotunnel.com + python -m scripts.preview.cloudflare delete-dns --api-token TOKEN \\ + --zone-id ID --record-id ID [--record-id ID ...] + python -m scripts.preview.cloudflare delete-tunnel --account-id ID \\ + --api-token TOKEN --tunnel-id ID +""" + +from __future__ import annotations + +import argparse +import base64 +import secrets +import sys + +from . import gha +from .http import HTTPRequestError, request_json + +API_ROOT = "https://api.cloudflare.com/client/v4" + + +class TunnelCreationError(RuntimeError): + pass + + +class ZoneNotFoundError(RuntimeError): + pass + + +def _headers(api_token: str) -> dict[str, str]: + return {"Authorization": f"Bearer {api_token}"} + + +def create_or_reuse_tunnel(account_id: str, api_token: str, tunnel_name: str) -> str: + """Create a named Tunnel, or reuse one already created under this name. + + A GitHub Actions retry reuses the same run id (only run_attempt changes), + so a re-run after an attempt that already created this tunnel (and didn't + get to clean it up) hits a name conflict. Reuse the existing tunnel by + name instead of failing -- it doesn't need the original tunnel secret, + just a fresh token from ``get_tunnel_token``. + """ + tunnel_secret = base64.b64encode(secrets.token_bytes(32)).decode() + try: + result = request_json( + "POST", + f"{API_ROOT}/accounts/{account_id}/cfd_tunnel", + headers=_headers(api_token), + body={"name": tunnel_name, "config_src": "cloudflare", "tunnel_secret": tunnel_secret}, + ) + return result["result"]["id"] + except HTTPRequestError: + existing = request_json( + "GET", + f"{API_ROOT}/accounts/{account_id}/cfd_tunnel?name={tunnel_name}&is_deleted=false", + headers=_headers(api_token), + ) + matches = existing.get("result") or [] + if not matches: + raise TunnelCreationError( + f"tunnel creation failed and no existing tunnel named {tunnel_name!r} was found" + ) from None + return matches[0]["id"] + + +def get_tunnel_token(account_id: str, api_token: str, tunnel_id: str) -> str: + result = request_json( + "GET", + f"{API_ROOT}/accounts/{account_id}/cfd_tunnel/{tunnel_id}/token", + headers=_headers(api_token), + ) + return result["result"] + + +def configure_ingress( + account_id: str, api_token: str, tunnel_id: str, hostname_services: list[tuple[str, str]] +) -> None: + """Point each (hostname, service-url) pair at the tunnel, 404 for everything else.""" + ingress = [{"hostname": host, "service": service} for host, service in hostname_services] + ingress.append({"service": "http_status:404"}) + request_json( + "PUT", + f"{API_ROOT}/accounts/{account_id}/cfd_tunnel/{tunnel_id}/configurations", + headers=_headers(api_token), + body={"config": {"ingress": ingress}}, + ) + + +def delete_tunnel(account_id: str, api_token: str, tunnel_id: str) -> None: + request_json( + "DELETE", + f"{API_ROOT}/accounts/{account_id}/cfd_tunnel/{tunnel_id}", + headers=_headers(api_token), + ) + + +def resolve_zone_id(api_token: str, domain: str) -> str: + result = request_json("GET", f"{API_ROOT}/zones?name={domain}", headers=_headers(api_token)) + matches = result.get("result") or [] + if not matches: + raise ZoneNotFoundError(f"could not resolve a zone id for {domain!r}") + return matches[0]["id"] + + +def create_dns_record(api_token: str, zone_id: str, hostname: str, target: str) -> str: + result = request_json( + "POST", + f"{API_ROOT}/zones/{zone_id}/dns_records", + headers=_headers(api_token), + body={"type": "CNAME", "name": hostname, "content": target, "proxied": True}, + ) + return result["result"]["id"] + + +def delete_dns_record(api_token: str, zone_id: str, record_id: str) -> None: + request_json( + "DELETE", + f"{API_ROOT}/zones/{zone_id}/dns_records/{record_id}", + headers=_headers(api_token), + ) + + +def _cmd_create_tunnel(args: argparse.Namespace) -> int: + try: + tunnel_id = create_or_reuse_tunnel(args.account_id, args.api_token, args.name) + token = get_tunnel_token(args.account_id, args.api_token, tunnel_id) + configure_ingress( + args.account_id, args.api_token, tunnel_id, + [(args.preview_hostname, args.preview_service), (args.keycloak_hostname, args.keycloak_service)], + ) + except (TunnelCreationError, HTTPRequestError) as exc: + gha.error(f"Cloudflare Tunnel setup failed: {exc}") + return 1 + gha.mask(token) + gha.write_output("tunnel_id", tunnel_id) + gha.write_env("TUNNEL_ID", tunnel_id) + gha.write_env("TUNNEL_TOKEN", token) + return 0 + + +def _cmd_create_dns(args: argparse.Namespace) -> int: + try: + zone_id = resolve_zone_id(args.api_token, args.domain) + record_id = create_dns_record(args.api_token, zone_id, args.preview_hostname, args.target) + keycloak_record_id = create_dns_record(args.api_token, zone_id, args.keycloak_hostname, args.target) + except (ZoneNotFoundError, HTTPRequestError) as exc: + gha.error(f"DNS record creation failed: {exc}") + return 1 + gha.write_env("ZONE_ID", zone_id) + gha.write_env("DNS_RECORD_ID", record_id) + gha.write_env("KEYCLOAK_DNS_RECORD_ID", keycloak_record_id) + gha.write_output("url", f"https://{args.preview_hostname}") + gha.write_output("keycloak_url", f"https://{args.keycloak_hostname}") + return 0 + + +def _cmd_delete_dns(args: argparse.Namespace) -> int: + """Best-effort: cleanup steps must never fail the job.""" + if not args.zone_id: + return 0 + for record_id in args.record_id: + if not record_id: + continue + try: + delete_dns_record(args.api_token, args.zone_id, record_id) + except HTTPRequestError: + pass + return 0 + + +def _cmd_delete_tunnel(args: argparse.Namespace) -> int: + """Best-effort: cleanup steps must never fail the job.""" + if not args.tunnel_id: + return 0 + try: + delete_tunnel(args.account_id, args.api_token, args.tunnel_id) + except HTTPRequestError: + pass + return 0 + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(prog="cloudflare") + sub = parser.add_subparsers(dest="command", required=True) + + p = sub.add_parser("create-tunnel") + p.add_argument("--account-id", required=True) + p.add_argument("--api-token", required=True) + p.add_argument("--name", required=True) + p.add_argument("--preview-hostname", required=True) + p.add_argument("--preview-service", required=True) + p.add_argument("--keycloak-hostname", required=True) + p.add_argument("--keycloak-service", required=True) + p.set_defaults(func=_cmd_create_tunnel) + + p = sub.add_parser("create-dns") + p.add_argument("--api-token", required=True) + p.add_argument("--domain", required=True) + p.add_argument("--preview-hostname", required=True) + p.add_argument("--keycloak-hostname", required=True) + p.add_argument("--target", required=True) + p.set_defaults(func=_cmd_create_dns) + + p = sub.add_parser("delete-dns") + p.add_argument("--api-token", required=True) + p.add_argument("--zone-id", required=True) + p.add_argument("--record-id", action="append", default=[]) + p.set_defaults(func=_cmd_delete_dns) + + p = sub.add_parser("delete-tunnel") + p.add_argument("--account-id", required=True) + p.add_argument("--api-token", required=True) + p.add_argument("--tunnel-id", required=True) + p.set_defaults(func=_cmd_delete_tunnel) + + args = parser.parse_args(argv[1:]) + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/scripts/preview/gha.py b/scripts/preview/gha.py new file mode 100644 index 0000000..2a4ad24 --- /dev/null +++ b/scripts/preview/gha.py @@ -0,0 +1,36 @@ +"""Helpers for writing GitHub Actions step outputs/env vars/masks/errors. + +``path`` defaults to the real ``$GITHUB_OUTPUT``/``$GITHUB_ENV`` files GitHub +Actions provides at runtime; tests pass an explicit path instead. +""" + +from __future__ import annotations + +import os +import uuid + + +def _write_kv(path: str, name: str, value: str) -> None: + if "\n" in value: + delimiter = f"ghadelim_{uuid.uuid4().hex}" + block = f"{name}<<{delimiter}\n{value}\n{delimiter}\n" + else: + block = f"{name}={value}\n" + with open(path, "a", encoding="utf-8") as f: + f.write(block) + + +def write_output(name: str, value: str, path: str | None = None) -> None: + _write_kv(path or os.environ["GITHUB_OUTPUT"], name, value) + + +def write_env(name: str, value: str, path: str | None = None) -> None: + _write_kv(path or os.environ["GITHUB_ENV"], name, value) + + +def mask(value: str) -> None: + print(f"::add-mask::{value}") + + +def error(message: str) -> None: + print(f"::error::{message}") diff --git a/tests/unit/test_preview_cloudflare.py b/tests/unit/test_preview_cloudflare.py new file mode 100644 index 0000000..56037e1 --- /dev/null +++ b/tests/unit/test_preview_cloudflare.py @@ -0,0 +1,237 @@ +"""Tests for scripts/preview/cloudflare.py's Cloudflare Tunnel + DNS helpers.""" + +from __future__ import annotations + +import pytest + +from scripts.preview import cloudflare +from scripts.preview.http import HTTPRequestError + +ACCOUNT_ID = "acct-1" +API_TOKEN = "cf-token" + + +def _capture(monkeypatch): + calls = [] + + def fake_request_json(method, url, headers=None, body=None, timeout=15): + calls.append({"method": method, "url": url, "headers": headers, "body": body}) + return fake_request_json.next_result + + fake_request_json.next_result = {} + monkeypatch.setattr("scripts.preview.cloudflare.request_json", fake_request_json) + return calls, fake_request_json + + +# --- tunnel create/reuse ----------------------------------------------------- + + +def test_create_or_reuse_tunnel_returns_id_on_successful_create(monkeypatch): + calls, fake = _capture(monkeypatch) + fake.next_result = {"result": {"id": "tunnel-abc"}} + + tunnel_id = cloudflare.create_or_reuse_tunnel(ACCOUNT_ID, API_TOKEN, "pr-205-run-1") + + assert tunnel_id == "tunnel-abc" + assert calls[0]["method"] == "POST" + assert calls[0]["body"]["name"] == "pr-205-run-1" + assert calls[0]["body"]["config_src"] == "cloudflare" + assert "tunnel_secret" in calls[0]["body"] + + +def test_create_or_reuse_tunnel_reuses_existing_on_name_conflict(monkeypatch): + lookup_result = {"result": [{"id": "existing-tunnel"}]} + + def fake_request_json(method, url, headers=None, body=None, timeout=15): + if method == "POST": + raise HTTPRequestError(method, url, 409, "tunnel with name already exists") + return lookup_result + + monkeypatch.setattr("scripts.preview.cloudflare.request_json", fake_request_json) + + tunnel_id = cloudflare.create_or_reuse_tunnel(ACCOUNT_ID, API_TOKEN, "pr-205-run-1") + + assert tunnel_id == "existing-tunnel" + + +def test_create_or_reuse_tunnel_raises_when_conflict_and_no_existing_found(monkeypatch): + def fake_request_json(method, url, headers=None, body=None, timeout=15): + if method == "POST": + raise HTTPRequestError(method, url, 409, "conflict") + return {"result": []} + + monkeypatch.setattr("scripts.preview.cloudflare.request_json", fake_request_json) + + with pytest.raises(cloudflare.TunnelCreationError): + cloudflare.create_or_reuse_tunnel(ACCOUNT_ID, API_TOKEN, "pr-205-run-1") + + +# --- tunnel token / ingress / delete ------------------------------------------ + + +def test_get_tunnel_token_returns_result_string(monkeypatch): + _, fake = _capture(monkeypatch) + fake.next_result = {"result": "the-token"} + + assert cloudflare.get_tunnel_token(ACCOUNT_ID, API_TOKEN, "tunnel-abc") == "the-token" + + +def test_configure_ingress_sends_hostname_rules_with_catchall(monkeypatch): + calls, _ = _capture(monkeypatch) + + cloudflare.configure_ingress( + ACCOUNT_ID, API_TOKEN, "tunnel-abc", + [("pr-205.example.com", "http://localhost:8000"), ("kc-pr-205.example.com", "http://localhost:8001")], + ) + + ingress = calls[0]["body"]["config"]["ingress"] + assert ingress[0] == {"hostname": "pr-205.example.com", "service": "http://localhost:8000"} + assert ingress[1] == {"hostname": "kc-pr-205.example.com", "service": "http://localhost:8001"} + assert ingress[-1] == {"service": "http_status:404"} + + +def test_delete_tunnel_calls_delete_endpoint(monkeypatch): + calls, _ = _capture(monkeypatch) + + cloudflare.delete_tunnel(ACCOUNT_ID, API_TOKEN, "tunnel-abc") + + assert calls[0]["method"] == "DELETE" + assert calls[0]["url"].endswith("/accounts/acct-1/cfd_tunnel/tunnel-abc") + + +# --- DNS ----------------------------------------------------------------------- + + +def test_resolve_zone_id_returns_first_match(monkeypatch): + _, fake = _capture(monkeypatch) + fake.next_result = {"result": [{"id": "zone-1"}]} + + assert cloudflare.resolve_zone_id(API_TOKEN, "example.com") == "zone-1" + + +def test_resolve_zone_id_raises_when_not_found(monkeypatch): + _, fake = _capture(monkeypatch) + fake.next_result = {"result": []} + + with pytest.raises(cloudflare.ZoneNotFoundError): + cloudflare.resolve_zone_id(API_TOKEN, "example.com") + + +def test_create_dns_record_returns_record_id(monkeypatch): + calls, fake = _capture(monkeypatch) + fake.next_result = {"result": {"id": "record-1"}} + + record_id = cloudflare.create_dns_record(API_TOKEN, "zone-1", "pr-205.example.com", "tunnel-abc.cfargotunnel.com") + + assert record_id == "record-1" + assert calls[0]["body"] == { + "type": "CNAME", + "name": "pr-205.example.com", + "content": "tunnel-abc.cfargotunnel.com", + "proxied": True, + } + + +def test_delete_dns_record_calls_delete_endpoint(monkeypatch): + calls, _ = _capture(monkeypatch) + + cloudflare.delete_dns_record(API_TOKEN, "zone-1", "record-1") + + assert calls[0]["method"] == "DELETE" + assert calls[0]["url"].endswith("/zones/zone-1/dns_records/record-1") + + +# --- CLI (main) --------------------------------------------------------------- + + +def test_main_create_tunnel_writes_outputs_and_masks_secrets(monkeypatch, tmp_path): + monkeypatch.setattr(cloudflare, "create_or_reuse_tunnel", lambda *a, **k: "tunnel-abc") + monkeypatch.setattr(cloudflare, "get_tunnel_token", lambda *a, **k: "tok-xyz") + configured = {} + monkeypatch.setattr( + cloudflare, "configure_ingress", + lambda account_id, api_token, tunnel_id, hosts: configured.update(hosts=hosts), + ) + out_file = tmp_path / "output" + env_file = tmp_path / "env" + out_file.write_text("") + env_file.write_text("") + monkeypatch.setenv("GITHUB_OUTPUT", str(out_file)) + monkeypatch.setenv("GITHUB_ENV", str(env_file)) + + rc = cloudflare.main([ + "cloudflare", "create-tunnel", + "--account-id", "acct-1", "--api-token", "tok", "--name", "pr-205-1", + "--preview-hostname", "pr-205.example.com", "--preview-service", "http://localhost:8000", + "--keycloak-hostname", "kc-pr-205.example.com", "--keycloak-service", "http://localhost:8001", + ]) + + assert rc == 0 + assert "tunnel_id=tunnel-abc" in out_file.read_text() + assert "TUNNEL_ID=tunnel-abc" in env_file.read_text() + assert "TUNNEL_TOKEN=tok-xyz" in env_file.read_text() + assert configured["hosts"] == [ + ("pr-205.example.com", "http://localhost:8000"), + ("kc-pr-205.example.com", "http://localhost:8001"), + ] + + +def test_main_create_tunnel_exits_1_on_failure(monkeypatch, capsys): + def boom(*a, **k): + raise cloudflare.TunnelCreationError("no dice") + + monkeypatch.setattr(cloudflare, "create_or_reuse_tunnel", boom) + + rc = cloudflare.main([ + "cloudflare", "create-tunnel", + "--account-id", "acct-1", "--api-token", "tok", "--name", "pr-205-1", + "--preview-hostname", "pr-205.example.com", "--preview-service", "http://localhost:8000", + "--keycloak-hostname", "kc-pr-205.example.com", "--keycloak-service", "http://localhost:8001", + ]) + + assert rc == 1 + assert "::error::" in capsys.readouterr().out + + +def test_main_delete_dns_noops_when_zone_id_missing(monkeypatch): + called = [] + monkeypatch.setattr(cloudflare, "delete_dns_record", lambda *a, **k: called.append(a)) + + rc = cloudflare.main(["cloudflare", "delete-dns", "--api-token", "tok", "--zone-id", "", "--record-id", "r1"]) + + assert rc == 0 + assert called == [] + + +def test_main_delete_dns_deletes_each_nonempty_record_id(monkeypatch): + called = [] + monkeypatch.setattr(cloudflare, "delete_dns_record", lambda api_token, zone_id, record_id: called.append(record_id)) + + rc = cloudflare.main([ + "cloudflare", "delete-dns", "--api-token", "tok", "--zone-id", "zone-1", + "--record-id", "r1", "--record-id", "", + ]) + + assert rc == 0 + assert called == ["r1"] + + +def test_main_delete_dns_swallows_api_errors(monkeypatch): + def boom(*a, **k): + raise HTTPRequestError("DELETE", "https://x", 500, "boom") + + monkeypatch.setattr(cloudflare, "delete_dns_record", boom) + + rc = cloudflare.main(["cloudflare", "delete-dns", "--api-token", "tok", "--zone-id", "zone-1", "--record-id", "r1"]) + + assert rc == 0 # cleanup steps are best-effort, never fail the job + + +def test_main_delete_tunnel_noops_when_tunnel_id_missing(monkeypatch): + called = [] + monkeypatch.setattr(cloudflare, "delete_tunnel", lambda *a, **k: called.append(a)) + + rc = cloudflare.main(["cloudflare", "delete-tunnel", "--account-id", "acct-1", "--api-token", "tok", "--tunnel-id", ""]) + + assert rc == 0 + assert called == [] diff --git a/tests/unit/test_preview_gha.py b/tests/unit/test_preview_gha.py new file mode 100644 index 0000000..367a805 --- /dev/null +++ b/tests/unit/test_preview_gha.py @@ -0,0 +1,59 @@ +"""Tests for scripts/preview/gha.py's GitHub Actions I/O helpers.""" + +from __future__ import annotations + +from scripts.preview import gha + + +def test_write_output_appends_single_line_key_value(tmp_path, capsys): + out_file = tmp_path / "output" + out_file.write_text("") + + gha.write_output("url", "https://example.com", path=str(out_file)) + + assert out_file.read_text() == "url=https://example.com\n" + + +def test_write_output_appends_to_existing_content(tmp_path): + out_file = tmp_path / "output" + out_file.write_text("existing=1\n") + + gha.write_output("url", "https://example.com", path=str(out_file)) + + assert out_file.read_text() == "existing=1\nurl=https://example.com\n" + + +def test_write_output_multiline_value_uses_delimiter_block(tmp_path): + out_file = tmp_path / "output" + out_file.write_text("") + + gha.write_output("body", "line one\nline two", path=str(out_file)) + + content = out_file.read_text() + lines = content.splitlines() + assert lines[0].startswith("body<<") + delim = lines[0].split("<<", 1)[1] + assert lines[1] == "line one" + assert lines[2] == "line two" + assert lines[3] == delim + + +def test_write_env_appends_single_line_key_value(tmp_path): + env_file = tmp_path / "env" + env_file.write_text("") + + gha.write_env("TUNNEL_ID", "abc-123", path=str(env_file)) + + assert env_file.read_text() == "TUNNEL_ID=abc-123\n" + + +def test_mask_prints_add_mask_command(capsys): + gha.mask("super-secret-token") + + assert capsys.readouterr().out == "::add-mask::super-secret-token\n" + + +def test_error_prints_error_command(capsys): + gha.error("tunnel creation failed") + + assert capsys.readouterr().out == "::error::tunnel creation failed\n" From 6844087422675c07a0de8c83af72a45109278310 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Wed, 2 Sep 2026 15:39:43 +0100 Subject: [PATCH 51/54] refactor: move Keycloak + k8s readiness logic into scripts/preview/ Three workflow steps shrink from ~110 lines of inline bash to 1-3 line CLI calls: - Create a test login user in Keycloak -> scripts/preview/keycloak.py (admin token fetch + reviewer user creation). - Point Keycloak's own hostname at the public tunnel route -> scripts/preview/keycloak_gitops.py (the GitOps file rewrite, the ArgoCD-applied-yet poll loop, and the operator-deployment lookup are now pure, independently tested functions instead of an inline sed + jq + bash for-loop). - Wait for operator to provision the Keycloak client secret / Restart hub until it picks up the operator secret -> scripts/preview/k8s_wait.py (the two kubectl retry loops). Also fixes http.request_json to send a str body as-is instead of JSON-encoding it (needed for Keycloak's form-urlencoded token endpoint) -- request_json only handled dict/list bodies before. --- .github/workflows/k8s-preview.yaml | 125 +++------------- scripts/preview/http.py | 11 +- scripts/preview/k8s_wait.py | 132 +++++++++++++++++ scripts/preview/keycloak.py | 90 ++++++++++++ scripts/preview/keycloak_gitops.py | 162 +++++++++++++++++++++ tests/unit/test_preview_http.py | 22 +++ tests/unit/test_preview_k8s_wait.py | 79 ++++++++++ tests/unit/test_preview_keycloak.py | 82 +++++++++++ tests/unit/test_preview_keycloak_gitops.py | 120 +++++++++++++++ 9 files changed, 714 insertions(+), 109 deletions(-) create mode 100644 scripts/preview/k8s_wait.py create mode 100644 scripts/preview/keycloak.py create mode 100644 scripts/preview/keycloak_gitops.py create mode 100644 tests/unit/test_preview_k8s_wait.py create mode 100644 tests/unit/test_preview_keycloak.py create mode 100644 tests/unit/test_preview_keycloak_gitops.py diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index bdd8bd0..bc5c3db 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -205,72 +205,17 @@ jobs: kubectl create namespace pr-preview --dry-run=client -o yaml | kubectl apply -f - kubectl label namespace pr-preview nebari.dev/managed=true --overwrite - # Keycloak's own KC_HOSTNAME (codecentric/keycloakx, via NIC's - # values/keycloak/base.yaml) is fixed to NIC's internal - # keycloak.nebari.local by default -- every self-referencing URL - # Keycloak renders (login form action, issuer, redirects) is - # absolute and uses that value regardless of the incoming Host - # header, confirmed live: a request reaching Keycloak still rendered - # a login form whose action="https://keycloak.nebari.local/..." -- - # unreachable from a real browser, since .local is never publicly - # resolvable. - # - # A direct `kubectl set env` patch onto the live StatefulSet/ - # Deployment DOES apply immediately but gets silently reverted: - # both Keycloak and nebari-operator are ArgoCD Applications with - # selfHeal: true (pkg/argocd/templates/apps/{keycloak,nebari- - # operator}.yaml in nebari-infrastructure-core), continuously - # reconciled against NIC's auto-created local GitOps repo - # (~/.nic/gitops/, repository.local: {} in NIC's - # config) -- confirmed live: the patch step's own logs showed the - # StatefulSet/Deployment genuinely rolling to new pods, yet hub's - # redirect still landed on keycloak.nebari.local a couple minutes - # later once ArgoCD's reconcile loop caught the drift and reverted - # it. Editing the GitOps repo itself and forcing a hard refresh - # (the same pattern used for this project's other ArgoCD-managed - # clusters) lets selfHeal work for this change instead of against it. + # Why this is needed (ArgoCD selfHeal reverting a direct kubectl + # patch, Keycloak's KC_HOSTNAME being otherwise unreachable from a + # real browser) is explained in scripts/preview/keycloak_gitops.py's + # module docstring, next to the logic itself. - name: Point Keycloak's own hostname at the public tunnel route env: KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} run: | - kc_public_url="https://keycloak-pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }}" - gitops_dir="$HOME/.nic/gitops/${{ steps.sandbox.outputs.cluster-name }}" - - echo "--- current rendered values (pre-patch) ---" - grep -n "keycloak.nebari.local" "$gitops_dir/values/keycloak/base.yaml" "$gitops_dir/manifests/nebari-operator/deployment-patch.yaml" - - sed -i "s#https://keycloak\.nebari\.local#${kc_public_url}#g" \ - "$gitops_dir/values/keycloak/base.yaml" \ - "$gitops_dir/manifests/nebari-operator/deployment-patch.yaml" - - git -C "$gitops_dir" -c user.email="ci@example.com" -c user.name="k8s-preview-ci" \ - commit -am "Point Keycloak hostname at the public preview tunnel route" - - kubectl -n argocd annotate application/keycloak application/nebari-operator \ - argocd.argoproj.io/refresh=hard --overwrite - - operator_deploy=$(kubectl get deploy -A -o json | jq -r '.items[] | select(.metadata.name | test("operator")) | "\(.metadata.namespace)/\(.metadata.name)"' | head -1) - - # `rollout status` alone can return instantly if ArgoCD hasn't - # actually applied the refreshed spec yet (nothing new requested - # from kubectl's point of view) -- poll for the live env var to - # actually change before trusting rollout status to mean anything. - for i in $(seq 1 24); do - kc_live=$(kubectl -n keycloak get statefulset keycloak-keycloakx -o jsonpath='{.spec.template.spec.containers[0].env}' | jq -r '.[] | select(.name=="KC_HOSTNAME") | .value') - op_live=$(kubectl -n "${operator_deploy%%/*}" get deploy "${operator_deploy##*/}" -o jsonpath='{.spec.template.spec.containers[0].env}' | jq -r '.[] | select(.name=="KEYCLOAK_EXTERNAL_URL") | .value') - if [ "$kc_live" = "$kc_public_url" ] && [ "$op_live" = "$kc_public_url" ]; then - echo "ArgoCD applied the GitOps change after $(( i * 5 ))s" - break - fi - sleep 5 - done - if [ "$kc_live" != "$kc_public_url" ] || [ "$op_live" != "$kc_public_url" ]; then - echo "::error::ArgoCD never applied the GitOps hostname change within 2m (keycloak=$kc_live, operator=$op_live)" - exit 1 - fi - - kubectl -n keycloak rollout status statefulset/keycloak-keycloakx --timeout=180s - kubectl -n "${operator_deploy%%/*}" rollout status deployment/"${operator_deploy##*/}" --timeout=180s + python -m scripts.preview.keycloak_gitops patch \ + --gitops-dir "$HOME/.nic/gitops/${{ steps.sandbox.outputs.cluster-name }}" \ + --kc-public-url "https://keycloak-pr-${{ github.event.pull_request.number }}-data-science-pack.${{ env.PREVIEW_DOMAIN }}" # No --wait here: with nebariapp.auth.enabled=true, the hub pod reads # the operator-provisioned Keycloak client Secret at import time @@ -334,45 +279,24 @@ jobs: # restart triggered right after the Secret's first appearance still # hits FileNotFoundError on /etc/oauth/issuer-url specifically -- wait # for that key's actual (non-empty) value, not just the object. + # Why a mere Secret-exists check or a single restart isn't reliable + # here (the operator populates issuer-url on a later reconcile pass; + # kubelet's Secret volume cache can hand a fresh pod a stale + # snapshot) is explained in scripts/preview/k8s_wait.py. - name: Wait for operator to provision the Keycloak client secret env: KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} run: | - for i in $(seq 1 36); do - issuer_b64=$(kubectl -n pr-preview get secret preview-nebari-data-science-pack-oidc-client -o jsonpath='{.data.issuer-url}' 2>/dev/null) - if [ -n "$issuer_b64" ]; then - echo "operator secret's issuer-url populated after $(( i * 5 ))s" - exit 0 - fi - sleep 5 - done - echo "::error::operator never populated issuer-url on the Keycloak client secret within 3m" - kubectl -n pr-preview get nebariapp -o yaml || true - exit 1 - - # A single restart isn't reliable here even though the API server - # confirms issuer-url is populated: kubelet's own Secret volume cache - # (node-local, ~1min TTL) can still hand a freshly-restarted pod the - # pre-population snapshot it fetched for the pod's first, crash-looped - # attempt -- confirmed by the same FileNotFoundError recurring on a - # restart issued 5s after the secret was already confirmed complete. - # Retry the restart until a rollout actually succeeds, giving the - # kubelet cache time to expire between attempts, instead of assuming - # one restart is enough. + python -m scripts.preview.k8s_wait wait-for-secret-key \ + --namespace pr-preview --secret preview-nebari-data-science-pack-oidc-client \ + --key issuer-url --timeout-s 180 --poll-interval-s 5 + - name: Restart hub until it picks up the operator secret env: KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} run: | - for attempt in 1 2 3 4 5; do - kubectl -n pr-preview rollout restart deployment/hub - if kubectl -n pr-preview rollout status deployment/hub --timeout=90s; then - echo "hub ready on attempt $attempt" - exit 0 - fi - echo "hub not ready on attempt $attempt, retrying..." - done - echo "::error::hub never became ready after 5 restart attempts" - exit 1 + python -m scripts.preview.k8s_wait restart-until-ready \ + --namespace pr-preview --deployment hub --rollout-timeout-s 90 --max-attempts 5 - name: Wait for proxy env: @@ -413,18 +337,9 @@ jobs: env: KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} run: | - admin_token=$(curl -sS -X POST "http://localhost:8001/realms/master/protocol/openid-connect/token" \ - -d "grant_type=password" -d "client_id=admin-cli" -d "username=admin" \ - --data-urlencode "password=$(kubectl -n keycloak get secret keycloak-admin-credentials -o jsonpath='{.data.admin-password}' | base64 -d)" \ - | jq -r '.access_token') - if [ -z "$admin_token" ] || [ "$admin_token" = "null" ]; then - echo "::error::Failed to obtain a Keycloak admin token" - exit 1 - fi - - curl -sS -o /dev/null -w 'create user: HTTP %{http_code}\n' -X POST "http://localhost:8001/admin/realms/nebari/users" \ - -H "Authorization: Bearer $admin_token" -H "Content-Type: application/json" \ - -d '{"username":"reviewer","enabled":true,"email":"reviewer@example.com","emailVerified":true,"firstName":"Preview","lastName":"Reviewer","credentials":[{"type":"password","value":"admin","temporary":false}]}' + admin_password=$(kubectl -n keycloak get secret keycloak-admin-credentials -o jsonpath='{.data.admin-password}' | base64 -d) + python -m scripts.preview.keycloak create-reviewer-user \ + --base-url "http://localhost:8001" --realm nebari --admin-password "$admin_password" # jhub-apps runs as a managed service subprocess inside the hub pod, # not its own Deployment, so a crash there doesn't fail `helm --wait` diff --git a/scripts/preview/http.py b/scripts/preview/http.py index 821d4c0..23aa05f 100644 --- a/scripts/preview/http.py +++ b/scripts/preview/http.py @@ -33,13 +33,16 @@ def request_json( ) -> Any: """Send an HTTP request, returning the parsed JSON response body. - ``body``, if given, is JSON-encoded and sent with an - application/json Content-Type. Returns ``{}`` for an empty (e.g. 204) - response body. + ``body``, if a ``str``, is sent as-is (the caller sets its own + Content-Type, e.g. for a form-urlencoded Keycloak token request); any + other value is JSON-encoded with an application/json Content-Type. + Returns ``{}`` for an empty (e.g. 204) response body. """ data = None req_headers = dict(headers or {}) - if body is not None: + if isinstance(body, str): + data = body.encode("utf-8") + elif body is not None: data = json.dumps(body).encode("utf-8") req_headers.setdefault("Content-Type", "application/json") diff --git a/scripts/preview/k8s_wait.py b/scripts/preview/k8s_wait.py new file mode 100644 index 0000000..e77cf42 --- /dev/null +++ b/scripts/preview/k8s_wait.py @@ -0,0 +1,132 @@ +"""kubectl readiness retry loops for the k8s preview environment. + +Usage: + python -m scripts.preview.k8s_wait wait-for-secret-key --namespace NS \\ + --secret NAME --key KEY [--timeout-s 180] [--poll-interval-s 5] + python -m scripts.preview.k8s_wait restart-until-ready --namespace NS \\ + --deployment NAME [--rollout-timeout-s 90] [--max-attempts 5] +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +import time +from collections.abc import Callable + +from . import gha + + +def wait_for_secret_key( + get_value: Callable[[], str], + timeout_s: int = 180, + poll_interval_s: int = 5, + clock: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, +) -> bool: + """Poll ``get_value()`` until it returns a non-empty string, or timeout. + + The operator creates the Secret with client-id/client-secret on its + first reconcile pass, then patches in the issuer-url key on a later + pass -- checking the Secret merely EXISTS isn't enough, a hub restart + right after its first appearance still crashes on the not-yet-populated + key. + """ + deadline = clock() + timeout_s + while clock() < deadline: + if get_value(): + return True + sleep(poll_interval_s) + return False + + +def restart_until_ready( + restart: Callable[[], None], + check_status: Callable[[], bool], + max_attempts: int = 5, +) -> tuple[bool, int]: + """Restart, then check readiness; repeat until ready or ``max_attempts``. + + A single restart isn't reliable here even when the API server confirms + the Secret is fully populated: kubelet's own Secret volume cache + (node-local, ~1min TTL) can still hand a freshly-restarted pod the + pre-population snapshot it fetched for the pod's first, crash-looped + attempt. Retrying gives the kubelet cache time to expire between + attempts. + + Returns (became_ready, attempts_used). + """ + for attempt in range(1, max_attempts + 1): + restart() + if check_status(): + return True, attempt + return False, max_attempts + + +def _cmd_wait_for_secret_key(args: argparse.Namespace) -> int: + def get_value() -> str: + result = subprocess.run( + ["kubectl", "-n", args.namespace, "get", "secret", args.secret, + "-o", f"jsonpath={{.data.{args.key}}}"], + text=True, capture_output=True, check=False, + ) + return result.stdout.strip() if result.returncode == 0 else "" + + ok = wait_for_secret_key(get_value, timeout_s=args.timeout_s, poll_interval_s=args.poll_interval_s) + if not ok: + gha.error( + f"operator never populated {args.key!r} on secret {args.secret!r} " + f"within {args.timeout_s}s" + ) + subprocess.run(["kubectl", "-n", args.namespace, "get", "nebariapp", "-o", "yaml"], check=False) + return 1 + return 0 + + +def _cmd_restart_until_ready(args: argparse.Namespace) -> int: + target = f"deployment/{args.deployment}" + + def restart() -> None: + subprocess.run(["kubectl", "-n", args.namespace, "rollout", "restart", target], check=False) + + def check_status() -> bool: + result = subprocess.run([ + "kubectl", "-n", args.namespace, "rollout", "status", target, + f"--timeout={args.rollout_timeout_s}s", + ], check=False) + return result.returncode == 0 + + ok, attempts = restart_until_ready(restart, check_status, max_attempts=args.max_attempts) + if ok: + print(f"{args.deployment} ready on attempt {attempts}") + return 0 + gha.error(f"{args.deployment} never became ready after {args.max_attempts} restart attempts") + return 1 + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(prog="k8s_wait") + sub = parser.add_subparsers(dest="command", required=True) + + p = sub.add_parser("wait-for-secret-key") + p.add_argument("--namespace", required=True) + p.add_argument("--secret", required=True) + p.add_argument("--key", required=True) + p.add_argument("--timeout-s", type=int, default=180) + p.add_argument("--poll-interval-s", type=int, default=5) + p.set_defaults(func=_cmd_wait_for_secret_key) + + p = sub.add_parser("restart-until-ready") + p.add_argument("--namespace", required=True) + p.add_argument("--deployment", required=True) + p.add_argument("--rollout-timeout-s", type=int, default=90) + p.add_argument("--max-attempts", type=int, default=5) + p.set_defaults(func=_cmd_restart_until_ready) + + args = parser.parse_args(argv[1:]) + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/scripts/preview/keycloak.py b/scripts/preview/keycloak.py new file mode 100644 index 0000000..bb4d6ec --- /dev/null +++ b/scripts/preview/keycloak.py @@ -0,0 +1,90 @@ +"""Keycloak admin API calls for the k8s preview environment's test user. + +Cloudflare Access is the real security boundary for this preview (only +allow-listed accounts reach the tunnel at all), so a simple, known password +for the Keycloak-side login is fine -- reviewers don't need to hunt for real +credentials on a throwaway cluster. + +Usage: + python -m scripts.preview.keycloak create-reviewer-user --base-url URL \\ + --realm REALM --admin-password PASSWORD +""" + +from __future__ import annotations + +import argparse +import sys +from urllib.parse import urlencode + +from . import gha +from .http import request_json + + +class KeycloakAuthError(RuntimeError): + pass + + +def get_admin_token(base_url: str, admin_password: str) -> str: + body = urlencode( + { + "grant_type": "password", + "client_id": "admin-cli", + "username": "admin", + "password": admin_password, + } + ) + result = request_json( + "POST", + f"{base_url}/realms/master/protocol/openid-connect/token", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + body=body, + ) + token = result.get("access_token") + if not token: + raise KeycloakAuthError(f"no access_token in Keycloak admin token response: {result}") + return token + + +def create_reviewer_user(base_url: str, realm: str, admin_token: str) -> None: + request_json( + "POST", + f"{base_url}/admin/realms/{realm}/users", + headers={"Authorization": f"Bearer {admin_token}"}, + body={ + "username": "reviewer", + "enabled": True, + "email": "reviewer@example.com", + "emailVerified": True, + "firstName": "Preview", + "lastName": "Reviewer", + "credentials": [{"type": "password", "value": "admin", "temporary": False}], + }, + ) + + +def _cmd_create_reviewer_user(args: argparse.Namespace) -> int: + try: + token = get_admin_token(args.base_url, args.admin_password) + create_reviewer_user(args.base_url, args.realm, token) + except Exception as exc: # noqa: BLE001 - report and fail the step either way + gha.error(f"Failed to create the Keycloak reviewer user: {exc}") + return 1 + return 0 + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(prog="keycloak") + sub = parser.add_subparsers(dest="command", required=True) + + p = sub.add_parser("create-reviewer-user") + p.add_argument("--base-url", required=True) + p.add_argument("--realm", required=True) + p.add_argument("--admin-password", required=True) + p.set_defaults(func=_cmd_create_reviewer_user) + + args = parser.parse_args(argv[1:]) + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/scripts/preview/keycloak_gitops.py b/scripts/preview/keycloak_gitops.py new file mode 100644 index 0000000..28bdb16 --- /dev/null +++ b/scripts/preview/keycloak_gitops.py @@ -0,0 +1,162 @@ +"""Repoint Keycloak's own hostname via NIC's local GitOps repo. + +Keycloak's own KC_HOSTNAME (codecentric/keycloakx, via NIC's +values/keycloak/base.yaml) is fixed to NIC's internal keycloak.nebari.local +by default -- every self-referencing URL Keycloak renders (login form +action, issuer, redirects) is absolute and uses that value regardless of +the incoming Host header. + +A direct `kubectl set env` patch onto the live StatefulSet/Deployment DOES +apply immediately but gets silently reverted: both Keycloak and +nebari-operator are ArgoCD Applications with selfHeal: true, continuously +reconciled against NIC's auto-created local GitOps repo +(~/.nic/gitops/). Editing the GitOps repo itself and forcing +a hard refresh lets selfHeal work for this change instead of against it. + +Usage: + python -m scripts.preview.keycloak_gitops patch \\ + --gitops-dir DIR --kc-public-url URL +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +import time +from collections.abc import Callable +from pathlib import Path + +from . import gha + +OLD_HOSTNAME_URL = "https://keycloak.nebari.local" +GITOPS_FILES = ("values/keycloak/base.yaml", "manifests/nebari-operator/deployment-patch.yaml") + + +class OperatorNotFoundError(RuntimeError): + pass + + +def rewrite_hostname(text: str, old_url: str, new_url: str) -> str: + """Replace every literal occurrence of ``old_url`` with ``new_url``.""" + return text.replace(old_url, new_url) + + +def find_operator_deployment(deployments_json: dict) -> tuple[str, str]: + """Return (namespace, name) of the deployment whose name contains "operator".""" + for item in deployments_json.get("items", []): + name = item["metadata"]["name"] + if re.search("operator", name): + return item["metadata"]["namespace"], name + raise OperatorNotFoundError("no deployment with 'operator' in its name was found") + + +def extract_env_value(env: list[dict], var_name: str) -> str: + """Return the value of ``var_name`` in a container env list, or "" if absent.""" + for entry in env: + if entry.get("name") == var_name: + return entry.get("value", "") + return "" + + +def wait_until_both_match( + get_values: Callable[[], tuple[str, str]], + expected: str, + timeout_s: int = 120, + poll_interval_s: int = 5, + clock: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, +) -> tuple[bool, tuple[str, str]]: + """Poll ``get_values()`` until both returned values equal ``expected``. + + `rollout status` alone can return instantly if ArgoCD hasn't actually + applied the refreshed spec yet (nothing new requested from kubectl's + point of view) -- poll for the live values to actually change before + trusting rollout status to mean anything. + """ + deadline = clock() + timeout_s + values = ("", "") + while clock() < deadline: + values = get_values() + if values[0] == expected and values[1] == expected: + return True, values + sleep(poll_interval_s) + return False, values + + +def _run(cmd: list[str]) -> subprocess.CompletedProcess: + return subprocess.run(cmd, check=True, text=True, capture_output=True) + + +def _cmd_patch(args: argparse.Namespace) -> int: + gitops_dir = Path(args.gitops_dir) + + for relative_path in GITOPS_FILES: + path = gitops_dir / relative_path + original = path.read_text() + print(f"--- {relative_path}: {original.count(OLD_HOSTNAME_URL)} occurrence(s) of {OLD_HOSTNAME_URL} ---") + path.write_text(rewrite_hostname(original, OLD_HOSTNAME_URL, args.kc_public_url)) + + _run([ + "git", "-C", str(gitops_dir), + "-c", "user.email=ci@example.com", "-c", "user.name=k8s-preview-ci", + "commit", "-am", "Point Keycloak hostname at the public preview tunnel route", + ]) + + _run([ + "kubectl", "-n", "argocd", "annotate", + "application/keycloak", "application/nebari-operator", + "argocd.argoproj.io/refresh=hard", "--overwrite", + ]) + + deployments = json.loads(_run(["kubectl", "get", "deploy", "-A", "-o", "json"]).stdout) + try: + operator_namespace, operator_name = find_operator_deployment(deployments) + except OperatorNotFoundError as exc: + gha.error(str(exc)) + return 1 + + def get_values() -> tuple[str, str]: + kc_env = json.loads(_run([ + "kubectl", "-n", "keycloak", "get", "statefulset", "keycloak-keycloakx", + "-o", "jsonpath={.spec.template.spec.containers[0].env}", + ]).stdout or "[]") + op_env = json.loads(_run([ + "kubectl", "-n", operator_namespace, "get", "deploy", operator_name, + "-o", "jsonpath={.spec.template.spec.containers[0].env}", + ]).stdout or "[]") + return ( + extract_env_value(kc_env, "KC_HOSTNAME"), + extract_env_value(op_env, "KEYCLOAK_EXTERNAL_URL"), + ) + + matched, (kc_live, op_live) = wait_until_both_match(get_values, args.kc_public_url) + if not matched: + gha.error( + "ArgoCD never applied the GitOps hostname change within 2m " + f"(keycloak={kc_live}, operator={op_live})" + ) + return 1 + + _run(["kubectl", "-n", "keycloak", "rollout", "status", "statefulset/keycloak-keycloakx", "--timeout=180s"]) + _run(["kubectl", "-n", operator_namespace, "rollout", "status", f"deployment/{operator_name}", "--timeout=180s"]) + return 0 + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(prog="keycloak_gitops") + sub = parser.add_subparsers(dest="command", required=True) + + p = sub.add_parser("patch") + p.add_argument("--gitops-dir", required=True) + p.add_argument("--kc-public-url", required=True) + p.set_defaults(func=_cmd_patch) + + args = parser.parse_args(argv[1:]) + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/tests/unit/test_preview_http.py b/tests/unit/test_preview_http.py index eb41e92..8f09e02 100644 --- a/tests/unit/test_preview_http.py +++ b/tests/unit/test_preview_http.py @@ -59,6 +59,28 @@ def fake_urlopen(request, timeout): assert captured["content_type"] == "application/json" +def test_string_body_is_sent_as_is_not_json_encoded(monkeypatch): + captured = {} + + def fake_urlopen(request, timeout): + captured["data"] = request.data + captured["content_type"] = request.get_header("Content-type") + return _FakeResponse(b"{}") + + monkeypatch.setattr("scripts.preview.http.urllib.request.urlopen", fake_urlopen) + + request_json( + "POST", + "https://api.example.com/token", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + body="grant_type=password&username=admin", + ) + + assert captured["data"] == b"grant_type=password&username=admin" + # Content-Type came from the caller's headers, not auto-set to json. + assert captured["content_type"] == "application/x-www-form-urlencoded" + + def test_custom_headers_are_forwarded(monkeypatch): captured = {} diff --git a/tests/unit/test_preview_k8s_wait.py b/tests/unit/test_preview_k8s_wait.py new file mode 100644 index 0000000..735c142 --- /dev/null +++ b/tests/unit/test_preview_k8s_wait.py @@ -0,0 +1,79 @@ +"""Tests for scripts/preview/k8s_wait.py's kubectl readiness retry loops.""" + +from __future__ import annotations + +from scripts.preview import k8s_wait + + +def _fake_time(): + state = {"t": 0.0} + return (lambda: state["t"]), (lambda s: state.__setitem__("t", state["t"] + s)) + + +# --- wait_for_secret_key ------------------------------------------------- + + +def test_wait_for_secret_key_returns_true_once_value_nonempty(): + calls = [] + + def get_value(): + calls.append(1) + return "" if len(calls) == 1 else "aGVsbG8=" + + clock, sleep = _fake_time() + + ok = k8s_wait.wait_for_secret_key(get_value, timeout_s=300, poll_interval_s=5, clock=clock, sleep=sleep) + + assert ok is True + assert len(calls) == 2 + + +def test_wait_for_secret_key_returns_false_after_timeout(): + clock, sleep = _fake_time() + + ok = k8s_wait.wait_for_secret_key(lambda: "", timeout_s=10, poll_interval_s=5, clock=clock, sleep=sleep) + + assert ok is False + + +# --- restart_until_ready --------------------------------------------------- + + +def test_restart_until_ready_stops_on_first_successful_attempt(): + restart_calls = [] + status_calls = [] + + ok, attempts = k8s_wait.restart_until_ready( + restart=lambda: restart_calls.append(1), + check_status=lambda: (status_calls.append(1), True)[1], + max_attempts=5, + ) + + assert ok is True + assert attempts == 1 + assert len(restart_calls) == 1 + assert len(status_calls) == 1 + + +def test_restart_until_ready_retries_until_success(): + status_results = iter([False, False, True]) + + ok, attempts = k8s_wait.restart_until_ready( + restart=lambda: None, + check_status=lambda: next(status_results), + max_attempts=5, + ) + + assert ok is True + assert attempts == 3 + + +def test_restart_until_ready_gives_up_after_max_attempts(): + ok, attempts = k8s_wait.restart_until_ready( + restart=lambda: None, + check_status=lambda: False, + max_attempts=3, + ) + + assert ok is False + assert attempts == 3 diff --git a/tests/unit/test_preview_keycloak.py b/tests/unit/test_preview_keycloak.py new file mode 100644 index 0000000..a9a7e71 --- /dev/null +++ b/tests/unit/test_preview_keycloak.py @@ -0,0 +1,82 @@ +"""Tests for scripts/preview/keycloak.py's admin API helpers.""" + +from __future__ import annotations + +import pytest + +from scripts.preview import keycloak +from scripts.preview.http import HTTPRequestError + +BASE_URL = "http://localhost:8001" + + +def _capture(monkeypatch): + calls = [] + + def fake_request_json(method, url, headers=None, body=None, timeout=15): + calls.append({"method": method, "url": url, "headers": headers, "body": body}) + return fake_request_json.next_result + + fake_request_json.next_result = {} + monkeypatch.setattr("scripts.preview.keycloak.request_json", fake_request_json) + return calls, fake_request_json + + +def test_get_admin_token_sends_password_grant_as_form_body(monkeypatch): + calls, fake = _capture(monkeypatch) + fake.next_result = {"access_token": "the-token"} + + token = keycloak.get_admin_token(BASE_URL, "adminpw") + + assert token == "the-token" + assert calls[0]["url"] == f"{BASE_URL}/realms/master/protocol/openid-connect/token" + assert calls[0]["headers"]["Content-Type"] == "application/x-www-form-urlencoded" + body = calls[0]["body"] + assert "grant_type=password" in body + assert "client_id=admin-cli" in body + assert "username=admin" in body + assert "password=adminpw" in body + + +def test_get_admin_token_url_encodes_special_characters_in_password(monkeypatch): + calls, fake = _capture(monkeypatch) + fake.next_result = {"access_token": "tok"} + + keycloak.get_admin_token(BASE_URL, "p@ss w/ord&x") + + assert "password=p%40ss+w%2Ford%26x" in calls[0]["body"] + + +def test_get_admin_token_raises_when_response_has_no_access_token(monkeypatch): + _, fake = _capture(monkeypatch) + fake.next_result = {"error": "invalid_grant"} + + with pytest.raises(keycloak.KeycloakAuthError): + keycloak.get_admin_token(BASE_URL, "wrong") + + +def test_get_admin_token_propagates_http_errors(monkeypatch): + def fake_request_json(method, url, headers=None, body=None, timeout=15): + raise HTTPRequestError(method, url, 401, "unauthorized") + + monkeypatch.setattr("scripts.preview.keycloak.request_json", fake_request_json) + + with pytest.raises(HTTPRequestError): + keycloak.get_admin_token(BASE_URL, "wrong") + + +def test_create_reviewer_user_posts_expected_payload(monkeypatch): + calls, _ = _capture(monkeypatch) + + keycloak.create_reviewer_user(BASE_URL, "nebari", "admin-token") + + assert calls[0]["url"] == f"{BASE_URL}/admin/realms/nebari/users" + assert calls[0]["headers"]["Authorization"] == "Bearer admin-token" + body = calls[0]["body"] + assert body["username"] == "reviewer" + assert body["enabled"] is True + assert body["email"] == "reviewer@example.com" + assert body["emailVerified"] is True + assert body["firstName"] == "Preview" + assert body["lastName"] == "Reviewer" + assert body["credentials"] == [{"type": "password", "value": "admin", "temporary": False}] diff --git a/tests/unit/test_preview_keycloak_gitops.py b/tests/unit/test_preview_keycloak_gitops.py new file mode 100644 index 0000000..bf6ec39 --- /dev/null +++ b/tests/unit/test_preview_keycloak_gitops.py @@ -0,0 +1,120 @@ +"""Tests for scripts/preview/keycloak_gitops.py. + +Covers the GitOps hostname rewrite for Keycloak's own KC_HOSTNAME -- see +that module's docstring for why this can't just be a live kubectl patch +(ArgoCD selfHeal reverts it). +""" + +from __future__ import annotations + +from scripts.preview import keycloak_gitops as kg + +# --- rewrite_hostname (pure) -------------------------------------------------- + + +def test_rewrite_hostname_replaces_every_occurrence(): + text = ( + "issuerUrl: https://keycloak.nebari.local/realms/nebari\n" + "redirectUri: https://keycloak.nebari.local/callback\n" + ) + + result = kg.rewrite_hostname(text, "https://keycloak.nebari.local", "https://kc-pr-205.example.com") + + assert "keycloak.nebari.local" not in result + assert result.count("https://kc-pr-205.example.com") == 2 + + +def test_rewrite_hostname_leaves_unrelated_text_untouched(): + text = "some_other_key: value\n" + + result = kg.rewrite_hostname(text, "https://keycloak.nebari.local", "https://kc-pr-205.example.com") + + assert result == text + + +# --- find_operator_deployment (pure) ------------------------------------------ + + +def test_find_operator_deployment_matches_name_containing_operator(): + deployments = { + "items": [ + {"metadata": {"namespace": "kube-system", "name": "coredns"}}, + {"metadata": {"namespace": "nic-system", "name": "nebari-operator-controller"}}, + ] + } + + namespace, name = kg.find_operator_deployment(deployments) + + assert namespace == "nic-system" + assert name == "nebari-operator-controller" + + +def test_find_operator_deployment_raises_when_none_found(): + deployments = {"items": [{"metadata": {"namespace": "kube-system", "name": "coredns"}}]} + + try: + kg.find_operator_deployment(deployments) + raise AssertionError("expected OperatorNotFoundError") + except kg.OperatorNotFoundError: + pass + + +# --- extract_env_value (pure) ------------------------------------------------- + + +def test_extract_env_value_finds_named_var(): + env = [{"name": "OTHER", "value": "x"}, {"name": "KC_HOSTNAME", "value": "https://kc.example.com"}] + + assert kg.extract_env_value(env, "KC_HOSTNAME") == "https://kc.example.com" + + +def test_extract_env_value_returns_empty_string_when_absent(): + assert kg.extract_env_value([{"name": "OTHER", "value": "x"}], "KC_HOSTNAME") == "" + + +# --- wait_until_both_match (pure, time/sleep injected) ------------------------ + + +def test_wait_until_both_match_returns_true_as_soon_as_both_equal_expected(): + calls = [] + + def get_values(): + calls.append(1) + # Wrong on the first call, correct on the second. + return ("wrong", "wrong") if len(calls) == 1 else ("expected", "expected") + + fake_time = {"t": 0.0} + matched, values = kg.wait_until_both_match( + get_values, "expected", timeout_s=60, poll_interval_s=5, + clock=lambda: fake_time["t"], + sleep=lambda s: fake_time.__setitem__("t", fake_time["t"] + s), + ) + + assert matched is True + assert values == ("expected", "expected") + assert len(calls) == 2 + + +def test_wait_until_both_match_returns_false_after_timeout(): + fake_time = {"t": 0.0} + + matched, values = kg.wait_until_both_match( + lambda: ("wrong", "wrong"), "expected", timeout_s=20, poll_interval_s=5, + clock=lambda: fake_time["t"], + sleep=lambda s: fake_time.__setitem__("t", fake_time["t"] + s), + ) + + assert matched is False + assert values == ("wrong", "wrong") + + +def test_wait_until_both_match_false_when_only_one_side_matches(): + fake_time = {"t": 0.0} + + matched, _ = kg.wait_until_both_match( + lambda: ("expected", "wrong"), "expected", timeout_s=1, poll_interval_s=5, + clock=lambda: fake_time["t"], + sleep=lambda s: fake_time.__setitem__("t", fake_time["t"] + s), + ) + + assert matched is False From 037c993a18e3056f9f2644f6aad80b163df567d3 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Wed, 2 Sep 2026 15:53:39 +0100 Subject: [PATCH 52/54] refactor: move remaining GitHub API/comment logic into scripts/preview/ Rounds out the k8s-preview.yaml extraction: comment.py builds the three PR-comment bodies (ready/expired/stopped) as pure, tested functions instead of inline message: blocks -- the sticky-comment action still owns the actual find-or-edit-in-place mechanics, only the content moved. github_api.py gains a CLI covering the remaining bare `gh api`/`gh label`/`gh run cancel` calls: ensure-label-exists, create-and-activate (deployment + success status), mark-inactive, mark-latest-inactive, and cancel-in-flight-run. cleanup-preview didn't have a Checkout step before (never needed one for raw gh api calls); adds one so it can invoke these scripts too. --- .github/workflows/k8s-preview.yaml | 121 +++++++++++-------------- scripts/preview/comment.py | 122 ++++++++++++++++++++++++++ scripts/preview/github_api.py | 109 +++++++++++++++++++++++ tests/unit/test_preview_comment.py | 122 ++++++++++++++++++++++++++ tests/unit/test_preview_github_api.py | 104 ++++++++++++++++++++++ 5 files changed, 508 insertions(+), 70 deletions(-) create mode 100644 scripts/preview/comment.py create mode 100644 tests/unit/test_preview_comment.py diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index bc5c3db..c51d5a9 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -139,13 +139,11 @@ jobs: # `gh label create` errors if the label already exists; that's fine, # this step is just making sure it's there. - name: Ensure the extend-preview label exists - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - gh label create extend-preview --repo "${{ github.repository }}" \ - --color BFD4F2 \ - --description "Push this preview's expiry back 20 minutes" \ - 2>/dev/null || true + python -m scripts.preview.github_api ensure-label-exists \ + --repo "${{ github.repository }}" --token "${{ secrets.GITHUB_TOKEN }}" \ + --name extend-preview --color BFD4F2 \ + --description "Push this preview's expiry back 20 minutes" # v0.32.0+ required: the sandbox's kind cluster uses containerd's # config v4 format, which `kind load` on older CLI releases can't @@ -484,30 +482,26 @@ jobs: # status call) marks any prior deployment to this same per-PR # environment inactive, so redeploys don't leave stale green boxes. - name: Create GitHub deployment - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - deployment_json=$(jq -n \ - --arg ref "${{ github.event.pull_request.head.sha }}" \ - --arg environment "pr-${{ github.event.pull_request.number }}-preview" \ - '{ref: $ref, environment: $environment, task: "deploy:preview", - auto_merge: false, transient_environment: true, - production_environment: false, required_contexts: [], - description: "K8s stack preview"}') - deployment_id=$(gh api "repos/${{ github.repository }}/deployments" \ - -X POST --input - --jq '.id' <<< "$deployment_json") - if [ -z "$deployment_id" ] || [ "$deployment_id" = "null" ]; then - echo "::error::Failed to create GitHub deployment" - exit 1 - fi - echo "DEPLOYMENT_ID=${deployment_id}" >> "$GITHUB_ENV" - - gh api "repos/${{ github.repository }}/deployments/${deployment_id}/statuses" \ - -X POST \ - -f state=success \ - -f environment_url="${{ steps.cf_dns.outputs.url }}" \ - -f log_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ - -f description="Live for 20 minutes" > /dev/null + python -m scripts.preview.github_api create-and-activate \ + --repo "${{ github.repository }}" --token "${{ secrets.GITHUB_TOKEN }}" \ + --ref "${{ github.event.pull_request.head.sha }}" \ + --environment "pr-${{ github.event.pull_request.number }}-preview" \ + --task "deploy:preview" --description "K8s stack preview" \ + --environment-url "${{ steps.cf_dns.outputs.url }}" \ + --log-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ + --status-description "Live for 20 minutes" + + - name: Render the ready-preview comment body + id: render_ready + run: | + python -m scripts.preview.comment render-ready \ + --url "${{ steps.cf_dns.outputs.url }}" --keycloak-url "${{ steps.cf_dns.outputs.keycloak_url }}" \ + --deployed-at "${{ steps.timestamps.outputs.deployed_at }}" \ + --deployed-at-iso "${{ steps.timestamps.outputs.deployed_at_iso }}" \ + --expires-at "${{ steps.timestamps.outputs.expires_at }}" \ + --expires-at-iso "${{ steps.timestamps.outputs.expires_at_iso }}" \ + ${{ github.event.pull_request.head.repo.fork && '--fork' || '' }} - name: Comment preview link on PR id: comment_preview @@ -515,16 +509,7 @@ jobs: with: header: k8s-preview GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - message: | - The latest K8s stack preview for this PR. - - | Project | Deployment | Actions | Updated | - | --- | --- | --- | --- | - | `nebari-data-science-pack` | 🟢 [Ready](${{ steps.cf_dns.outputs.url }}) | [Preview](${{ steps.cf_dns.outputs.url }}) · [Keycloak](${{ steps.cf_dns.outputs.keycloak_url }}) | ${{ steps.timestamps.outputs.deployed_at }} | - - ${{ github.event.pull_request.head.repo.fork && '⚠️ **This PR is from a fork**: the code running in this preview is not from a trusted maintainer branch.' || '' }} - - Expires ${{ steps.timestamps.outputs.expires_at }}. Add the `extend-preview` label any time before then for 20 more minutes, or push a new commit or re-add `deploy-preview` to redeploy from scratch. + message: ${{ steps.render_ready.outputs.body }} # Bounded to 20min by default so the live preview doesn't sit open # (and burn CI minutes) indefinitely, and so this step ends itself @@ -594,12 +579,10 @@ jobs: # showing green with a link that no longer resolves. - name: Mark GitHub deployment inactive if: always() - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - [ -n "${DEPLOYMENT_ID:-}" ] || exit 0 - gh api "repos/${{ github.repository }}/deployments/${DEPLOYMENT_ID}/statuses" \ - -X POST -f state=inactive -f description="Preview expired" > /dev/null || true + python -m scripts.preview.github_api mark-inactive \ + --repo "${{ github.repository }}" --token "${{ secrets.GITHUB_TOKEN }}" \ + --deployment-id "${DEPLOYMENT_ID:-}" --description "Preview expired" # The comment above still says "Ready" / "Expires in N minutes" once # the tunnel closes -- it was only ever written once, at deploy time, @@ -610,20 +593,21 @@ jobs: # when the first post succeeded (skip if the run failed before ever # reaching that step, since there is no live-preview comment to # correct in that case). + - name: Render the expired-preview comment body + id: render_expired + if: always() && steps.comment_preview.outcome == 'success' + run: | + python -m scripts.preview.comment render-expired \ + --expires-at "${{ steps.timestamps.outputs.expires_at }}" \ + --expires-at-iso "${{ steps.timestamps.outputs.expires_at_iso }}" + - name: Comment that the preview expired if: always() && steps.comment_preview.outcome == 'success' uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 with: header: k8s-preview GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - message: | - The K8s stack preview for this PR has expired. - - | Project | Deployment | Actions | Updated | - | --- | --- | --- | --- | - | `nebari-data-science-pack` | ⚫ Expired | - | ${{ steps.timestamps.outputs.expires_at }} | - - Push a new commit or re-add the `deploy-preview` label to redeploy. + message: ${{ steps.render_expired.outputs.body }} - name: Delete DNS record if: always() @@ -652,32 +636,29 @@ jobs: actions: write deployments: write steps: + - name: Checkout + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Cancel the in-flight preview run for this PR - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - run_id=$(gh api "repos/${{ github.repository }}/actions/runs?event=pull_request&status=in_progress" \ - --jq '.workflow_runs[] | select(.name == "K8s Stack Preview") | select(.pull_requests[]?.number == ${{ github.event.pull_request.number }}) | .id' \ - | head -1) - if [ -n "$run_id" ]; then - gh run cancel "$run_id" --repo "${{ github.repository }}" - fi + python -m scripts.preview.github_api cancel-in-flight-run \ + --repo "${{ github.repository }}" --token "${{ secrets.GITHUB_TOKEN }}" \ + --workflow-name "K8s Stack Preview" --pr "${{ github.event.pull_request.number }}" - name: Mark GitHub deployment inactive - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - deployment_id=$(gh api "repos/${{ github.repository }}/deployments?environment=pr-${{ github.event.pull_request.number }}-preview&per_page=1" --jq '.[0].id' 2>/dev/null || true) - [ -n "$deployment_id" ] && [ "$deployment_id" != "null" ] || exit 0 - gh api "repos/${{ github.repository }}/deployments/${deployment_id}/statuses" \ - -X POST -f state=inactive -f description="Preview stopped (label removed)" > /dev/null || true + python -m scripts.preview.github_api mark-latest-inactive \ + --repo "${{ github.repository }}" --token "${{ secrets.GITHUB_TOKEN }}" \ + --environment "pr-${{ github.event.pull_request.number }}-preview" \ + --description "Preview stopped (label removed)" + + - name: Render the stopped-preview comment body + id: render_stopped + run: python -m scripts.preview.comment render-stopped - name: Comment that the preview stopped uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 with: header: k8s-preview GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - message: | - **K8s stack preview** stopped: the `deploy-preview` label was removed. - - Add it again to redeploy. + message: ${{ steps.render_stopped.outputs.body }} diff --git a/scripts/preview/comment.py b/scripts/preview/comment.py new file mode 100644 index 0000000..3b514f2 --- /dev/null +++ b/scripts/preview/comment.py @@ -0,0 +1,122 @@ +"""PR-comment body builders for the k8s preview environment. + +Pure string-building functions -- the actual find-or-edit-in-place +mechanics stay in marocchino/sticky-pull-request-comment (an already-vetted +action doing exactly what it's for); only the message *content* lives here. +Uses GitHub's own web component (used all over its UI for +"3 minutes ago") instead of a static UTC string the reader has to convert +by hand -- confirmed via `gh api /markdown` to survive comment sanitization +unstripped. + +Usage: + python -m scripts.preview.comment render-ready --url URL \\ + --keycloak-url URL --deployed-at STR --deployed-at-iso ISO \\ + --expires-at STR --expires-at-iso ISO [--fork] + python -m scripts.preview.comment render-expired \\ + --expires-at STR --expires-at-iso ISO + python -m scripts.preview.comment render-stopped +""" + +from __future__ import annotations + +import argparse +import sys + +from . import gha + +PROJECT = "nebari-data-science-pack" + + +def render_ready( + url: str, + keycloak_url: str, + deployed_at: str, + deployed_at_iso: str, + expires_at: str, + expires_at_iso: str, + is_fork: bool, +) -> str: + fork_warning = ( + "\n\n⚠️ **This PR is from a fork**: the code running in this preview " + "is not from a trusted maintainer branch." + if is_fork + else "" + ) + return ( + "The latest K8s stack preview for this PR.\n\n" + "| Project | Deployment | Actions | Updated |\n" + "| --- | --- | --- | --- |\n" + f"| `{PROJECT}` | 🟢 [Ready]({url}) | [Preview]({url}) · [Keycloak]({keycloak_url}) | " + f'{deployed_at} |' + f"{fork_warning}\n\n" + f'Expires {expires_at}. ' + "Add the `extend-preview` label any time before then for 20 more minutes, " + "or push a new commit or re-add `deploy-preview` to redeploy from scratch." + ) + + +def render_expired(expires_at: str, expires_at_iso: str) -> str: + return ( + "The K8s stack preview for this PR has expired.\n\n" + "| Project | Deployment | Actions | Updated |\n" + "| --- | --- | --- | --- |\n" + f"| `{PROJECT}` | ⚫ Expired | - | " + f'{expires_at} |\n\n' + "Push a new commit or re-add the `deploy-preview` label to redeploy." + ) + + +def render_stopped() -> str: + return ( + "**K8s stack preview** stopped: the `deploy-preview` label was removed.\n\n" + "Add it again to redeploy." + ) + + +def _cmd_render_ready(args: argparse.Namespace) -> int: + body = render_ready( + args.url, args.keycloak_url, args.deployed_at, args.deployed_at_iso, + args.expires_at, args.expires_at_iso, args.fork, + ) + gha.write_output("body", body) + return 0 + + +def _cmd_render_expired(args: argparse.Namespace) -> int: + gha.write_output("body", render_expired(args.expires_at, args.expires_at_iso)) + return 0 + + +def _cmd_render_stopped(args: argparse.Namespace) -> int: + gha.write_output("body", render_stopped()) + return 0 + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(prog="comment") + sub = parser.add_subparsers(dest="command", required=True) + + p = sub.add_parser("render-ready") + p.add_argument("--url", required=True) + p.add_argument("--keycloak-url", required=True) + p.add_argument("--deployed-at", required=True) + p.add_argument("--deployed-at-iso", required=True) + p.add_argument("--expires-at", required=True) + p.add_argument("--expires-at-iso", required=True) + p.add_argument("--fork", action="store_true") + p.set_defaults(func=_cmd_render_ready) + + p = sub.add_parser("render-expired") + p.add_argument("--expires-at", required=True) + p.add_argument("--expires-at-iso", required=True) + p.set_defaults(func=_cmd_render_expired) + + p = sub.add_parser("render-stopped") + p.set_defaults(func=_cmd_render_stopped) + + args = parser.parse_args(argv[1:]) + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/scripts/preview/github_api.py b/scripts/preview/github_api.py index 60a5979..889792f 100644 --- a/scripts/preview/github_api.py +++ b/scripts/preview/github_api.py @@ -3,10 +3,27 @@ All calls go through ``http.request_json`` (stdlib ``urllib.request``, no new dependency) rather than shelling out to the ``gh`` CLI, so the actual request being made is a plain, testable function call. + +Usage: + python -m scripts.preview.github_api ensure-label-exists --repo R \\ + --token T --name NAME --color COLOR --description DESC + python -m scripts.preview.github_api create-and-activate --repo R \\ + --token T --ref SHA --environment ENV --task TASK --description DESC \\ + --environment-url URL --log-url URL --status-description DESC + python -m scripts.preview.github_api mark-inactive --repo R --token T \\ + --deployment-id ID --description DESC + python -m scripts.preview.github_api mark-latest-inactive --repo R \\ + --token T --environment ENV --description DESC + python -m scripts.preview.github_api cancel-in-flight-run --repo R \\ + --token T --workflow-name NAME --pr N """ from __future__ import annotations +import argparse +import sys + +from . import gha from .http import HTTPRequestError, request_json API_ROOT = "https://api.github.com" @@ -150,3 +167,95 @@ def cancel_in_flight_run(repo: str, workflow_name: str, pr_number: int, token: s ) return run["id"] return None + + +def _cmd_ensure_label_exists(args: argparse.Namespace) -> int: + ensure_label_exists(args.repo, args.name, args.color, args.description, args.token) + return 0 + + +def _cmd_create_and_activate(args: argparse.Namespace) -> int: + deployment_id = create_deployment( + args.repo, ref=args.ref, environment=args.environment, token=args.token, + task=args.task, description=args.description, + ) + gha.write_env("DEPLOYMENT_ID", str(deployment_id)) + set_deployment_status( + args.repo, deployment_id, "success", args.token, + environment_url=args.environment_url, log_url=args.log_url, + description=args.status_description, + ) + return 0 + + +def _cmd_mark_inactive(args: argparse.Namespace) -> int: + if not args.deployment_id: + return 0 + mark_deployment_inactive(args.repo, int(args.deployment_id), args.token, description=args.description) + return 0 + + +def _cmd_mark_latest_inactive(args: argparse.Namespace) -> int: + deployment_id = find_latest_deployment_id(args.repo, args.environment, args.token) + if deployment_id is None: + return 0 + mark_deployment_inactive(args.repo, deployment_id, args.token, description=args.description) + return 0 + + +def _cmd_cancel_in_flight_run(args: argparse.Namespace) -> int: + cancel_in_flight_run(args.repo, args.workflow_name, args.pr, args.token) + return 0 + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(prog="github_api") + sub = parser.add_subparsers(dest="command", required=True) + + p = sub.add_parser("ensure-label-exists") + p.add_argument("--repo", required=True) + p.add_argument("--token", required=True) + p.add_argument("--name", required=True) + p.add_argument("--color", required=True) + p.add_argument("--description", required=True) + p.set_defaults(func=_cmd_ensure_label_exists) + + p = sub.add_parser("create-and-activate") + p.add_argument("--repo", required=True) + p.add_argument("--token", required=True) + p.add_argument("--ref", required=True) + p.add_argument("--environment", required=True) + p.add_argument("--task", required=True) + p.add_argument("--description", required=True) + p.add_argument("--environment-url", required=True) + p.add_argument("--log-url", required=True) + p.add_argument("--status-description", required=True) + p.set_defaults(func=_cmd_create_and_activate) + + p = sub.add_parser("mark-inactive") + p.add_argument("--repo", required=True) + p.add_argument("--token", required=True) + p.add_argument("--deployment-id", required=True) + p.add_argument("--description", required=True) + p.set_defaults(func=_cmd_mark_inactive) + + p = sub.add_parser("mark-latest-inactive") + p.add_argument("--repo", required=True) + p.add_argument("--token", required=True) + p.add_argument("--environment", required=True) + p.add_argument("--description", required=True) + p.set_defaults(func=_cmd_mark_latest_inactive) + + p = sub.add_parser("cancel-in-flight-run") + p.add_argument("--repo", required=True) + p.add_argument("--token", required=True) + p.add_argument("--workflow-name", required=True) + p.add_argument("--pr", required=True, type=int) + p.set_defaults(func=_cmd_cancel_in_flight_run) + + args = parser.parse_args(argv[1:]) + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/tests/unit/test_preview_comment.py b/tests/unit/test_preview_comment.py new file mode 100644 index 0000000..d4d1077 --- /dev/null +++ b/tests/unit/test_preview_comment.py @@ -0,0 +1,122 @@ +"""Tests for scripts/preview/comment.py's PR-comment body builders. + +These strings were manually verified against GitHub's real markdown +renderer (`gh api /markdown`) earlier -- this module just moves the exact +same content into a testable function instead of a workflow `message:` +block, so these tests pin the content, not re-verify GitHub's rendering. +""" + +from __future__ import annotations + +from scripts.preview import comment + +URL = "https://pr-205-data-science-pack.openteams.app" +KC_URL = "https://keycloak-pr-205-data-science-pack.openteams.app" +DEPLOYED_AT = "2026-09-01 12:21 UTC" +DEPLOYED_AT_ISO = "2026-09-01T12:21:18Z" +EXPIRES_AT = "2026-09-01 12:41 UTC" +EXPIRES_AT_ISO = "2026-09-01T12:41:18Z" + + +def test_render_ready_contains_the_status_row_and_links(): + body = comment.render_ready(URL, KC_URL, DEPLOYED_AT, DEPLOYED_AT_ISO, EXPIRES_AT, EXPIRES_AT_ISO, is_fork=False) + + assert "| Project | Deployment | Actions | Updated |" in body + assert f"🟢 [Ready]({URL})" in body + assert f"[Preview]({URL})" in body + assert f"[Keycloak]({KC_URL})" in body + assert f'{DEPLOYED_AT}' in body + assert f'{EXPIRES_AT}' in body + assert "extend-preview" in body + assert "deploy-preview" in body + + +def test_render_ready_omits_fork_warning_when_not_a_fork(): + body = comment.render_ready(URL, KC_URL, DEPLOYED_AT, DEPLOYED_AT_ISO, EXPIRES_AT, EXPIRES_AT_ISO, is_fork=False) + + assert "fork" not in body.lower() + + +def test_render_ready_includes_fork_warning_when_a_fork(): + body = comment.render_ready(URL, KC_URL, DEPLOYED_AT, DEPLOYED_AT_ISO, EXPIRES_AT, EXPIRES_AT_ISO, is_fork=True) + + assert "This PR is from a fork" in body + assert "not from a trusted maintainer branch" in body + + +def test_render_ready_has_no_em_or_en_dashes(): + body = comment.render_ready(URL, KC_URL, DEPLOYED_AT, DEPLOYED_AT_ISO, EXPIRES_AT, EXPIRES_AT_ISO, is_fork=True) + + assert "—" not in body + assert "–" not in body + + +def test_render_expired_shows_expired_status_and_no_live_links(): + body = comment.render_expired(EXPIRES_AT, EXPIRES_AT_ISO) + + assert "has expired" in body + assert "⚫ Expired" in body + assert URL not in body + assert f'{EXPIRES_AT}' in body + + +def test_render_stopped_mentions_the_label(): + body = comment.render_stopped() + + assert "stopped" in body + assert "deploy-preview" in body + + +def test_main_render_ready_writes_body_output(monkeypatch, tmp_path): + out_file = tmp_path / "output" + out_file.write_text("") + monkeypatch.setenv("GITHUB_OUTPUT", str(out_file)) + + rc = comment.main([ + "comment", "render-ready", + "--url", URL, "--keycloak-url", KC_URL, + "--deployed-at", DEPLOYED_AT, "--deployed-at-iso", DEPLOYED_AT_ISO, + "--expires-at", EXPIRES_AT, "--expires-at-iso", EXPIRES_AT_ISO, + ]) + + assert rc == 0 + content = out_file.read_text() + assert content.startswith("body<<") + assert "🟢 [Ready]" in content + + +def test_main_render_ready_with_fork_flag_includes_warning(monkeypatch, tmp_path): + out_file = tmp_path / "output" + out_file.write_text("") + monkeypatch.setenv("GITHUB_OUTPUT", str(out_file)) + + comment.main([ + "comment", "render-ready", "--fork", + "--url", URL, "--keycloak-url", KC_URL, + "--deployed-at", DEPLOYED_AT, "--deployed-at-iso", DEPLOYED_AT_ISO, + "--expires-at", EXPIRES_AT, "--expires-at-iso", EXPIRES_AT_ISO, + ]) + + assert "This PR is from a fork" in out_file.read_text() + + +def test_main_render_expired_writes_body_output(monkeypatch, tmp_path): + out_file = tmp_path / "output" + out_file.write_text("") + monkeypatch.setenv("GITHUB_OUTPUT", str(out_file)) + + rc = comment.main(["comment", "render-expired", "--expires-at", EXPIRES_AT, "--expires-at-iso", EXPIRES_AT_ISO]) + + assert rc == 0 + assert "has expired" in out_file.read_text() + + +def test_main_render_stopped_writes_body_output(monkeypatch, tmp_path): + out_file = tmp_path / "output" + out_file.write_text("") + monkeypatch.setenv("GITHUB_OUTPUT", str(out_file)) + + rc = comment.main(["comment", "render-stopped"]) + + assert rc == 0 + assert "stopped" in out_file.read_text() diff --git a/tests/unit/test_preview_github_api.py b/tests/unit/test_preview_github_api.py index 33a2b89..8190d85 100644 --- a/tests/unit/test_preview_github_api.py +++ b/tests/unit/test_preview_github_api.py @@ -179,3 +179,107 @@ def fake_request_json(method, url, headers=None, body=None, timeout=15): monkeypatch.setattr("scripts.preview.github_api.request_json", fake_request_json) assert github_api.cancel_in_flight_run(REPO, "K8s Stack Preview", 205, TOKEN) is None + + +# --- CLI (main) --------------------------------------------------------------- + + +def test_main_ensure_label_exists_calls_through(monkeypatch): + called = [] + monkeypatch.setattr(github_api, "ensure_label_exists", lambda *a: called.append(a)) + + rc = github_api.main([ + "github_api", "ensure-label-exists", "--repo", REPO, "--token", TOKEN, + "--name", "extend-preview", "--color", "BFD4F2", "--description", "desc", + ]) + + assert rc == 0 + assert called == [(REPO, "extend-preview", "BFD4F2", "desc", TOKEN)] + + +def test_main_create_and_activate_writes_deployment_id_env(monkeypatch, tmp_path): + monkeypatch.setattr(github_api, "create_deployment", lambda *a, **k: 999) + set_calls = [] + monkeypatch.setattr(github_api, "set_deployment_status", lambda *a, **k: set_calls.append((a, k))) + env_file = tmp_path / "env" + env_file.write_text("") + monkeypatch.setenv("GITHUB_ENV", str(env_file)) + + rc = github_api.main([ + "github_api", "create-and-activate", "--repo", REPO, "--token", TOKEN, + "--ref", "abc123", "--environment", "pr-205-preview", "--task", "deploy:preview", + "--description", "K8s stack preview", "--environment-url", "https://x", "--log-url", "https://y", + "--status-description", "Live for 20 minutes", + ]) + + assert rc == 0 + assert "DEPLOYMENT_ID=999" in env_file.read_text() + assert set_calls[0][0] == (REPO, 999, "success", TOKEN) + assert set_calls[0][1] == {"environment_url": "https://x", "log_url": "https://y", "description": "Live for 20 minutes"} + + +def test_main_mark_inactive_noops_when_deployment_id_missing(monkeypatch): + called = [] + monkeypatch.setattr(github_api, "mark_deployment_inactive", lambda *a, **k: called.append(a)) + + rc = github_api.main([ + "github_api", "mark-inactive", "--repo", REPO, "--token", TOKEN, + "--deployment-id", "", "--description", "unused", + ]) + + assert rc == 0 + assert called == [] + + +def test_main_mark_inactive_calls_through_when_present(monkeypatch): + called = [] + monkeypatch.setattr(github_api, "mark_deployment_inactive", lambda *a, **k: called.append((a, k))) + + rc = github_api.main([ + "github_api", "mark-inactive", "--repo", REPO, "--token", TOKEN, + "--deployment-id", "999", "--description", "Preview expired", + ]) + + assert rc == 0 + assert called == [((REPO, 999, TOKEN), {"description": "Preview expired"})] + + +def test_main_mark_latest_inactive_noops_when_none_found(monkeypatch): + monkeypatch.setattr(github_api, "find_latest_deployment_id", lambda *a: None) + called = [] + monkeypatch.setattr(github_api, "mark_deployment_inactive", lambda *a, **k: called.append(a)) + + rc = github_api.main([ + "github_api", "mark-latest-inactive", "--repo", REPO, "--token", TOKEN, + "--environment", "pr-205-preview", "--description", "stopped", + ]) + + assert rc == 0 + assert called == [] + + +def test_main_mark_latest_inactive_marks_when_found(monkeypatch): + monkeypatch.setattr(github_api, "find_latest_deployment_id", lambda *a: 42) + called = [] + monkeypatch.setattr(github_api, "mark_deployment_inactive", lambda *a, **k: called.append((a, k))) + + rc = github_api.main([ + "github_api", "mark-latest-inactive", "--repo", REPO, "--token", TOKEN, + "--environment", "pr-205-preview", "--description", "stopped", + ]) + + assert rc == 0 + assert called == [((REPO, 42, TOKEN), {"description": "stopped"})] + + +def test_main_cancel_in_flight_run_calls_through(monkeypatch): + called = [] + monkeypatch.setattr(github_api, "cancel_in_flight_run", lambda *a: called.append(a) or 3) + + rc = github_api.main([ + "github_api", "cancel-in-flight-run", "--repo", REPO, "--token", TOKEN, + "--workflow-name", "K8s Stack Preview", "--pr", "205", + ]) + + assert rc == 0 + assert called == [(REPO, "K8s Stack Preview", 205, TOKEN)] From 9305abe95ab828c422ab2236f72de835fbcc367c Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Wed, 2 Sep 2026 15:56:19 +0100 Subject: [PATCH 53/54] refactor: move the extend-preview tunnel loop into scripts/preview/tunnel.py The last and messiest piece: replaces the background-cloudflared + kill-0/wait polling loop with a Python subprocess.Popen equivalent. The deadline/extend decision logic (next_deadline, should_stop) is now pure and independently tested, and run() itself is tested with a fake controllable process and faked label list/delete calls -- including a hand-traced case proving an extend genuinely resets the deadline rather than just coincidentally lining up. This was the final increment of the k8s-preview.yaml business-logic extraction: every step with real branching/parsing/retry logic now calls into scripts/preview/, leaving the workflow itself as orchestration (checkout, helm, kubectl waits, docker build) plus 1-3 line CLI invocations. --- .github/workflows/k8s-preview.yaml | 50 ++--------- scripts/preview/tunnel.py | 118 ++++++++++++++++++++++++++ tests/unit/test_preview_tunnel.py | 130 +++++++++++++++++++++++++++++ 3 files changed, 255 insertions(+), 43 deletions(-) create mode 100644 scripts/preview/tunnel.py create mode 100644 tests/unit/test_preview_tunnel.py diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index c51d5a9..9a3c604 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -518,51 +518,15 @@ jobs: # "cancelled" with a "job has exceeded the maximum execution time" # failure annotation, which looks like a real failure even though # nothing broke -- the preview is *meant* to expire eventually. - # - # Runs cloudflared in the background instead of blocking on it, so - # this loop can poll for the `extend-preview` label between checks - # and push the deadline back 20 minutes from whenever it actually - # sees the label (not just tack 20 more onto whatever's left) -- - # each occurrence is a one-shot reset, consumed by removing the - # label, so it can be reused any number of times before expiry. - # Ultimately still bounded by the job's own 90min timeout-minutes - # regardless of how many times it's extended. - # - # `kill -0 $pid` checks liveness without signaling; if cloudflared - # exits on its own (a real crash, not us closing it), `wait` reaps - # its actual exit code so a genuine failure still fails the step -- - # `|| ec=$?` catches that under `bash -e` (GitHub Actions' default - # for run: steps) without the step aborting mid-check. + # Extend the deadline any time before then via the extend-preview + # label; see scripts/preview/tunnel.py. - name: Run tunnel until it times out (extend via the extend-preview label) - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - /tmp/cloudflared tunnel --no-autoupdate run --token "${TUNNEL_TOKEN}" & - cloudflared_pid=$! - - deadline=$(( $(date +%s) + 1200 )) - ec=0 - while true; do - if ! kill -0 "$cloudflared_pid" 2>/dev/null; then - wait "$cloudflared_pid" || ec=$? - break - fi - now=$(date +%s) - if [ "$now" -ge "$deadline" ]; then - kill "$cloudflared_pid" 2>/dev/null || true - wait "$cloudflared_pid" 2>/dev/null || true - break - fi - if gh api "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels" --jq '.[].name' 2>/dev/null \ - | grep -qx "extend-preview"; then - deadline=$(( now + 1200 )) - gh api -X DELETE "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels/extend-preview" \ - > /dev/null 2>&1 || true - echo "extend-preview seen -- new deadline: $(date -u -d "@$deadline" +'%Y-%m-%d %H:%M UTC')" - fi - sleep 15 - done - [ "$ec" -eq 0 ] || exit "$ec" + python -m scripts.preview.tunnel run \ + --cloudflared /tmp/cloudflared --token "${TUNNEL_TOKEN}" \ + --repo "${{ github.repository }}" --pr "${{ github.event.pull_request.number }}" \ + --github-token "${{ secrets.GITHUB_TOKEN }}" \ + --initial-seconds 1200 --poll-seconds 15 --extend-seconds 1200 # Runs once the tunnel step above ends (its own timeout, cloudflared # exiting, or a manual cancel), so this captures anything logged in diff --git a/scripts/preview/tunnel.py b/scripts/preview/tunnel.py new file mode 100644 index 0000000..5b25673 --- /dev/null +++ b/scripts/preview/tunnel.py @@ -0,0 +1,118 @@ +"""Runs cloudflared and lets an extend-preview label push its deadline back. + +Bounded to `initial_seconds` by default so the live preview doesn't sit +open (and burn CI minutes) indefinitely. Polls for the `extend_label` on +the PR between checks; each occurrence resets the deadline to +now + extend_seconds (not a cumulative add onto whatever's left) and is +consumed by removing the label, so it can be reused any number of times +before expiry. Still ultimately bounded by the calling job's own +timeout-minutes regardless of how many times it's extended. + +Usage: + python -m scripts.preview.tunnel run --cloudflared PATH --token TOKEN \\ + --repo OWNER/REPO --pr N --github-token TOKEN \\ + [--initial-seconds 1200] [--poll-seconds 15] [--extend-seconds 1200] \\ + [--extend-label extend-preview] +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +import time +from collections.abc import Callable + +from .github_api import delete_label, list_labels + + +def next_deadline(now: float, current_deadline: float, label_present: bool, extend_seconds: int) -> float: + """What should the deadline be this tick? + + A reset to now + extend_seconds when the label is present, not a + cumulative add onto whatever's left, so an extend always means + "extend_seconds more from right now." + """ + return now + extend_seconds if label_present else current_deadline + + +def should_stop(alive: bool, now: float, deadline: float) -> bool: + return (not alive) or now >= deadline + + +def run( + cloudflared_path: str, + tunnel_token: str, + repo: str, + pr_number: int, + github_token: str, + initial_seconds: int = 1200, + poll_seconds: int = 15, + extend_seconds: int = 1200, + extend_label: str = "extend-preview", + popen: Callable[..., subprocess.Popen] = subprocess.Popen, + clock: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, + list_labels_fn: Callable[[str, int, str], list[str]] = list_labels, + delete_label_fn: Callable[[str, int, str, str], None] = delete_label, +) -> int: + """Run cloudflared until its deadline, or until it exits on its own. + + Returns cloudflared's real exit code if it exited on its own (a + genuine crash), or 0 if we closed it ourselves (deadline reached). + """ + proc = popen([cloudflared_path, "tunnel", "--no-autoupdate", "run", "--token", tunnel_token]) + deadline = clock() + initial_seconds + + while True: + alive = proc.poll() is None + now = clock() + if should_stop(alive, now, deadline): + if not alive: + return proc.returncode + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + return 0 + + if extend_label in list_labels_fn(repo, pr_number, github_token): + deadline = next_deadline(now, deadline, True, extend_seconds) + delete_label_fn(repo, pr_number, extend_label, github_token) + print(f"{extend_label} seen -- new deadline: {time.strftime('%Y-%m-%d %H:%M UTC', time.gmtime(deadline))}") + + sleep(poll_seconds) + + +def _cmd_run(args: argparse.Namespace) -> int: + return run( + args.cloudflared, args.token, args.repo, args.pr, args.github_token, + initial_seconds=args.initial_seconds, poll_seconds=args.poll_seconds, + extend_seconds=args.extend_seconds, extend_label=args.extend_label, + ) + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(prog="tunnel") + sub = parser.add_subparsers(dest="command", required=True) + + p = sub.add_parser("run") + p.add_argument("--cloudflared", required=True) + p.add_argument("--token", required=True) + p.add_argument("--repo", required=True) + p.add_argument("--pr", required=True, type=int) + p.add_argument("--github-token", required=True) + p.add_argument("--initial-seconds", type=int, default=1200) + p.add_argument("--poll-seconds", type=int, default=15) + p.add_argument("--extend-seconds", type=int, default=1200) + p.add_argument("--extend-label", default="extend-preview") + p.set_defaults(func=_cmd_run) + + args = parser.parse_args(argv[1:]) + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/tests/unit/test_preview_tunnel.py b/tests/unit/test_preview_tunnel.py new file mode 100644 index 0000000..f452b4c --- /dev/null +++ b/tests/unit/test_preview_tunnel.py @@ -0,0 +1,130 @@ +"""Tests for scripts/preview/tunnel.py's cloudflared runner + extend-preview loop.""" + +from __future__ import annotations + +from scripts.preview import tunnel + +REPO = "nebari-dev/data-science-pack" +TOKEN = "gh-token" + + +# --- next_deadline / should_stop (pure) --------------------------------------- + + +def test_next_deadline_resets_to_now_plus_extend_when_label_present(): + # A reset, not a cumulative add: the old deadline is irrelevant once + # the label is seen. + assert tunnel.next_deadline(now=100.0, current_deadline=105.0, label_present=True, extend_seconds=1200) == 1300.0 + + +def test_next_deadline_unchanged_when_label_absent(): + assert tunnel.next_deadline(now=100.0, current_deadline=105.0, label_present=False, extend_seconds=1200) == 105.0 + + +def test_should_stop_true_when_process_no_longer_alive(): + assert tunnel.should_stop(alive=False, now=0.0, deadline=1000.0) is True + + +def test_should_stop_true_when_deadline_reached(): + assert tunnel.should_stop(alive=True, now=1000.0, deadline=1000.0) is True + + +def test_should_stop_false_while_alive_and_before_deadline(): + assert tunnel.should_stop(alive=True, now=500.0, deadline=1000.0) is False + + +# --- run (subprocess + label polling, all injected) --------------------------- + + +class _FakeProcess: + def __init__(self, exit_after_polls=None, exit_code=0): + self._polls = 0 + self._exit_after_polls = exit_after_polls + self.exit_code = exit_code + self.returncode = None + self.terminated = False + self.killed = False + + def poll(self): + self._polls += 1 + if self._exit_after_polls is not None and self._polls >= self._exit_after_polls: + self.returncode = self.exit_code + return self.returncode + + def terminate(self): + self.terminated = True + self.returncode = 0 + + def kill(self): + self.killed = True + self.returncode = -9 + + def wait(self, timeout=None): + return self.returncode + + +def _fake_clock_sleep(): + state = {"t": 0.0} + return (lambda: state["t"]), (lambda s: state.__setitem__("t", state["t"] + s)) + + +def test_run_returns_cloudflared_exit_code_on_real_crash(): + proc = _FakeProcess(exit_after_polls=2, exit_code=7) + clock, sleep = _fake_clock_sleep() + + rc = tunnel.run( + cloudflared_path="/tmp/cloudflared", tunnel_token="tok", repo=REPO, pr_number=205, + github_token=TOKEN, initial_seconds=1200, poll_seconds=15, + popen=lambda *a, **k: proc, clock=clock, sleep=sleep, + list_labels_fn=lambda *a: [], delete_label_fn=lambda *a: None, + ) + + assert rc == 7 + assert proc.terminated is False + + +def test_run_terminates_process_and_returns_0_at_deadline(): + proc = _FakeProcess() # never exits on its own + clock, sleep = _fake_clock_sleep() + + rc = tunnel.run( + cloudflared_path="/tmp/cloudflared", tunnel_token="tok", repo=REPO, pr_number=205, + github_token=TOKEN, initial_seconds=30, poll_seconds=15, + popen=lambda *a, **k: proc, clock=clock, sleep=sleep, + list_labels_fn=lambda *a: [], delete_label_fn=lambda *a: None, + ) + + assert rc == 0 + assert proc.terminated is True + + +def test_run_extends_deadline_when_label_seen_and_deletes_it(): + # initial_seconds=30, poll_seconds=15: checks happen at t=0 (absent), + # t=15 (present -> deadline resets to 15+30=45, clearly past the + # original 30s deadline), t=30 (absent, 30 < 45 so it keeps going), + # t=45 (stop: 45 >= 45). If the extend hadn't taken effect, the loop + # would have stopped at t=30 instead. + proc = _FakeProcess() + clock, sleep = _fake_clock_sleep() + label_calls = [] + delete_calls = [] + + def list_labels_fn(*a): + label_calls.append(1) + return ["extend-preview"] if len(label_calls) == 2 else [] + + def delete_label_fn(*a): + delete_calls.append(a) + + rc = tunnel.run( + cloudflared_path="/tmp/cloudflared", tunnel_token="tok", repo=REPO, pr_number=205, + github_token=TOKEN, initial_seconds=30, poll_seconds=15, extend_seconds=30, + popen=lambda *a, **k: proc, clock=clock, sleep=sleep, + list_labels_fn=list_labels_fn, delete_label_fn=delete_label_fn, + ) + + assert rc == 0 + assert len(delete_calls) == 1 + assert delete_calls[0] == (REPO, 205, "extend-preview", TOKEN) + # Stopped at t=45 (the extended deadline), not t=30 (the original one). + assert clock() == 45.0 From df6d00764a74f8b25bb69e81c5e8eca5c1c12be9 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Wed, 2 Sep 2026 17:12:23 +0100 Subject: [PATCH 54/54] ci: dump ArgoCD Application status on deploy failure Keycloak failed to appear in its namespace after a 600s wait twice in a row today, with no error anywhere in the log explaining why -- the sandbox action's own wait only polls for K8s resources existing, never whether ArgoCD actually attempted (or failed) the sync that's supposed to create them. Dumps every ArgoCD Application plus the full keycloak/nebari-operator Application detail so a repeat is actually diagnosable from the CI log instead of requiring an interactive tmate session. --- .github/workflows/k8s-preview.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 9a3c604..7a80005 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -370,6 +370,21 @@ jobs: env: KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} run: | + # ArgoCD Application sync status -- the sandbox's own + # wait-platform.sh only polls for K8s resources existing per + # namespace, never checks whether ArgoCD ever actually synced + # the Application that's supposed to create them. Confirmed + # live: Keycloak's namespace had 0 resources after a 600s wait + # with no error anywhere else in the log -- this is the only + # place that would show whether ArgoCD ever attempted the sync + # at all, and why, if it didn't. + echo "=== ArgoCD Applications ===" + kubectl -n argocd get applications -o wide || true + for app in keycloak nebari-operator; do + echo "--- ArgoCD application/$app ---" + kubectl -n argocd get application "$app" -o yaml || true + done + for ns in pr-preview keycloak; do echo "=== namespace: $ns ===" kubectl -n "$ns" get pods -o wide || true