observability.docs.plus.yml

# Server-only observability stack deploy. Decoupled from the app's blue-green
# release: no quality-gate dependency and outside the app deploy concurrency
# group — deliberate (the self-hosted runner serializes jobs).
name: CI/CD Observability

on:
  workflow_call:
    inputs:
      action:
        required: false
        default: setup
        type: string
  workflow_dispatch:
    inputs:
      action:
        description: 'Action to run on the stack'
        required: true
        default: setup
        type: choice
        options: [setup, update, restart, down]

defaults:
  run:
    shell: bash

permissions:
  contents: read

env:
  COMPOSE_FILE: docker-compose.observability.yml
  ENV_SOURCE: /opt/projects/prod.docs.plus/.env.observability
  ENV_FILE: .env.observability

jobs:
  deploy:
    name: 🔭 Deploy Observability
    runs-on: prod.docs.plus
    timeout-minutes: 15
    concurrency:
      group: observability-deploy
      cancel-in-progress: false
    # Gate on the action input, NOT github.event_name: inside workflow_call that is
    # the CALLER's event ('push'), so an event check skipped every orchestrated deploy.
    # github.ref is the caller's ref too, so the push to main still runs, and a
    # dispatch from any other branch skips (#437).
    if: github.ref == 'refs/heads/main' && inputs.action != 'down'
    # No `environment: production` — it would inherit the production environment's
    # required-reviewer protection and gate every monitoring deploy on manual approval.
    permissions:
      contents: read
    steps:
      - name: 📦 Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 1

      - name: 🔐 Prepare environment
        run: |
          cp "${ENV_SOURCE}" "${ENV_FILE}"
          # Require a real first character: dotenv strips quotes, so KEY="" or
          # whitespace-only values are empty at runtime and must fail here.
          for k in GRAFANA_ADMIN_PASSWORD GLITCHTIP_SECRET_KEY GLITCHTIP_POSTGRES_PASSWORD TELEGRAM_BOT_TOKEN TELEGRAM_CHAT_ID DATABASE_URL DEADMAN_WEBHOOK_URL; do
            if ! grep -Eq "^${k}=[\"']?[^\"'[:space:]]" "${ENV_FILE}"; then
              echo "::error::${k} missing or empty in ${ENV_SOURCE}"; exit 1
            fi
          done
          echo "✅ Environment ready"

      - name: 📨 Render Telegram contact point
        # Grafana's native ${VAR} expansion in alerting provisioning mistypes a numeric
        # chat id (-100…) into a JSON number and fails to load; envsubst renders a literal
        # quoted string instead. Only the allowlisted vars are substituted so the Go
        # message template and the LogQL/PromQL `$` in rules-*.yml are left intact.
        run: |
          set -a
          # shellcheck disable=SC1090
          . "./${ENV_FILE}"
          set +a
          # Warnings route falls back to the main ops chat until a dedicated one is set.
          : "${TELEGRAM_WARNINGS_CHAT_ID:=${TELEGRAM_CHAT_ID}}"
          export TELEGRAM_WARNINGS_CHAT_ID
          # shellcheck disable=SC2016
          envsubst '${TELEGRAM_BOT_TOKEN} ${TELEGRAM_CHAT_ID} ${TELEGRAM_WARNINGS_CHAT_ID} ${DEADMAN_WEBHOOK_URL}' \
            < scripts/observability/grafana/provisioning/alerting/contactpoints.yml.tmpl \
            > scripts/observability/grafana/provisioning/alerting/contactpoints.yml

      - name: 💾 Disk guard
        run: |
          AVAIL_GB=$(( $(df --output=avail / | tail -1) / 1024 / 1024 ))
          echo "📊 ${AVAIL_GB} GB free on /"
          if [ "${AVAIL_GB}" -lt 10 ]; then
            echo "::error::Less than 10 GB free on /. Aborting."; df -h /; exit 1
          fi

      - name: 🔧 Ensure network
        run: docker network create docsplus-network 2>/dev/null || true

      - name: 🚀 Deploy stack
        env:
          ACTION_IN: ${{ inputs.action }}
        run: |
          ACTION="${ACTION_IN:-setup}"
          case "${ACTION}" in
            setup|update|restart|down) ;;
            *) echo "::error::unknown observability action '${ACTION}'"; exit 1 ;;
          esac
          case "${ACTION}" in
            restart)
              docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" restart ;;
            *)
              docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" pull
              # Validate the mounted config before touching the running stack.
              docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
                run --rm --entrypoint promtool prometheus check config /etc/prometheus/prometheus.yml
              docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" up -d --remove-orphans
              # `up -d` never re-reads bind-mounted config files on unchanged containers;
              # restart forces prometheus.yml + grafana provisioning to be picked up.
              docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" restart prometheus grafana ;;
          esac

      - name: 🩺 Verify
        run: |
          docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" ps
          # Grafana ships wget; a single probe after a short settle is fine.
          sleep 10
          docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" exec -T grafana \
            wget -qO- http://localhost:3000/api/health | grep -q ok && echo "✅ grafana healthy"
          # GlitchTip cold-boots + migrates and ships no wget/curl (only python3) — poll up to ~90s.
          ok=0
          for _i in $(seq 1 18); do
            if docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" exec -T glitchtip-web \
                 python3 -c 'import urllib.request,sys; sys.exit(0 if urllib.request.urlopen("http://localhost:8080/_health/").status==200 else 1)' 2>/dev/null; then
              ok=1; echo "✅ glitchtip healthy"; break
            fi
            sleep 5
          done
          [ "${ok}" = 1 ] || { echo "::error::glitchtip-web not healthy after ~90s"; exit 1; }

      - name: 📣 Notify failure
        if: failure()
        # Reads ENV_SOURCE (not the copied file) — the copy step itself may be what failed.
        run: bash scripts/ci/notify-telegram.sh "${ENV_SOURCE}" "🔭 Observability deploy FAILED (${GITHUB_SHA}, run ${GITHUB_RUN_ID})"

  teardown:
    name: 🧹 Down
    runs-on: prod.docs.plus
    timeout-minutes: 10
    if: github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && inputs.action == 'down'
    permissions:
      contents: read
    steps:
      - name: 📦 Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - name: 🔐 Prepare environment
        run: cp "${ENV_SOURCE}" "${ENV_FILE}"
      - name: 🧹 Down (keep volumes)
        run: docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" down