diff --git a/deploy/templates/service-setup.yaml b/deploy/templates/service-setup.yaml index 42259c32c..23978309b 100644 --- a/deploy/templates/service-setup.yaml +++ b/deploy/templates/service-setup.yaml @@ -135,8 +135,14 @@ spec: # SPI's cached JAAS Subject. Without a restart # the SPI's outbound calls to F+ Auth time # out until the pod is bounced manually. + # openid is gated: with openid.enabled=false the + # deployment does not exist, and an ungated restart would + # exit 1 on every run. The Job is restartPolicy OnFailure + # with backoffLimit 9999, so each retry would bounce mqtt + # again - restarting the broker every few minutes for + # weeks while helm upgrade reports success. NAMESPACE={{.Release.Namespace}} - for D in mqtt openid; do + for D in mqtt{{ if .Values.openid.enabled }} openid{{ end }}; do echo "Restarting deployment: $D" kubectl rollout restart "deployment/$D" -n "$NAMESPACE" || exit 1 kubectl rollout status "deployment/$D" -n "$NAMESPACE" --timeout=120s || exit 1 diff --git a/deploy/values.yaml b/deploy/values.yaml index 342edd4f9..a35280ec4 100644 --- a/deploy/values.yaml +++ b/deploy/values.yaml @@ -454,7 +454,17 @@ grafana: # v13 migrates dashboards to a new schema ON FIRST USE and the migration # is one-way — see "Grafana upgraded from v10 to v13" in the release # notes before downgrading this on an existing installation. - tag: 13.1.0 + # Not 13.x. Grafana 13 deleted the sqlstore migration that adds + # playlist.created_at/updated_at (present 10.2.0 - 12.4.x) while + # still running a unified-storage playlist migration whose SQL + # requires those columns, so any database created before 10.2.0 - + # including every ACS v5.1.0 site, which shipped Grafana 10.0.1 - + # cannot start under 13. 12.4.x still ships the migration, adds the + # columns through Grafana's own framework, and then completes the + # unified-storage migration itself, recording it in + # unifiedstorage_migration_log. A later move to 13 is then safe, + # because 13 skips the migration on that log row. + tag: 12.4.5 enabled: true persistence: enabled: true @@ -533,25 +543,20 @@ grafana: # revoking Grafana.Perm.* in F+ downgrades on next login rather # than letting the 60s SPI cache staleness become permanent. role_attribute_strict: true - # Grafana 13 runs a unified-storage data migration for playlists on - # boot, enabled by default (MigratedUnifiedResources in - # pkg/setting/setting_unified_storage.go). Its SQL selects - # playlist.created_at, but the sqlstore migration which adds that - # column existed only in 11.x/12.x: v13 deleted playlist_mig.go - # entirely, so a database created by Grafana 10.x and upgraded - # straight to 13.x never gains the column. Grafana then dies on boot - # with "no such column: p.created_at" - the SELECT fails at prepare, - # so having no playlists does not help. - # - # ACS v5.1.0 shipped Grafana 10.0.1, so every site upgrading with a - # kept Grafana PVC hits this. Fresh installs do not: v13 seeds the - # column from its schema snapshot, which is why CI stayed green. - # - # Skipping the migration leaves the legacy playlist table untouched - # and unified storage with no playlists. ACS does not use playlists. - # Drop this key and add the columns by hand if you ever need them. - unified_storage.playlists.playlist.grafana.app: - enableMigration: false + # Keep folders and dashboards on legacy storage on every site, not + # just the big ones. Grafana 12.4.x auto-migrates them into unified + # storage when a site has fewer dashboards than + # DefaultAutoMigrationThreshold (10, setting_unified_storage.go), and + # that migration is one-way. ACS provisions five dashboards, so + # without this a fresh install silently takes a migration that an + # established site does not, splitting the estate into two storage + # modes on a row count - and leaving CI exercising a different path + # from production. A threshold of 1 is below any real dashboard + # count, so shouldAutoMigrate is false everywhere. + unified_storage.folders.folder.grafana.app: + autoMigrationThreshold: 1 + unified_storage.dashboards.dashboard.grafana.app: + autoMigrationThreshold: 1 extraConfigmapMounts: - name: acs-grafana-config mountPath: /etc/acs-config diff --git a/docs/reference/release-notes.md b/docs/reference/release-notes.md index 960800413..12a065c3a 100644 --- a/docs/reference/release-notes.md +++ b/docs/reference/release-notes.md @@ -94,21 +94,35 @@ grafana: disable_login_form: false ``` -### Grafana upgraded from v10 to v13 +### Grafana upgraded from v10 to v12 -The bundled Grafana image moves from 10.0.1 to 13.1.0. Grafana 10.x +The bundled Grafana image moves from 10.0.1 to 12.4.5. Grafana 10.x left support in 2024, so this catches up two majors of security fixes, and the newer `auth.jwt` support is what lets machine-to-machine consumers sign in with tokens (see below). +This release deliberately stops at 12.4.x rather than 13.x. Grafana 13 +removed the schema migration that adds the `playlist.created_at` and +`updated_at` columns, which were introduced in 10.2.0, while still +running a playlist migration that requires them. A database created by +the Grafana 10.0.1 that ACS v5.1.0 shipped therefore cannot start under +Grafana 13 at all. Grafana 12.4.x still ships that migration, so it +adds the columns and completes the playlist migration itself; a later +move to 13 will then be safe. + Three upstream Grafana changes are worth knowing about when upgrading an existing installation: - Dashboards are migrated to Grafana's new dashboard schema as they are - used, and the migration is one-way: once v13 has written to the - Grafana database you cannot return to the previous image without + used, and the migration is one-way: once the new image has written to + the Grafana database you cannot return to the previous one without losing dashboards. Snapshot the Grafana PersistentVolume before upgrading if the dashboards matter. +- Your playlists are migrated into Grafana's unified storage on first + boot. This is also one-way, and the same snapshot covers it. Check + after upgrading that the playlists you expect are all present before + you rely on the installation; the migration logs a warning rather + than failing if it rejects an individual playlist. - Support for AngularJS plugins has been removed (disabled by default in Grafana 11, gone entirely in 12). Grafana's built-in panels are unaffected and legacy Graph/Table panels are migrated automatically,