From a7c2909bdecf48f2e2d42090e60d9d54d5b1be5d Mon Sep 17 00:00:00 2001 From: Damilola Edwards Date: Sat, 4 Jul 2026 16:41:51 +0100 Subject: [PATCH] Keep autodiscovered datasource when a probe errors reconcileClickHouseAutodiscover removed a present datasource whenever a probe reported unavailable, treating a probe failure the same as a confirmed absence. A transient timeout, network blip, or 5xx therefore tore down a healthy datasource, which flapped back on the next successful probe. Handle the probe error before the add and remove branches and leave the current state unchanged on error, so removal only happens when a probe succeeds and reports the database absent. Update the reconcile test to cover surviving a transient error and removal on genuine absence. --- pkg/proxy/autodiscover.go | 17 +++++++++++------ pkg/proxy/server_test.go | 11 ++++++++++- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/pkg/proxy/autodiscover.go b/pkg/proxy/autodiscover.go index ba1f45fc..37306bd5 100644 --- a/pkg/proxy/autodiscover.go +++ b/pkg/proxy/autodiscover.go @@ -107,6 +107,17 @@ func (s *server) reconcileClickHouseAutodiscover(ctx context.Context, entry Clic return } + // A failed probe means we could not determine availability, not that the + // datasource is gone. Leave the current state untouched so a transient + // error (timeout, network blip, 5xx) does not tear down a healthy + // datasource. Removal only happens when a probe succeeds and reports the + // database absent. + if err != nil { + logger.WithError(err).Debug("ClickHouse autodiscovery probe failed; leaving current state unchanged") + + return + } + if available && !*present { if !s.addAutodiscoveredClickHouseCluster(target.cluster) { return @@ -126,12 +137,6 @@ func (s *server) reconcileClickHouseAutodiscover(ctx context.Context, entry Clic return } - if err != nil { - logger.WithError(err).Debug("ClickHouse autodiscovery probe failed") - - return - } - logger.WithField("available", available).Debug("ClickHouse autodiscovery state unchanged") } diff --git a/pkg/proxy/server_test.go b/pkg/proxy/server_test.go index d5071355..7d1390b6 100644 --- a/pkg/proxy/server_test.go +++ b/pkg/proxy/server_test.go @@ -744,8 +744,17 @@ func TestAutodiscoverReconcilesClickHouseDatasource(t *testing.T) { t.Fatalf("/datasources ClickHouseInfo[0].Description = %q, want custom description", datasources.ClickHouseInfo[0].Description) } + // A transient probe error must not remove a healthy datasource. pingOK = false srv.reconcileClickHouseAutodiscover(t.Context(), entry, &present) + if !present || !srv.clickhouseHandler.HasCluster("local-kurtosis") { + t.Fatal("expected datasource to survive a transient probe error") + } + pingOK = true + + // A successful probe that reports the database absent removes it. + databaseExists = false + srv.reconcileClickHouseAutodiscover(t.Context(), entry, &present) if present { t.Fatal("expected datasource to become absent") } @@ -756,7 +765,7 @@ func TestAutodiscoverReconcilesClickHouseDatasource(t *testing.T) { t.Fatalf("ClickHouseDatasourceInfo() after removal = %v, want empty", got) } - pingOK = true + databaseExists = true srv.reconcileClickHouseAutodiscover(t.Context(), entry, &present) if !present { t.Fatal("expected datasource to become present again")