Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 19 additions & 4 deletions pkg/server/api/debug.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,25 @@ import (

func (h *Server) DebugHealth(c *gin.Context) {
status := http.StatusOK
if h.isClosing.Load() || !h.mgr.NsMgr.Ready() {
health := config.HealthInfo{
ConfigChecksum: h.mgr.CfgMgr.GetConfigChecksum(),
}
if h.unhealthyMark.Load() {
status = http.StatusBadGateway
} else if h.isClosing.Load() || !h.mgr.NsMgr.Ready() {
status = http.StatusBadGateway
}
c.JSON(status, config.HealthInfo{
ConfigChecksum: h.mgr.CfgMgr.GetConfigChecksum(),
})
c.JSON(status, health)
}

func (h *Server) DebugSetHealthUnhealthy(c *gin.Context) {
h.unhealthyMark.Store(true)
c.JSON(http.StatusOK, "")
}

func (h *Server) DebugUnsetHealthUnhealthy(c *gin.Context) {
h.unhealthyMark.Store(false)
c.JSON(http.StatusOK, "")
}

func (h *Server) DebugRedirect(c *gin.Context) {
Expand All @@ -39,5 +52,7 @@ func (h *Server) DebugRedirect(c *gin.Context) {
func (h *Server) registerDebug(group *gin.RouterGroup) {
group.POST("/redirect", h.DebugRedirect)
group.GET("/health", h.DebugHealth)
group.POST("/health/unhealthy", h.DebugSetHealthUnhealthy)
group.DELETE("/health/unhealthy", h.DebugUnsetHealthUnhealthy)
pprof.RouteRegister(group, "/pprof")
}
35 changes: 35 additions & 0 deletions pkg/server/api/debug_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
package api

import (
"encoding/json"
"net/http"
"testing"

Expand Down Expand Up @@ -39,3 +40,37 @@ func TestDebug(t *testing.T) {
require.Equal(t, http.StatusBadGateway, r.StatusCode)
})
}

func TestDebugHealthManualUnhealthy(t *testing.T) {
_, doHTTP := createServer(t)

doHTTP(t, http.MethodGet, "/api/debug/health", httpOpts{}, func(t *testing.T, r *http.Response) {
require.Equal(t, http.StatusOK, r.StatusCode)
var health map[string]any
require.NoError(t, json.NewDecoder(r.Body).Decode(&health))
_, ok := health["unhealthy_reason"]
require.False(t, ok)
})

doHTTP(t, http.MethodPost, "/api/debug/health/unhealthy", httpOpts{}, func(t *testing.T, r *http.Response) {
require.Equal(t, http.StatusOK, r.StatusCode)
})
doHTTP(t, http.MethodGet, "/api/debug/health", httpOpts{}, func(t *testing.T, r *http.Response) {
require.Equal(t, http.StatusBadGateway, r.StatusCode)
var health map[string]any
require.NoError(t, json.NewDecoder(r.Body).Decode(&health))
_, ok := health["unhealthy_reason"]
require.False(t, ok)
})

doHTTP(t, http.MethodDelete, "/api/debug/health/unhealthy", httpOpts{}, func(t *testing.T, r *http.Response) {
require.Equal(t, http.StatusOK, r.StatusCode)
})
doHTTP(t, http.MethodGet, "/api/debug/health", httpOpts{}, func(t *testing.T, r *http.Response) {
require.Equal(t, http.StatusOK, r.StatusCode)
var health map[string]any
require.NoError(t, json.NewDecoder(r.Body).Decode(&health))
_, ok := health["unhealthy_reason"]
require.False(t, ok)
})
}
18 changes: 10 additions & 8 deletions pkg/server/api/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,14 +61,16 @@ type Managers struct {
}

type Server struct {
listener net.Listener
wg waitgroup.WaitGroup
limit ratelimit.Limiter
ready *atomic.Bool
lg *zap.Logger
grpc *grpc.Server
isClosing atomic.Bool
mgr Managers
listener net.Listener
wg waitgroup.WaitGroup
limit ratelimit.Limiter
ready *atomic.Bool
lg *zap.Logger
grpc *grpc.Server
isClosing atomic.Bool
// unhealthyMark makes the health endpoint return unhealthy so that the external LB stops routing new traffic to this TiProxy.
unhealthyMark atomic.Bool
mgr Managers
}

func NewServer(cfg config.API, lg *zap.Logger, mgr Managers, handler HTTPHandler, ready *atomic.Bool) (*Server, error) {
Expand Down