From c834a5b0a2e1519d57e37b253af1d29d468f436a Mon Sep 17 00:00:00 2001 From: nhsmw Date: Wed, 5 Aug 2026 18:28:23 +0800 Subject: [PATCH 01/18] config: Enable data sharing in default event store config --- pkg/config/debug.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/config/debug.go b/pkg/config/debug.go index 9186170b52..0d60d0440d 100644 --- a/pkg/config/debug.go +++ b/pkg/config/debug.go @@ -113,7 +113,7 @@ func NewDefaultEventStoreConfig() *EventStoreConfig { return &EventStoreConfig{ CompressionThreshold: 16384, // 16KB EnableZstdCompression: false, - EnableDataSharing: false, + EnableDataSharing: true, } } From 1f6c652f9ec4cd289ce0a8a001fe72d3a3c299c2 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Wed, 12 Aug 2026 10:17:06 +0000 Subject: [PATCH 02/18] add dispatcher count in log service Signed-off-by: wk989898 --- coordinator/controller.go | 2 + coordinator/controller_drain.go | 27 +- coordinator/controller_drain_test.go | 51 ++- coordinator/drain/controller.go | 23 ++ coordinator/drain/controller_test.go | 37 ++ heartbeatpb/heartbeat.pb.go | 419 ++++++++++++---------- heartbeatpb/heartbeat.proto | 3 + logservice/eventstore/event_store.go | 9 + logservice/eventstore/event_store_test.go | 4 + maintainer/maintainer_manager_node.go | 21 +- maintainer/node_liveness_test.go | 14 + pkg/eventservice/event_service_test.go | 9 + 12 files changed, 414 insertions(+), 205 deletions(-) diff --git a/coordinator/controller.go b/coordinator/controller.go index 39114c7a7d..682e7e995f 100644 --- a/coordinator/controller.go +++ b/coordinator/controller.go @@ -1163,6 +1163,8 @@ func (c *Controller) RemoveNode(id node.ID) { observation.drainingObserved, observation.stoppingObserved, observation.remaining, + observation.logServiceDispatcherCount, + observation.logServiceDispatcherCountObserved, ) } diff --git a/coordinator/controller_drain.go b/coordinator/controller_drain.go index 86f5889f17..0c0ef47e8e 100644 --- a/coordinator/controller_drain.go +++ b/coordinator/controller_drain.go @@ -205,6 +205,8 @@ func (c *Controller) DrainNode(ctx context.Context, target node.ID) (int, error) observation.drainingObserved, observation.stoppingObserved, observation.remaining, + observation.logServiceDispatcherCount, + observation.logServiceDispatcherCountObserved, ) if completionObserved { @@ -225,6 +227,8 @@ func (c *Controller) DrainNode(ctx context.Context, target node.ID) (int, error) zap.Int("dispatcherCountOnTarget", observation.dispatcherCountOnTarget), zap.Int("targetInflightDrainMoveCount", observation.targetInflightDrainMoveCount), zap.Int("pendingStatusCount", observation.pendingStatusCount), + zap.Uint32("logServiceDispatcherCount", observation.logServiceDispatcherCount), + zap.Bool("logServiceDispatcherCountObserved", observation.logServiceDispatcherCountObserved), zap.Int("remaining", observation.remaining)) return ensureDrainRemainingNonZero(observation.remaining), nil } @@ -240,6 +244,8 @@ func (c *Controller) observeRemovedActiveDrainTarget(target node.ID, epoch uint6 observation.drainingObserved, observation.stoppingObserved, observation.remaining, + observation.logServiceDispatcherCount, + observation.logServiceDispatcherCountObserved, ) if completionObserved { log.Info("drain completion observed for removed active target", @@ -259,6 +265,8 @@ func (c *Controller) observeRemovedActiveDrainTarget(target node.ID, epoch uint6 zap.Int("dispatcherCountOnTarget", observation.dispatcherCountOnTarget), zap.Int("targetInflightDrainMoveCount", observation.targetInflightDrainMoveCount), zap.Int("pendingStatusCount", observation.pendingStatusCount), + zap.Uint32("logServiceDispatcherCount", observation.logServiceDispatcherCount), + zap.Bool("logServiceDispatcherCountObserved", observation.logServiceDispatcherCountObserved), zap.Int("remaining", observation.remaining)) return ensureDrainRemainingNonZero(observation.remaining) } @@ -310,10 +318,12 @@ type drainNodeObservation struct { // pendingStatusCount is the number of running changefeeds not converged to the active target epoch. pendingStatusCount int // remaining is the max of all workload dimensions used by drain completion gating. - remaining int - nodeState drain.State - drainingObserved bool - stoppingObserved bool + remaining int + nodeState drain.State + drainingObserved bool + stoppingObserved bool + logServiceDispatcherCount uint32 + logServiceDispatcherCountObserved bool } func (c *Controller) observeDrainNode(target node.ID, epoch uint64) drainNodeObservation { @@ -332,6 +342,8 @@ func (c *Controller) observeDrainNode(target node.ID, epoch uint64) drainNodeObs ) _, observation.drainingObserved, observation.stoppingObserved = c.drainController.GetStatus(target) + observation.logServiceDispatcherCount, observation.logServiceDispatcherCountObserved = + c.drainController.GetLogServiceDispatcherCount(target) observation.nodeState = c.drainController.GetState(target) return observation } @@ -1082,11 +1094,16 @@ func isBestEffortDrainComplete( drainingObserved bool, stoppingObserved bool, remaining int, + logServiceDispatcherCount uint32, + logServiceDispatcherCountObserved bool, ) bool { if nodeState == drain.StateUnknown || !drainingObserved { return false } - return stoppingObserved && remaining == 0 + return stoppingObserved && + remaining == 0 && + logServiceDispatcherCountObserved && + logServiceDispatcherCount == 0 } // drainRemainingEstimate uses the larger workload dimension to avoid obvious double counting. diff --git a/coordinator/controller_drain_test.go b/coordinator/controller_drain_test.go index cd72ad7ca6..b9f84c7b97 100644 --- a/coordinator/controller_drain_test.go +++ b/coordinator/controller_drain_test.go @@ -154,6 +154,39 @@ func TestDrainNodeCompletesAfterCompletionObserved(t *testing.T) { require.Equal(t, epoch, c.drainSession.epoch) } +func TestDrainNodeWaitsForLogServiceDispatchers(t *testing.T) { + c, drainController, target := newDrainTestController(t) + setDrainProtocolVersion(c, target, heartbeatpb.CurrentDrainProtocolVersion) + cf := addRunningChangefeed(c, "cf1", node.ID("other"), 100) + + remaining, err := c.DrainNode(context.Background(), target) + require.NoError(t, err) + require.Equal(t, 1, remaining) + + _, epoch, ok := c.getDispatcherDrainTarget() + require.True(t, ok) + setChangefeedDrainStatus(cf, target, epoch, 0, 0) + + // A STOPPING response alone does not contain the log service dispatcher count. + drainController.ObserveSetNodeLivenessResponse(target, &heartbeatpb.SetNodeLivenessResponse{ + Applied: heartbeatpb.NodeLiveness_STOPPING, + NodeEpoch: 1, + }) + remaining, err = c.DrainNode(context.Background(), target) + require.NoError(t, err) + require.Equal(t, 1, remaining) + + setTargetStoppingHeartbeat(drainController, target, 2) + remaining, err = c.DrainNode(context.Background(), target) + require.NoError(t, err) + require.Equal(t, 1, remaining) + + setTargetStoppingHeartbeat(drainController, target, 0) + remaining, err = c.DrainNode(context.Background(), target) + require.NoError(t, err) + require.Equal(t, 0, remaining) +} + func TestDrainNodeDispatcherCountBlocksCompletion(t *testing.T) { c, drainController, target := newDrainTestController(t) setDrainProtocolVersion(c, target, heartbeatpb.CurrentDrainProtocolVersion) @@ -1171,11 +1204,19 @@ func setTargetStoppingObserved( drainController *drain.Controller, target node.ID, ) { - resp := &heartbeatpb.SetNodeLivenessResponse{ - Applied: heartbeatpb.NodeLiveness_STOPPING, - NodeEpoch: 1, - } - drainController.ObserveSetNodeLivenessResponse(target, resp) + setTargetStoppingHeartbeat(drainController, target, 0) +} + +func setTargetStoppingHeartbeat( + drainController *drain.Controller, + target node.ID, + logServiceDispatcherCount uint32, +) { + drainController.ObserveHeartbeat(target, &heartbeatpb.NodeHeartbeat{ + Liveness: heartbeatpb.NodeLiveness_STOPPING, + NodeEpoch: 1, + LogServiceDispatcherCount: logServiceDispatcherCount, + }) } func drainMessageChannel(ch chan *messaging.TargetMessage) { diff --git a/coordinator/drain/controller.go b/coordinator/drain/controller.go index 81b0b900ee..4301bbfa3d 100644 --- a/coordinator/drain/controller.go +++ b/coordinator/drain/controller.go @@ -64,6 +64,11 @@ type nodeState struct { lastSeen time.Time nodeEpoch uint64 liveness heartbeatpb.NodeLiveness + + // logServiceDispatcherCount is valid only after a STOPPING heartbeat has + // been observed for the current node epoch. + logServiceDispatcherCount uint32 + logServiceDispatcherCountObserved bool } type drainTargetSchedulerGate struct { @@ -185,6 +190,11 @@ func (c *Controller) ObserveHeartbeat(nodeID node.ID, hb *heartbeatpb.NodeHeartb c.mu.Lock() defer c.mu.Unlock() c.observeLivenessLocked(nodeID, hb.NodeEpoch, hb.Liveness) + st := c.ensureNodeStateLocked(nodeID) + if hb.NodeEpoch == st.nodeEpoch && hb.Liveness == heartbeatpb.NodeLiveness_STOPPING { + st.logServiceDispatcherCount = hb.GetLogServiceDispatcherCount() + st.logServiceDispatcherCountObserved = true + } c.observeTargetSchedulerAckLocked(nodeID, hb) } @@ -415,6 +425,19 @@ func (c *Controller) GetStatus(nodeID node.ID) (drainRequested, drainingObserved return st.drainRequested, st.drainingObserved, st.stoppingObserved } +// GetLogServiceDispatcherCount returns the dispatcher count reported by a +// STOPPING heartbeat for the current node epoch. +func (c *Controller) GetLogServiceDispatcherCount(nodeID node.ID) (uint32, bool) { + c.mu.Lock() + defer c.mu.Unlock() + + st, ok := c.nodes[nodeID] + if !ok || !st.logServiceDispatcherCountObserved { + return 0, false + } + return st.logServiceDispatcherCount, true +} + // GetDrainProtocolVersion returns the bootstrap-observed drain capability for a node. func (c *Controller) GetDrainProtocolVersion(nodeID node.ID) (uint32, bool) { c.mu.Lock() diff --git a/coordinator/drain/controller_test.go b/coordinator/drain/controller_test.go index d02e009693..b6c72f13a0 100644 --- a/coordinator/drain/controller_test.go +++ b/coordinator/drain/controller_test.go @@ -126,6 +126,43 @@ func TestDrainControllerResetObservedStateForNewEpoch(t *testing.T) { c.mu.Unlock() } +func TestDrainControllerTracksStoppingLogServiceDispatcherCountByEpoch(t *testing.T) { + c := NewController(messaging.NewMockMessageCenter()) + target := node.ID("n1") + + c.ObserveSetNodeLivenessResponse(target, &heartbeatpb.SetNodeLivenessResponse{ + Applied: heartbeatpb.NodeLiveness_STOPPING, + NodeEpoch: 42, + }) + _, observed := c.GetLogServiceDispatcherCount(target) + require.False(t, observed) + + c.ObserveHeartbeat(target, &heartbeatpb.NodeHeartbeat{ + Liveness: heartbeatpb.NodeLiveness_STOPPING, + NodeEpoch: 42, + LogServiceDispatcherCount: 2, + }) + count, observed := c.GetLogServiceDispatcherCount(target) + require.True(t, observed) + require.Equal(t, uint32(2), count) + + c.ObserveHeartbeat(target, &heartbeatpb.NodeHeartbeat{ + Liveness: heartbeatpb.NodeLiveness_ALIVE, + NodeEpoch: 43, + }) + _, observed = c.GetLogServiceDispatcherCount(target) + require.False(t, observed) + + // A delayed heartbeat from the old process must not satisfy the new epoch. + c.ObserveHeartbeat(target, &heartbeatpb.NodeHeartbeat{ + Liveness: heartbeatpb.NodeLiveness_STOPPING, + NodeEpoch: 42, + LogServiceDispatcherCount: 0, + }) + _, observed = c.GetLogServiceDispatcherCount(target) + require.False(t, observed) +} + func TestDrainControllerSkipStoppingForNewEpochWithoutDraining(t *testing.T) { mc := messaging.NewMockMessageCenter() c := NewController(mc) diff --git a/heartbeatpb/heartbeat.pb.go b/heartbeatpb/heartbeat.pb.go index 94ff2f969d..02015a4434 100644 --- a/heartbeatpb/heartbeat.pb.go +++ b/heartbeatpb/heartbeat.pb.go @@ -1625,6 +1625,9 @@ type NodeHeartbeat struct { // currently applied on this node. Empty target means the drain target is clear. DispatcherDrainTargetNodeId string `protobuf:"bytes,3,opt,name=dispatcher_drain_target_node_id,json=dispatcherDrainTargetNodeId,proto3" json:"dispatcher_drain_target_node_id,omitempty"` DispatcherDrainTargetEpoch uint64 `protobuf:"varint,4,opt,name=dispatcher_drain_target_epoch,json=dispatcherDrainTargetEpoch,proto3" json:"dispatcher_drain_target_epoch,omitempty"` + // log_service_dispatcher_count is the number of dispatchers currently + // reading from this node's event store. + LogServiceDispatcherCount uint32 `protobuf:"varint,5,opt,name=log_service_dispatcher_count,json=logServiceDispatcherCount,proto3" json:"log_service_dispatcher_count,omitempty"` } func (m *NodeHeartbeat) Reset() { *m = NodeHeartbeat{} } @@ -1688,6 +1691,13 @@ func (m *NodeHeartbeat) GetDispatcherDrainTargetEpoch() uint64 { return 0 } +func (m *NodeHeartbeat) GetLogServiceDispatcherCount() uint32 { + if m != nil { + return m.LogServiceDispatcherCount + } + return 0 +} + // SetNodeLivenessRequest asks a node to transition its local liveness. type SetNodeLivenessRequest struct { Target NodeLiveness `protobuf:"varint,1,opt,name=target,proto3,enum=heartbeatpb.NodeLiveness" json:"target,omitempty"` @@ -4006,196 +4016,198 @@ func init() { func init() { proto.RegisterFile("heartbeatpb/heartbeat.proto", fileDescriptor_6d584080fdadb670) } var fileDescriptor_6d584080fdadb670 = []byte{ - // 3020 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x1a, 0x4d, 0x8f, 0x1c, 0x47, - 0xd5, 0xdd, 0xf3, 0xfd, 0x76, 0x67, 0xb7, 0x5d, 0xb6, 0xd7, 0x6b, 0x7b, 0xbd, 0xde, 0x74, 0x02, - 0xda, 0x4c, 0x82, 0x8d, 0x9d, 0x98, 0x8f, 0x10, 0x12, 0xc6, 0x33, 0x9b, 0x78, 0xe4, 0x9d, 0xdd, - 0x55, 0xcd, 0x26, 0x46, 0xe1, 0x30, 0xf4, 0x76, 0x97, 0x67, 0x3b, 0x3b, 0xd3, 0x35, 0xe9, 0xee, - 0xf1, 0xda, 0x96, 0x00, 0x45, 0x88, 0x1b, 0x07, 0xb8, 0x71, 0x20, 0x17, 0x4e, 0x9c, 0x10, 0x7f, - 0x00, 0xc1, 0x81, 0x03, 0x27, 0x14, 0x71, 0x40, 0x39, 0x41, 0x94, 0xdc, 0x11, 0x12, 0x12, 0x5c, - 0x51, 0x7d, 0x74, 0x77, 0xf5, 0x4c, 0xcf, 0x7e, 0xb0, 0xa3, 0x88, 0xd3, 0xf4, 0x7b, 0xf5, 0xde, - 0xab, 0x57, 0xaf, 0x5e, 0xbd, 0x7a, 0xef, 0xd5, 0xc0, 0xb5, 0x7d, 0x62, 0xf9, 0xe1, 0x1e, 0xb1, - 0xc2, 0xe1, 0xde, 0xad, 0xf8, 0xfb, 0xe6, 0xd0, 0xa7, 0x21, 0x45, 0x73, 0xca, 0xa0, 0xf9, 0x14, - 0x2a, 0xbb, 0xd6, 0x5e, 0x9f, 0x74, 0x86, 0x96, 0x87, 0x96, 0xa1, 0xc4, 0x81, 0x56, 0x73, 0x59, - 0x5b, 0xd3, 0xd6, 0x73, 0x38, 0x02, 0xd1, 0x55, 0x28, 0x77, 0x42, 0xcb, 0x0f, 0x1f, 0x90, 0xa7, - 0xcb, 0xfa, 0x9a, 0xb6, 0x3e, 0x8f, 0x63, 0x18, 0x2d, 0x41, 0x71, 0xc3, 0x73, 0xd8, 0x48, 0x8e, - 0x8f, 0x48, 0x08, 0xad, 0x02, 0x3c, 0x20, 0x4f, 0x83, 0xa1, 0x65, 0x33, 0x81, 0xf9, 0x35, 0x6d, - 0xbd, 0x8a, 0x15, 0x8c, 0xf9, 0x57, 0x1d, 0x8c, 0xfb, 0x4c, 0x95, 0x7b, 0xc4, 0x0a, 0x31, 0xf9, - 0x60, 0x44, 0x82, 0x10, 0x7d, 0x1b, 0xe6, 0xed, 0x7d, 0xcb, 0xeb, 0x91, 0x47, 0x84, 0x38, 0x52, - 0x8f, 0xb9, 0x3b, 0x57, 0x6e, 0x2a, 0x3a, 0xdf, 0x6c, 0x28, 0x04, 0x38, 0x45, 0x8e, 0x5e, 0x85, - 0xca, 0xa1, 0x15, 0x12, 0x7f, 0x60, 0xf9, 0x07, 0x5c, 0xd1, 0xb9, 0x3b, 0x4b, 0x29, 0xde, 0x87, - 0xd1, 0x28, 0x4e, 0x08, 0xd1, 0xeb, 0x50, 0xf5, 0x89, 0x43, 0xe3, 0x31, 0xbe, 0x90, 0xe9, 0x9c, - 0x69, 0x62, 0xf4, 0x0d, 0x28, 0x07, 0xa1, 0x15, 0x8e, 0x02, 0x12, 0x2c, 0xe7, 0xd7, 0x72, 0xeb, - 0x73, 0x77, 0x56, 0x52, 0x8c, 0xb1, 0x7d, 0x3b, 0x9c, 0x0a, 0xc7, 0xd4, 0x68, 0x1d, 0x16, 0x6d, - 0x3a, 0x18, 0x92, 0x3e, 0x09, 0x89, 0x18, 0x5c, 0x2e, 0xac, 0x69, 0xeb, 0x65, 0x3c, 0x8e, 0x46, - 0x2f, 0x41, 0x8e, 0xf8, 0xfe, 0x72, 0x31, 0xc3, 0x1a, 0x78, 0xe4, 0x79, 0xae, 0xd7, 0xdb, 0xf0, - 0x7d, 0xea, 0x63, 0x46, 0x65, 0xfe, 0x44, 0x83, 0x4a, 0xa2, 0x9e, 0xc9, 0x2c, 0x4a, 0xec, 0x83, - 0x21, 0x75, 0xbd, 0x70, 0x37, 0xe0, 0x16, 0xcd, 0xe3, 0x14, 0x8e, 0x6d, 0x95, 0x4f, 0x02, 0xda, - 0x7f, 0x4c, 0x9c, 0xdd, 0x80, 0xdb, 0x2d, 0x8f, 0x15, 0x0c, 0x32, 0x20, 0x17, 0x90, 0x0f, 0xb8, - 0x59, 0xf2, 0x98, 0x7d, 0x32, 0xa9, 0x7d, 0x2b, 0x08, 0x3b, 0x4f, 0x3d, 0x9b, 0xf3, 0xe4, 0x85, - 0x54, 0x15, 0x67, 0xfe, 0x00, 0x8c, 0xa6, 0x1b, 0x0c, 0xad, 0xd0, 0xde, 0x27, 0x7e, 0xdd, 0x0e, - 0x5d, 0xea, 0xa1, 0x97, 0xa0, 0x68, 0xf1, 0x2f, 0xae, 0xc7, 0xc2, 0x9d, 0x0b, 0xa9, 0xb5, 0x08, - 0x22, 0x2c, 0x49, 0x98, 0xd7, 0x35, 0xe8, 0x60, 0xe0, 0x86, 0xb1, 0x52, 0x31, 0x8c, 0xd6, 0x60, - 0xae, 0x15, 0xb0, 0xa9, 0x76, 0xd8, 0x1a, 0xb8, 0x6a, 0x65, 0xac, 0xa2, 0xcc, 0x06, 0xe4, 0xea, - 0x8d, 0x07, 0x29, 0x21, 0xda, 0xd1, 0x42, 0xf4, 0x49, 0x21, 0x18, 0x50, 0xab, 0xe7, 0x51, 0x9f, - 0x38, 0xf7, 0xfa, 0xd4, 0x3e, 0x90, 0xdb, 0x71, 0x36, 0x99, 0x3f, 0xd6, 0xe1, 0x52, 0xcb, 0x7b, - 0xd4, 0x1f, 0x11, 0x66, 0xa8, 0xc4, 0x44, 0x01, 0xfa, 0x0e, 0x54, 0xe3, 0x81, 0xdd, 0xa7, 0x43, - 0x22, 0x8d, 0x74, 0x35, 0x65, 0xa4, 0x14, 0x05, 0x4e, 0x33, 0xa0, 0x37, 0xa1, 0x9a, 0x08, 0x6c, - 0x35, 0x99, 0xdd, 0x72, 0x13, 0x2e, 0xa3, 0x52, 0xe0, 0x34, 0x3d, 0x3f, 0xe9, 0xf6, 0x3e, 0x19, - 0x58, 0xad, 0x26, 0x37, 0x6a, 0x0e, 0xc7, 0x30, 0x7a, 0x00, 0x17, 0xc8, 0x13, 0xbb, 0x3f, 0x72, - 0x88, 0xc2, 0xe3, 0xf0, 0xbd, 0x3f, 0x72, 0x8a, 0x2c, 0x2e, 0xf3, 0x17, 0xba, 0xea, 0x1e, 0xd2, - 0xb0, 0xdf, 0x85, 0x4b, 0x6e, 0x96, 0x65, 0x64, 0x1c, 0x30, 0xb3, 0x0d, 0xa1, 0x52, 0xe2, 0x6c, - 0x01, 0xe8, 0x6e, 0xec, 0x78, 0x22, 0x2c, 0x5c, 0x9f, 0xa2, 0xee, 0x98, 0x0b, 0x9a, 0x90, 0xb3, - 0xec, 0x28, 0x20, 0x18, 0x69, 0x67, 0x6d, 0x3c, 0xc0, 0x6c, 0x10, 0x6d, 0x03, 0x72, 0x27, 0x7c, - 0x44, 0x5a, 0xe5, 0x46, 0x5a, 0xe3, 0x09, 0x32, 0x9c, 0xc1, 0x6a, 0x7e, 0xaa, 0xc1, 0x79, 0x25, - 0x32, 0x06, 0x43, 0xea, 0x05, 0xe4, 0xac, 0xa1, 0xb1, 0x0d, 0xc8, 0x19, 0x33, 0x37, 0x89, 0xdc, - 0x63, 0x9a, 0x31, 0x22, 0x1d, 0x27, 0x19, 0x11, 0x82, 0xfc, 0x80, 0x3a, 0x44, 0xfa, 0x08, 0xff, - 0x46, 0x2f, 0x82, 0x31, 0xb0, 0x5c, 0x2f, 0xb4, 0x5c, 0x8f, 0xf8, 0x5d, 0x32, 0xa4, 0xf6, 0xbe, - 0x0c, 0x0c, 0x8b, 0x09, 0x7e, 0x83, 0xa1, 0xcd, 0x27, 0x70, 0xa1, 0xa1, 0x44, 0xa0, 0x36, 0x09, - 0x02, 0xab, 0x77, 0xe6, 0x35, 0x8e, 0xc7, 0x3a, 0x7d, 0x32, 0xd6, 0x99, 0xbf, 0xd7, 0x60, 0x11, - 0x13, 0x87, 0xb6, 0x49, 0x68, 0xcd, 0x68, 0xda, 0xe3, 0xc2, 0xe7, 0xb8, 0x5a, 0xb9, 0x8c, 0x10, - 0x7c, 0x0a, 0xdb, 0xfd, 0x10, 0xae, 0xb3, 0x05, 0xe0, 0x78, 0x82, 0x1d, 0x9f, 0xf6, 0x7c, 0x12, - 0x04, 0x5f, 0xcc, 0x72, 0xcc, 0x5f, 0x6b, 0xb0, 0x92, 0x56, 0xe0, 0x2d, 0xea, 0x1f, 0x5a, 0xbe, - 0xf3, 0x05, 0x99, 0x33, 0xcb, 0x54, 0xb9, 0x6c, 0x53, 0xfd, 0x4b, 0x53, 0x83, 0x4c, 0x83, 0x7a, - 0x8f, 0xdc, 0x1e, 0xaa, 0x41, 0x3e, 0x18, 0x5a, 0x9e, 0x54, 0x6b, 0x29, 0xfb, 0xb2, 0xc6, 0x9c, - 0x86, 0xa5, 0x44, 0x01, 0x4b, 0x74, 0x62, 0x45, 0x22, 0x90, 0x2d, 0xd2, 0x51, 0x82, 0x9c, 0x0c, - 0x11, 0x47, 0x44, 0xc1, 0x14, 0x39, 0x8b, 0xb3, 0x41, 0x14, 0x67, 0xf3, 0x22, 0xce, 0x46, 0x70, - 0x7c, 0xb6, 0x0a, 0xca, 0xd9, 0xaa, 0x81, 0x11, 0x1c, 0xb8, 0xc3, 0x66, 0x7b, 0xb3, 0x1e, 0x74, - 0xa4, 0x46, 0x45, 0x7e, 0xb7, 0x4c, 0xe0, 0xcd, 0x3f, 0xe8, 0x70, 0x85, 0x05, 0x6d, 0x67, 0xd4, - 0x57, 0x62, 0xee, 0x8c, 0x52, 0xac, 0xbb, 0x50, 0xb4, 0xb9, 0x1d, 0x8f, 0x09, 0xa4, 0xc2, 0xd8, - 0x58, 0x12, 0xa3, 0x06, 0x2c, 0x04, 0x52, 0x25, 0x11, 0x62, 0xb9, 0xc1, 0x16, 0xee, 0x5c, 0x4b, - 0xb1, 0x77, 0x52, 0x24, 0x78, 0x8c, 0x85, 0xa9, 0x4e, 0x87, 0xc4, 0xb7, 0x42, 0xea, 0xf3, 0xeb, - 0x31, 0xcf, 0x45, 0xa4, 0x55, 0xdf, 0x56, 0x08, 0x70, 0x8a, 0x3c, 0xd3, 0x71, 0x0a, 0xd9, 0x8e, - 0xf3, 0x2b, 0x1d, 0x96, 0xda, 0xc4, 0xef, 0xcd, 0xde, 0x7e, 0x6f, 0x42, 0xd5, 0x39, 0xe5, 0x0d, - 0x9d, 0xa2, 0x47, 0x2d, 0x40, 0x03, 0xa6, 0x99, 0xd3, 0x3c, 0x95, 0xfb, 0x65, 0x30, 0xc5, 0x8e, - 0x96, 0x3f, 0x26, 0x88, 0x4f, 0x31, 0xd2, 0x0e, 0x5c, 0x68, 0xc7, 0xa8, 0xfb, 0xd1, 0xc4, 0xe8, - 0x9b, 0x4a, 0x42, 0xac, 0x65, 0xdc, 0x2f, 0x09, 0xcf, 0x78, 0x46, 0x6c, 0x7e, 0xa2, 0x41, 0xb5, - 0xe9, 0x5b, 0xae, 0x17, 0x85, 0x34, 0xf4, 0x02, 0x2c, 0x84, 0x96, 0xdf, 0x23, 0x61, 0xd7, 0xa3, - 0x0e, 0xe9, 0xba, 0x0e, 0xb7, 0x77, 0x05, 0xcf, 0x0b, 0xec, 0x16, 0x75, 0x48, 0xcb, 0x41, 0xcf, - 0x81, 0x84, 0xa5, 0xc2, 0xe2, 0xac, 0xce, 0x09, 0x1c, 0x57, 0x16, 0x7d, 0x0d, 0x2e, 0x4b, 0x92, - 0xc4, 0x9c, 0x5d, 0x9b, 0x8e, 0x64, 0xf2, 0x58, 0xc5, 0x97, 0xc4, 0xb0, 0xea, 0xc1, 0x23, 0x2f, - 0x44, 0x6f, 0xc1, 0x9a, 0xe4, 0x63, 0x89, 0x85, 0xdb, 0xdb, 0x0f, 0xbb, 0x0e, 0xd3, 0xb0, 0x3b, - 0xa0, 0x8f, 0x89, 0x14, 0x20, 0x8a, 0x9b, 0x15, 0x41, 0xd7, 0x92, 0x64, 0x7c, 0x1d, 0x6d, 0xfa, - 0x98, 0x70, 0x39, 0xe6, 0x6f, 0x72, 0x60, 0x8c, 0xaf, 0xfc, 0xac, 0xbe, 0x74, 0x1d, 0x80, 0x7d, - 0x75, 0x99, 0xfd, 0x08, 0x5f, 0x74, 0x05, 0x57, 0x18, 0x86, 0x89, 0x27, 0xe8, 0x36, 0x14, 0xc4, - 0x48, 0xd6, 0x51, 0x6b, 0xd0, 0xc1, 0x90, 0x7a, 0xc4, 0x0b, 0x39, 0x2d, 0x16, 0x94, 0xe8, 0x79, - 0xa8, 0x26, 0xd7, 0x52, 0x37, 0x8c, 0x13, 0xfb, 0xd4, 0x5d, 0x25, 0xab, 0x91, 0x42, 0x86, 0xe3, - 0x4e, 0x54, 0x23, 0xe8, 0x4b, 0xb0, 0xb0, 0x47, 0x69, 0x18, 0x84, 0xbe, 0x35, 0xec, 0x3a, 0xd4, - 0x23, 0x32, 0x6c, 0x55, 0x63, 0x6c, 0x93, 0x7a, 0x64, 0xa2, 0xa0, 0x28, 0x4d, 0x16, 0x14, 0xa8, - 0x0e, 0x0b, 0xc2, 0xf4, 0x43, 0xe9, 0x1d, 0xcb, 0x65, 0x6e, 0xaf, 0x74, 0x7e, 0x9c, 0xf2, 0x1f, - 0x5c, 0x75, 0x52, 0xee, 0x94, 0xe5, 0xdd, 0x95, 0x6c, 0xef, 0xfe, 0x87, 0x06, 0x55, 0xe6, 0x5e, - 0x89, 0x63, 0xdf, 0x85, 0x72, 0xdf, 0x7d, 0x4c, 0x3c, 0x36, 0xb3, 0x96, 0x11, 0x7a, 0x18, 0xf5, - 0xa6, 0x24, 0xc0, 0x31, 0x29, 0xdb, 0x25, 0xee, 0xbb, 0xaa, 0x6b, 0x56, 0x18, 0x46, 0x38, 0x66, - 0x13, 0x6e, 0x28, 0x1e, 0x29, 0x16, 0x38, 0xe6, 0xf2, 0x39, 0xbe, 0xb3, 0xd7, 0x12, 0x32, 0xbe, - 0xc6, 0x5d, 0xf5, 0x04, 0xd4, 0xe1, 0xfa, 0x34, 0x29, 0x6a, 0x32, 0x71, 0x35, 0x53, 0x86, 0x58, - 0xf0, 0xfb, 0xb0, 0xd4, 0x11, 0xf2, 0xe2, 0x45, 0xc8, 0x90, 0x77, 0x1b, 0x8a, 0x42, 0xd6, 0xf1, - 0xcb, 0x96, 0x84, 0xc7, 0x2c, 0xda, 0x1c, 0xc0, 0xe5, 0x89, 0xb9, 0x64, 0x9e, 0xfb, 0x0a, 0x94, - 0xac, 0xe1, 0xb0, 0xef, 0x12, 0xe7, 0xf8, 0xd9, 0x22, 0xca, 0xe3, 0xa6, 0x7b, 0x1f, 0x6e, 0x74, - 0xd4, 0xa3, 0xad, 0xac, 0x3d, 0x5a, 0xe3, 0xac, 0x02, 0x8d, 0xf9, 0x75, 0xb8, 0xd6, 0xa0, 0xd4, - 0x77, 0x5c, 0x8f, 0x5d, 0x3c, 0xf7, 0x22, 0x2f, 0x8f, 0xe6, 0x59, 0x86, 0xd2, 0x63, 0xe2, 0x07, - 0x51, 0x09, 0x9c, 0xc3, 0x11, 0xc8, 0x2a, 0xa2, 0x95, 0x6c, 0x4e, 0x69, 0x99, 0xff, 0x3d, 0xb0, - 0xa2, 0x57, 0x61, 0x29, 0x3e, 0x3a, 0x21, 0xb5, 0x69, 0xbf, 0x1b, 0x29, 0xa1, 0xf3, 0xd8, 0x75, - 0x31, 0x3a, 0x26, 0x7c, 0xf0, 0x5d, 0x31, 0xf6, 0xff, 0xe3, 0x9a, 0xff, 0xd6, 0xe0, 0x62, 0xdd, - 0x71, 0x92, 0x05, 0x46, 0xd6, 0x7c, 0x11, 0x74, 0xb9, 0x53, 0x47, 0x86, 0x4d, 0xdd, 0x75, 0xd0, - 0x52, 0x2a, 0x71, 0x99, 0x8f, 0x33, 0x93, 0x89, 0x90, 0x97, 0x95, 0x9e, 0xd7, 0xe0, 0xbc, 0x1b, - 0x74, 0x3d, 0x72, 0xd8, 0x4d, 0x02, 0x30, 0xd7, 0xbb, 0x8c, 0x17, 0xdd, 0x60, 0x8b, 0x1c, 0x26, - 0xd3, 0xa1, 0x1b, 0x30, 0x77, 0x20, 0xdb, 0x5c, 0xcc, 0x42, 0x05, 0xd1, 0xf9, 0x8a, 0x50, 0x2d, - 0x27, 0x33, 0x08, 0x15, 0xb3, 0x83, 0xd0, 0x1f, 0x35, 0xb8, 0x8c, 0x09, 0xbb, 0x6a, 0xce, 0xb4, - 0xf6, 0x65, 0x28, 0xd9, 0x56, 0x60, 0x5b, 0x0e, 0x91, 0x0d, 0x89, 0x08, 0x64, 0x23, 0x3e, 0x97, - 0xef, 0xc8, 0x1e, 0x4a, 0x04, 0x8e, 0x2f, 0x23, 0x7f, 0xa2, 0x65, 0x4c, 0xc9, 0x14, 0xfe, 0x9c, - 0x83, 0xab, 0xc9, 0x02, 0x26, 0xce, 0xc4, 0x19, 0xaf, 0xc1, 0x69, 0x3b, 0x7b, 0x85, 0x9f, 0x17, - 0x5f, 0xd9, 0xd4, 0x38, 0x7b, 0xb7, 0xe1, 0xb9, 0x90, 0xa5, 0xfa, 0xdd, 0xd0, 0x77, 0x7b, 0x3d, - 0xa6, 0xfe, 0x63, 0xe2, 0xa5, 0x52, 0x03, 0xf7, 0x04, 0x8d, 0x8d, 0xeb, 0x5c, 0xc6, 0xae, 0x10, - 0xb1, 0xc1, 0x24, 0xa8, 0x2d, 0x8e, 0x6c, 0xa7, 0x29, 0x64, 0x3b, 0x8d, 0xc5, 0xd2, 0x0c, 0x55, - 0x21, 0x9f, 0x38, 0x74, 0x4c, 0x9f, 0xe2, 0x71, 0xfa, 0xac, 0xa8, 0xfa, 0xb0, 0x12, 0x2d, 0xa5, - 0xce, 0xd8, 0x86, 0x96, 0x4e, 0xb4, 0xa1, 0xe5, 0xec, 0x0d, 0xfd, 0x4b, 0x0e, 0xae, 0x65, 0x6e, - 0xe8, 0x6c, 0x9a, 0x15, 0x77, 0xa1, 0xc0, 0xca, 0xaf, 0x28, 0x39, 0x4e, 0x77, 0x51, 0xe2, 0xd9, - 0x92, 0x62, 0x4d, 0x50, 0x47, 0x89, 0x49, 0xee, 0x24, 0x6d, 0xd2, 0x93, 0xa5, 0x3a, 0x2f, 0x03, - 0xe2, 0x1b, 0x91, 0xa6, 0x14, 0x5e, 0x6e, 0xb0, 0x11, 0xb5, 0x8b, 0x81, 0x9a, 0x50, 0x89, 0x0a, - 0x0e, 0x56, 0x9d, 0x31, 0xd5, 0xbf, 0x9c, 0x59, 0xdf, 0x4c, 0x54, 0x15, 0x38, 0x61, 0xcc, 0xdc, - 0x86, 0x52, 0xe6, 0x36, 0xa0, 0x4d, 0x58, 0xe4, 0x69, 0x7d, 0x37, 0x99, 0xb6, 0xcc, 0xa7, 0x7d, - 0x3e, 0x7d, 0x31, 0x64, 0x56, 0x32, 0x78, 0x81, 0xf3, 0x46, 0x05, 0x53, 0x60, 0xfe, 0x4d, 0x87, - 0xd5, 0x64, 0x53, 0x77, 0x68, 0x10, 0xce, 0xfa, 0xa4, 0x9e, 0xe8, 0xd8, 0xe9, 0x67, 0x3c, 0x76, - 0xb7, 0xa1, 0x24, 0x4a, 0x69, 0x76, 0xea, 0x99, 0x31, 0x2e, 0x4f, 0xec, 0xc1, 0xc0, 0x6a, 0x79, - 0x8f, 0x28, 0x8e, 0xe8, 0xd0, 0x6b, 0x30, 0xcf, 0xb7, 0x39, 0xe2, 0xcb, 0x1f, 0xcd, 0x37, 0xc7, - 0x88, 0x3b, 0x92, 0xf7, 0x14, 0x61, 0xf0, 0x23, 0x1d, 0x6e, 0x4c, 0x35, 0xf0, 0x6c, 0x4e, 0xce, - 0x17, 0x62, 0xe1, 0x53, 0x9d, 0xb3, 0x53, 0x75, 0x05, 0x21, 0xb1, 0x72, 0xaa, 0x15, 0xad, 0x8d, - 0xb5, 0xa2, 0x57, 0x23, 0xca, 0x2d, 0x6b, 0x10, 0x55, 0x3e, 0x0a, 0x06, 0xdd, 0x84, 0x22, 0x8f, - 0x0e, 0x91, 0x0b, 0x64, 0x74, 0x79, 0xf8, 0x4e, 0x4a, 0x2a, 0xb3, 0x21, 0xdf, 0xc1, 0xf8, 0xc4, - 0xd3, 0xdf, 0xc1, 0x56, 0x24, 0x99, 0x32, 0x6b, 0x82, 0x30, 0x7f, 0xa7, 0x03, 0x9a, 0x0c, 0x4e, - 0xec, 0x9e, 0x9e, 0xb2, 0x8f, 0x29, 0x9b, 0xeb, 0xf2, 0x9d, 0x2d, 0x5a, 0xb2, 0x3e, 0xb6, 0xe4, - 0xa8, 0x6d, 0x95, 0x3b, 0x41, 0xdb, 0xea, 0x2d, 0x30, 0xec, 0xa8, 0xbe, 0xeb, 0x06, 0x49, 0x43, - 0xfa, 0x98, 0x22, 0x70, 0xd1, 0x56, 0xe1, 0x51, 0x30, 0x19, 0x23, 0x0b, 0x19, 0x31, 0xf2, 0x15, - 0x98, 0xdb, 0xeb, 0x53, 0xfb, 0x40, 0x96, 0xa1, 0xe2, 0x96, 0x42, 0xe9, 0xb3, 0xc3, 0xc5, 0xc3, - 0x5e, 0xd4, 0xe4, 0x26, 0x71, 0xeb, 0xa1, 0x94, 0xb4, 0x1e, 0xcc, 0x5f, 0x6a, 0xb0, 0x94, 0x1c, - 0x8f, 0x46, 0x9f, 0x06, 0x64, 0x46, 0x71, 0x47, 0xc9, 0x72, 0xf4, 0x74, 0x96, 0x73, 0x8a, 0x66, - 0xe2, 0x47, 0x1a, 0x5c, 0x9e, 0x50, 0x6f, 0x36, 0xa7, 0x76, 0x19, 0x4a, 0xc1, 0xc8, 0xb6, 0x59, - 0x61, 0x29, 0xf5, 0x93, 0xe0, 0x69, 0xf4, 0xfb, 0xa9, 0x06, 0x46, 0xf2, 0x26, 0x22, 0x1c, 0x7b, - 0x06, 0x4f, 0x4a, 0x57, 0xa1, 0x2c, 0xdd, 0x5f, 0x5c, 0xc7, 0x39, 0x1c, 0xc3, 0x47, 0xbd, 0x16, - 0x99, 0xdf, 0x83, 0x02, 0xa7, 0x3b, 0xe6, 0x59, 0x79, 0x9a, 0xbb, 0xaf, 0x40, 0xa5, 0x33, 0xec, - 0xbb, 0x3c, 0x10, 0xc9, 0xd4, 0x34, 0x41, 0x98, 0x1f, 0xea, 0x70, 0x01, 0xd3, 0x51, 0x48, 0xb8, - 0xa8, 0xba, 0x33, 0x70, 0x03, 0x5e, 0xb1, 0xd4, 0xc0, 0xe8, 0xd0, 0x91, 0x6f, 0x13, 0x25, 0x3a, - 0x88, 0x3a, 0x6e, 0x02, 0x8f, 0xd6, 0x61, 0x51, 0xe0, 0xc6, 0x8f, 0xf4, 0x38, 0x9a, 0x49, 0x15, - 0xd5, 0x88, 0x22, 0x55, 0x14, 0x3e, 0x13, 0x78, 0x26, 0x55, 0xe0, 0x12, 0xa9, 0x79, 0x21, 0x75, - 0x0c, 0x8d, 0xde, 0x80, 0xa2, 0x6c, 0x85, 0x16, 0xf8, 0x9e, 0xa4, 0x53, 0x85, 0x8c, 0xd5, 0x45, - 0x6f, 0x53, 0xe2, 0xd7, 0xf4, 0x60, 0x21, 0xb2, 0x96, 0x70, 0xad, 0x23, 0x2c, 0xbd, 0x06, 0x73, - 0xdb, 0x7d, 0x67, 0xcc, 0xd8, 0x2a, 0x8a, 0x51, 0x6c, 0x91, 0xc3, 0xb1, 0xdd, 0x54, 0x51, 0xe6, - 0x7f, 0x72, 0x50, 0x10, 0x87, 0x77, 0x05, 0x2a, 0xad, 0x80, 0xbf, 0x58, 0xc9, 0x22, 0xbd, 0x8c, - 0x13, 0x04, 0xd3, 0x82, 0x7f, 0x26, 0x3d, 0x73, 0x09, 0xa2, 0x37, 0x61, 0x4e, 0x7c, 0x46, 0xa1, - 0x79, 0xb2, 0x81, 0x3c, 0xee, 0xc0, 0x58, 0xe5, 0x40, 0x0f, 0xe0, 0xfc, 0x16, 0x21, 0x4e, 0xd3, - 0xa7, 0xc3, 0x61, 0x44, 0x21, 0xd3, 0xf4, 0x63, 0xc4, 0x4c, 0xf2, 0xa1, 0xd7, 0x61, 0x91, 0x21, - 0xeb, 0x8e, 0x13, 0x8b, 0x12, 0x2d, 0x2d, 0x34, 0x19, 0x5b, 0xf1, 0x38, 0x29, 0x6a, 0xc0, 0xc2, - 0x3b, 0x43, 0xc7, 0x0a, 0x89, 0x34, 0x61, 0x94, 0xf0, 0x5d, 0xcb, 0x4a, 0x1a, 0xe4, 0x06, 0xe1, - 0x31, 0x96, 0xf1, 0xc7, 0xe2, 0xd2, 0xc4, 0x63, 0x31, 0xfa, 0x0a, 0xef, 0xe1, 0xf5, 0x08, 0x4f, - 0xc4, 0x17, 0xc6, 0x52, 0x92, 0xe8, 0xd1, 0xb0, 0x27, 0xfa, 0x77, 0x3d, 0x82, 0x76, 0xe1, 0x62, - 0x86, 0xe3, 0x04, 0xcb, 0x15, 0xae, 0xdb, 0xda, 0x71, 0x1e, 0x86, 0x33, 0xb9, 0xcd, 0x1f, 0xc1, - 0xc5, 0xf8, 0x86, 0x51, 0xdf, 0xc1, 0x4f, 0x71, 0xb3, 0xad, 0x47, 0xbd, 0x48, 0x7d, 0xea, 0xf5, - 0x20, 0x5b, 0x90, 0x19, 0x2f, 0x8b, 0xe6, 0x3f, 0x35, 0x76, 0xaa, 0x52, 0xff, 0xa3, 0x38, 0xcd, - 0xe4, 0x59, 0xd7, 0xa1, 0x3e, 0x8b, 0xeb, 0x30, 0xab, 0x55, 0x70, 0x1b, 0x2e, 0x89, 0x9c, 0x2b, - 0x70, 0x9f, 0x91, 0xee, 0x90, 0xf8, 0xdd, 0x80, 0xd8, 0xd4, 0x13, 0xe5, 0xa4, 0x8e, 0x11, 0x1f, - 0xec, 0xb8, 0xcf, 0xc8, 0x0e, 0xf1, 0x3b, 0x7c, 0x24, 0xeb, 0xc1, 0xc7, 0xfc, 0xad, 0x06, 0x48, - 0x7d, 0x28, 0x9e, 0xcd, 0x45, 0xf8, 0x36, 0x54, 0xf7, 0x12, 0xa1, 0xf1, 0x03, 0xf0, 0x73, 0xd9, - 0xd9, 0x84, 0x3a, 0x7f, 0x9a, 0x2f, 0x73, 0x97, 0x1c, 0x98, 0x57, 0xd3, 0x3f, 0x46, 0x13, 0xba, - 0x71, 0x00, 0xe6, 0xdf, 0x0c, 0xe7, 0x51, 0x27, 0x8a, 0xb4, 0xfc, 0x9b, 0xe1, 0xec, 0x48, 0x56, - 0x05, 0xf3, 0x6f, 0x16, 0x44, 0x06, 0xe2, 0x39, 0x51, 0x86, 0xcf, 0x08, 0x34, 0x5f, 0x85, 0xf9, - 0xf1, 0x47, 0x8c, 0x7d, 0xb7, 0xb7, 0x2f, 0xff, 0x88, 0xc1, 0xbf, 0x91, 0x01, 0xb9, 0x3e, 0x3d, - 0x94, 0xe1, 0x87, 0x7d, 0x32, 0xdd, 0x54, 0xb3, 0x9c, 0x8c, 0x8b, 0x6b, 0x9b, 0x04, 0x7b, 0xfe, - 0xcd, 0x2e, 0xad, 0xa8, 0x66, 0x96, 0xaa, 0xc5, 0xb0, 0xf9, 0x7d, 0xb8, 0xb1, 0x49, 0x7b, 0x4a, - 0x13, 0x2f, 0x79, 0x23, 0x9d, 0xcd, 0x06, 0x9a, 0x1f, 0x6a, 0xb0, 0x36, 0x7d, 0x8a, 0xd9, 0x64, - 0x23, 0xc7, 0x3d, 0x00, 0xf7, 0x99, 0x2d, 0x89, 0x7d, 0x10, 0x8c, 0x06, 0x6d, 0x12, 0x5a, 0xe8, - 0xab, 0xd1, 0xd9, 0xce, 0xca, 0x2d, 0x22, 0xca, 0xd4, 0x19, 0xaf, 0x81, 0x61, 0xab, 0xf8, 0x0e, - 0xf9, 0x40, 0xce, 0x33, 0x81, 0x37, 0x7f, 0xae, 0xc1, 0x25, 0xe5, 0x2f, 0x09, 0x24, 0x8c, 0x24, - 0xa2, 0x8b, 0x50, 0x10, 0xef, 0x2f, 0x62, 0x13, 0x05, 0xc0, 0x3c, 0xe7, 0x09, 0xf5, 0xef, 0xb3, - 0xcd, 0x95, 0xd7, 0x8f, 0x04, 0xd1, 0x12, 0x14, 0x9f, 0x50, 0x7f, 0x93, 0x1e, 0xca, 0x73, 0x2b, - 0x21, 0x91, 0x7d, 0x0d, 0x38, 0x47, 0x5e, 0xb6, 0x89, 0x04, 0xc8, 0x38, 0x82, 0xd1, 0x80, 0x71, - 0x88, 0xc4, 0x57, 0x42, 0x2c, 0x15, 0x5c, 0xcb, 0xd4, 0xa9, 0x6e, 0x1f, 0xcc, 0x6a, 0x17, 0x2e, - 0x42, 0x41, 0xed, 0x31, 0x0b, 0x20, 0xf3, 0x7f, 0x17, 0xf2, 0xef, 0x59, 0xf9, 0xf8, 0xef, 0x59, - 0xe6, 0xdf, 0x35, 0x30, 0x33, 0xf5, 0x13, 0xf7, 0xcf, 0x8c, 0x82, 0xc9, 0x19, 0x34, 0x44, 0x6f, - 0x40, 0x39, 0xda, 0x69, 0x6e, 0xdb, 0xf1, 0x3f, 0xf7, 0x64, 0x6a, 0x8f, 0x63, 0x9e, 0xda, 0xf5, - 0x28, 0x79, 0x42, 0x15, 0x28, 0x3c, 0xf4, 0xdd, 0x90, 0x18, 0xe7, 0x50, 0x19, 0xf2, 0x3b, 0x56, - 0x10, 0x18, 0x5a, 0x6d, 0x5d, 0xe4, 0x46, 0xca, 0xdb, 0x31, 0x40, 0xb1, 0xe1, 0x13, 0x8b, 0xd3, - 0x01, 0x14, 0x45, 0x53, 0xd5, 0xd0, 0x6a, 0x6d, 0x98, 0x57, 0x9f, 0x8c, 0x99, 0xb8, 0xed, 0x6e, - 0xdd, 0x71, 0x8c, 0x73, 0x68, 0x1e, 0xca, 0xdb, 0xdd, 0x88, 0x90, 0x31, 0x6d, 0x77, 0xdb, 0xec, - 0x5b, 0x47, 0x73, 0x50, 0xda, 0xee, 0xf2, 0x6c, 0xd4, 0xc8, 0x09, 0x80, 0xb7, 0x58, 0x8c, 0x7c, - 0xed, 0x2e, 0xcc, 0xab, 0x2f, 0x14, 0x4c, 0x5c, 0x7d, 0xb3, 0xf5, 0xee, 0x86, 0x10, 0xd7, 0xc4, - 0xf5, 0xd6, 0x56, 0x6b, 0xeb, 0x6d, 0x43, 0x63, 0x50, 0x67, 0x77, 0x7b, 0x67, 0x87, 0x41, 0x7a, - 0xed, 0x35, 0x80, 0xe4, 0x32, 0x67, 0xeb, 0xd8, 0xda, 0xde, 0x62, 0x3c, 0x73, 0x50, 0x7a, 0x58, - 0x6f, 0xed, 0x0a, 0x16, 0x06, 0x60, 0x01, 0xe8, 0x8c, 0xa6, 0xc9, 0x68, 0x72, 0xb5, 0x97, 0xc7, - 0x52, 0x7c, 0x54, 0x82, 0x5c, 0xbd, 0xdf, 0x37, 0xce, 0xa1, 0x22, 0xe8, 0xcd, 0x7b, 0x42, 0xf5, - 0x2d, 0xea, 0x0f, 0xac, 0xbe, 0xa1, 0xd7, 0xde, 0x86, 0x2b, 0x53, 0x53, 0x4b, 0xae, 0x6d, 0xb3, - 0xdd, 0xda, 0x15, 0x33, 0xe3, 0x8d, 0xcd, 0x8d, 0x7a, 0x67, 0xc3, 0xd0, 0x10, 0x82, 0x05, 0x09, - 0x74, 0x3b, 0x8d, 0xfb, 0x1b, 0xed, 0xba, 0xa1, 0xd7, 0x9e, 0xc1, 0x42, 0xfa, 0xbe, 0xe4, 0xfa, - 0x51, 0xff, 0xc0, 0xf5, 0x7a, 0x82, 0xbf, 0x13, 0xf2, 0x74, 0x4b, 0x68, 0x2e, 0xec, 0xe8, 0x18, - 0x3a, 0x32, 0x60, 0xbe, 0xe5, 0xb9, 0xa1, 0x6b, 0xf5, 0xdd, 0x67, 0x8c, 0x36, 0x87, 0xaa, 0x50, - 0xd9, 0xf1, 0xc9, 0xd0, 0xf2, 0x19, 0x98, 0x47, 0x0b, 0x00, 0xdc, 0x9c, 0x98, 0x58, 0xce, 0x53, - 0xa3, 0xc0, 0x18, 0x1e, 0x5a, 0x6e, 0xe8, 0x7a, 0x3d, 0x61, 0xe5, 0x62, 0xed, 0x5b, 0x50, 0x4d, - 0xc5, 0x15, 0x74, 0x1e, 0xaa, 0xef, 0x6c, 0xb5, 0xb6, 0x5a, 0xbb, 0xad, 0xfa, 0x66, 0xeb, 0xbd, - 0x8d, 0xa6, 0x30, 0x77, 0xbb, 0xd5, 0x69, 0xd7, 0x77, 0x1b, 0xf7, 0x0d, 0x8d, 0xad, 0x4c, 0x7c, - 0xea, 0xf7, 0xde, 0xf8, 0xd3, 0x67, 0xab, 0xda, 0xc7, 0x9f, 0xad, 0x6a, 0x9f, 0x7e, 0xb6, 0xaa, - 0xfd, 0xec, 0xf3, 0xd5, 0x73, 0x1f, 0x7f, 0xbe, 0x7a, 0xee, 0x93, 0xcf, 0x57, 0xcf, 0xbd, 0xf7, - 0x42, 0xcf, 0x0d, 0xf7, 0x47, 0x7b, 0x37, 0x6d, 0x3a, 0xb8, 0x35, 0x74, 0xbd, 0x9e, 0x6d, 0x0d, - 0x6f, 0x85, 0xae, 0xed, 0xd8, 0xb7, 0x14, 0xd7, 0xdc, 0x2b, 0xf2, 0x37, 0x94, 0x57, 0xfe, 0x1b, - 0x00, 0x00, 0xff, 0xff, 0x50, 0xf3, 0xe4, 0x7c, 0x66, 0x2b, 0x00, 0x00, + // 3045 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x1a, 0x4d, 0x73, 0x1c, 0x57, + 0xd1, 0x33, 0xfb, 0xdd, 0xd2, 0x4a, 0xe3, 0x67, 0x5b, 0x96, 0x6d, 0x59, 0x56, 0x26, 0x81, 0x52, + 0x36, 0xc1, 0xc6, 0x4e, 0xcc, 0x47, 0x08, 0x31, 0xeb, 0x5d, 0x25, 0xde, 0xb2, 0x56, 0x52, 0xbd, + 0x55, 0x62, 0x2a, 0x1c, 0x96, 0xd1, 0xcc, 0xf3, 0x6a, 0xa2, 0xdd, 0x79, 0x9b, 0x99, 0x59, 0xc9, + 0x76, 0x15, 0x50, 0x29, 0x8a, 0x1b, 0x07, 0xb8, 0x71, 0x20, 0x17, 0x4e, 0x9c, 0x80, 0x3f, 0x40, + 0xc1, 0x81, 0x03, 0x27, 0x2a, 0xc5, 0x81, 0xca, 0x09, 0x52, 0xc9, 0x1f, 0xa0, 0x8a, 0x2a, 0xb8, + 0x52, 0xef, 0x63, 0x3e, 0x77, 0x56, 0x2b, 0xa1, 0x2d, 0x17, 0xa7, 0x9d, 0xee, 0xd7, 0xdd, 0xaf, + 0x5f, 0xbf, 0x7e, 0xfd, 0xba, 0xfb, 0x2d, 0x5c, 0xdb, 0x27, 0x86, 0xeb, 0xef, 0x11, 0xc3, 0x1f, + 0xee, 0xdd, 0x0a, 0xbf, 0x6f, 0x0e, 0x5d, 0xea, 0x53, 0x34, 0x17, 0x1b, 0xd4, 0x9f, 0x42, 0x65, + 0xd7, 0xd8, 0xeb, 0x93, 0xce, 0xd0, 0x70, 0xd0, 0x32, 0x94, 0x38, 0xd0, 0x6a, 0x2e, 0x2b, 0x6b, + 0xca, 0x7a, 0x0e, 0x07, 0x20, 0xba, 0x0a, 0xe5, 0x8e, 0x6f, 0xb8, 0xfe, 0x43, 0xf2, 0x74, 0x59, + 0x5d, 0x53, 0xd6, 0xe7, 0x71, 0x08, 0xa3, 0x25, 0x28, 0x6e, 0x38, 0x16, 0x1b, 0xc9, 0xf1, 0x11, + 0x09, 0xa1, 0x55, 0x80, 0x87, 0xe4, 0xa9, 0x37, 0x34, 0x4c, 0x26, 0x30, 0xbf, 0xa6, 0xac, 0x57, + 0x71, 0x0c, 0xa3, 0xff, 0x4d, 0x05, 0xed, 0x01, 0x53, 0xe5, 0x3e, 0x31, 0x7c, 0x4c, 0x3e, 0x1c, + 0x11, 0xcf, 0x47, 0xdf, 0x86, 0x79, 0x73, 0xdf, 0x70, 0x7a, 0xe4, 0x31, 0x21, 0x96, 0xd4, 0x63, + 0xee, 0xce, 0x95, 0x9b, 0x31, 0x9d, 0x6f, 0x36, 0x62, 0x04, 0x38, 0x41, 0x8e, 0x5e, 0x87, 0xca, + 0x91, 0xe1, 0x13, 0x77, 0x60, 0xb8, 0x07, 0x5c, 0xd1, 0xb9, 0x3b, 0x4b, 0x09, 0xde, 0x47, 0xc1, + 0x28, 0x8e, 0x08, 0xd1, 0x9b, 0x50, 0x75, 0x89, 0x45, 0xc3, 0x31, 0xbe, 0x90, 0xc9, 0x9c, 0x49, + 0x62, 0xf4, 0x0d, 0x28, 0x7b, 0xbe, 0xe1, 0x8f, 0x3c, 0xe2, 0x2d, 0xe7, 0xd7, 0x72, 0xeb, 0x73, + 0x77, 0x56, 0x12, 0x8c, 0xa1, 0x7d, 0x3b, 0x9c, 0x0a, 0x87, 0xd4, 0x68, 0x1d, 0x16, 0x4d, 0x3a, + 0x18, 0x92, 0x3e, 0xf1, 0x89, 0x18, 0x5c, 0x2e, 0xac, 0x29, 0xeb, 0x65, 0x9c, 0x46, 0xa3, 0x57, + 0x20, 0x47, 0x5c, 0x77, 0xb9, 0x98, 0x61, 0x0d, 0x3c, 0x72, 0x1c, 0xdb, 0xe9, 0x6d, 0xb8, 0x2e, + 0x75, 0x31, 0xa3, 0xd2, 0x7f, 0xa2, 0x40, 0x25, 0x52, 0x4f, 0x67, 0x16, 0x25, 0xe6, 0xc1, 0x90, + 0xda, 0x8e, 0xbf, 0xeb, 0x71, 0x8b, 0xe6, 0x71, 0x02, 0xc7, 0xb6, 0xca, 0x25, 0x1e, 0xed, 0x1f, + 0x12, 0x6b, 0xd7, 0xe3, 0x76, 0xcb, 0xe3, 0x18, 0x06, 0x69, 0x90, 0xf3, 0xc8, 0x87, 0xdc, 0x2c, + 0x79, 0xcc, 0x3e, 0x99, 0xd4, 0xbe, 0xe1, 0xf9, 0x9d, 0xa7, 0x8e, 0xc9, 0x79, 0xf2, 0x42, 0x6a, + 0x1c, 0xa7, 0xff, 0x00, 0xb4, 0xa6, 0xed, 0x0d, 0x0d, 0xdf, 0xdc, 0x27, 0x6e, 0xdd, 0xf4, 0x6d, + 0xea, 0xa0, 0x57, 0xa0, 0x68, 0xf0, 0x2f, 0xae, 0xc7, 0xc2, 0x9d, 0x0b, 0x89, 0xb5, 0x08, 0x22, + 0x2c, 0x49, 0x98, 0xd7, 0x35, 0xe8, 0x60, 0x60, 0xfb, 0xa1, 0x52, 0x21, 0x8c, 0xd6, 0x60, 0xae, + 0xe5, 0xb1, 0xa9, 0x76, 0xd8, 0x1a, 0xb8, 0x6a, 0x65, 0x1c, 0x47, 0xe9, 0x0d, 0xc8, 0xd5, 0x1b, + 0x0f, 0x13, 0x42, 0x94, 0xe3, 0x85, 0xa8, 0xe3, 0x42, 0x30, 0xa0, 0x56, 0xcf, 0xa1, 0x2e, 0xb1, + 0xee, 0xf7, 0xa9, 0x79, 0x20, 0xb7, 0xe3, 0x6c, 0x32, 0x7f, 0xac, 0xc2, 0xa5, 0x96, 0xf3, 0xb8, + 0x3f, 0x22, 0xcc, 0x50, 0x91, 0x89, 0x3c, 0xf4, 0x1d, 0xa8, 0x86, 0x03, 0xbb, 0x4f, 0x87, 0x44, + 0x1a, 0xe9, 0x6a, 0xc2, 0x48, 0x09, 0x0a, 0x9c, 0x64, 0x40, 0xf7, 0xa0, 0x1a, 0x09, 0x6c, 0x35, + 0x99, 0xdd, 0x72, 0x63, 0x2e, 0x13, 0xa7, 0xc0, 0x49, 0x7a, 0x7e, 0xd2, 0xcd, 0x7d, 0x32, 0x30, + 0x5a, 0x4d, 0x6e, 0xd4, 0x1c, 0x0e, 0x61, 0xf4, 0x10, 0x2e, 0x90, 0x27, 0x66, 0x7f, 0x64, 0x91, + 0x18, 0x8f, 0xc5, 0xf7, 0xfe, 0xd8, 0x29, 0xb2, 0xb8, 0xf4, 0x5f, 0xa8, 0x71, 0xf7, 0x90, 0x86, + 0xfd, 0x2e, 0x5c, 0xb2, 0xb3, 0x2c, 0x23, 0xe3, 0x80, 0x9e, 0x6d, 0x88, 0x38, 0x25, 0xce, 0x16, + 0x80, 0xee, 0x86, 0x8e, 0x27, 0xc2, 0xc2, 0xf5, 0x09, 0xea, 0xa6, 0x5c, 0x50, 0x87, 0x9c, 0x61, + 0x06, 0x01, 0x41, 0x4b, 0x3a, 0x6b, 0xe3, 0x21, 0x66, 0x83, 0x68, 0x1b, 0x90, 0x3d, 0xe6, 0x23, + 0xd2, 0x2a, 0x37, 0x92, 0x1a, 0x8f, 0x91, 0xe1, 0x0c, 0x56, 0xfd, 0x33, 0x05, 0xce, 0xc7, 0x22, + 0xa3, 0x37, 0xa4, 0x8e, 0x47, 0xce, 0x1a, 0x1a, 0xdb, 0x80, 0xac, 0x94, 0xb9, 0x49, 0xe0, 0x1e, + 0x93, 0x8c, 0x11, 0xe8, 0x38, 0xce, 0x88, 0x10, 0xe4, 0x07, 0xd4, 0x22, 0xd2, 0x47, 0xf8, 0x37, + 0x7a, 0x19, 0xb4, 0x81, 0x61, 0x3b, 0xbe, 0x61, 0x3b, 0xc4, 0xed, 0x92, 0x21, 0x35, 0xf7, 0x65, + 0x60, 0x58, 0x8c, 0xf0, 0x1b, 0x0c, 0xad, 0x3f, 0x81, 0x0b, 0x8d, 0x58, 0x04, 0x6a, 0x13, 0xcf, + 0x33, 0x7a, 0x67, 0x5e, 0x63, 0x3a, 0xd6, 0xa9, 0xe3, 0xb1, 0x4e, 0xff, 0x83, 0x02, 0x8b, 0x98, + 0x58, 0xb4, 0x4d, 0x7c, 0x63, 0x46, 0xd3, 0x4e, 0x0b, 0x9f, 0x69, 0xb5, 0x72, 0x19, 0x21, 0xf8, + 0x14, 0xb6, 0xfb, 0x21, 0x5c, 0x67, 0x0b, 0xc0, 0xe1, 0x04, 0x3b, 0x2e, 0xed, 0xb9, 0xc4, 0xf3, + 0x9e, 0xcf, 0x72, 0xf4, 0x5f, 0x2b, 0xb0, 0x92, 0x54, 0xe0, 0x6d, 0xea, 0x1e, 0x19, 0xae, 0xf5, + 0x9c, 0xcc, 0x99, 0x65, 0xaa, 0x5c, 0xb6, 0xa9, 0xfe, 0xa5, 0xc4, 0x83, 0x4c, 0x83, 0x3a, 0x8f, + 0xed, 0x1e, 0xaa, 0x41, 0xde, 0x1b, 0x1a, 0x8e, 0x54, 0x6b, 0x29, 0xfb, 0xb2, 0xc6, 0x9c, 0x86, + 0xa5, 0x44, 0x1e, 0x4b, 0x74, 0x42, 0x45, 0x02, 0x90, 0x2d, 0xd2, 0x8a, 0x05, 0x39, 0x19, 0x22, + 0x8e, 0x89, 0x82, 0x09, 0x72, 0x16, 0x67, 0xbd, 0x20, 0xce, 0xe6, 0x45, 0x9c, 0x0d, 0xe0, 0xf0, + 0x6c, 0x15, 0x62, 0x67, 0xab, 0x06, 0x9a, 0x77, 0x60, 0x0f, 0x9b, 0xed, 0xcd, 0xba, 0xd7, 0x91, + 0x1a, 0x15, 0xf9, 0xdd, 0x32, 0x86, 0xd7, 0xff, 0xa8, 0xc2, 0x15, 0x16, 0xb4, 0xad, 0x51, 0x3f, + 0x16, 0x73, 0x67, 0x94, 0x62, 0xdd, 0x85, 0xa2, 0xc9, 0xed, 0x38, 0x25, 0x90, 0x0a, 0x63, 0x63, + 0x49, 0x8c, 0x1a, 0xb0, 0xe0, 0x49, 0x95, 0x44, 0x88, 0xe5, 0x06, 0x5b, 0xb8, 0x73, 0x2d, 0xc1, + 0xde, 0x49, 0x90, 0xe0, 0x14, 0x0b, 0x53, 0x9d, 0x0e, 0x89, 0x6b, 0xf8, 0xd4, 0xe5, 0xd7, 0x63, + 0x9e, 0x8b, 0x48, 0xaa, 0xbe, 0x1d, 0x23, 0xc0, 0x09, 0xf2, 0x4c, 0xc7, 0x29, 0x64, 0x3b, 0xce, + 0xaf, 0x54, 0x58, 0x6a, 0x13, 0xb7, 0x37, 0x7b, 0xfb, 0xdd, 0x83, 0xaa, 0x75, 0xca, 0x1b, 0x3a, + 0x41, 0x8f, 0x5a, 0x80, 0x06, 0x4c, 0x33, 0xab, 0x79, 0x2a, 0xf7, 0xcb, 0x60, 0x0a, 0x1d, 0x2d, + 0x3f, 0x25, 0x88, 0x4f, 0x30, 0xd2, 0x0e, 0x5c, 0x68, 0x87, 0xa8, 0x07, 0xc1, 0xc4, 0xe8, 0x9b, + 0xb1, 0x84, 0x58, 0xc9, 0xb8, 0x5f, 0x22, 0x9e, 0x74, 0x46, 0xac, 0x7f, 0xaa, 0x40, 0xb5, 0xe9, + 0x1a, 0xb6, 0x13, 0x84, 0x34, 0xf4, 0x12, 0x2c, 0xf8, 0x86, 0xdb, 0x23, 0x7e, 0xd7, 0xa1, 0x16, + 0xe9, 0xda, 0x16, 0xb7, 0x77, 0x05, 0xcf, 0x0b, 0xec, 0x16, 0xb5, 0x48, 0xcb, 0x42, 0x2f, 0x80, + 0x84, 0xa5, 0xc2, 0xe2, 0xac, 0xce, 0x09, 0x1c, 0x57, 0x16, 0x7d, 0x0d, 0x2e, 0x4b, 0x92, 0xc8, + 0x9c, 0x5d, 0x93, 0x8e, 0x64, 0xf2, 0x58, 0xc5, 0x97, 0xc4, 0x70, 0xdc, 0x83, 0x47, 0x8e, 0x8f, + 0xde, 0x86, 0x35, 0xc9, 0xc7, 0x12, 0x0b, 0xbb, 0xb7, 0xef, 0x77, 0x2d, 0xa6, 0x61, 0x77, 0x40, + 0x0f, 0x89, 0x14, 0x20, 0x8a, 0x9b, 0x15, 0x41, 0xd7, 0x92, 0x64, 0x7c, 0x1d, 0x6d, 0x7a, 0x48, + 0xb8, 0x1c, 0xfd, 0x37, 0x39, 0xd0, 0xd2, 0x2b, 0x3f, 0xab, 0x2f, 0x5d, 0x07, 0x60, 0x5f, 0x5d, + 0x66, 0x3f, 0xc2, 0x17, 0x5d, 0xc1, 0x15, 0x86, 0x61, 0xe2, 0x09, 0xba, 0x0d, 0x05, 0x31, 0x92, + 0x75, 0xd4, 0x1a, 0x74, 0x30, 0xa4, 0x0e, 0x71, 0x7c, 0x4e, 0x8b, 0x05, 0x25, 0x7a, 0x11, 0xaa, + 0xd1, 0xb5, 0xd4, 0xf5, 0xc3, 0xc4, 0x3e, 0x71, 0x57, 0xc9, 0x6a, 0xa4, 0x90, 0xe1, 0xb8, 0x63, + 0xd5, 0x08, 0xfa, 0x12, 0x2c, 0xec, 0x51, 0xea, 0x7b, 0xbe, 0x6b, 0x0c, 0xbb, 0x16, 0x75, 0x88, + 0x0c, 0x5b, 0xd5, 0x10, 0xdb, 0xa4, 0x0e, 0x19, 0x2b, 0x28, 0x4a, 0xe3, 0x05, 0x05, 0xaa, 0xc3, + 0x82, 0x30, 0xfd, 0x50, 0x7a, 0xc7, 0x72, 0x99, 0xdb, 0x2b, 0x99, 0x1f, 0x27, 0xfc, 0x07, 0x57, + 0xad, 0x84, 0x3b, 0x65, 0x79, 0x77, 0x25, 0xdb, 0xbb, 0x7f, 0xab, 0x42, 0x95, 0xb9, 0x57, 0xe4, + 0xd8, 0x77, 0xa1, 0xdc, 0xb7, 0x0f, 0x89, 0xc3, 0x66, 0x56, 0x32, 0x42, 0x0f, 0xa3, 0xde, 0x94, + 0x04, 0x38, 0x24, 0x65, 0xbb, 0xc4, 0x7d, 0x37, 0xee, 0x9a, 0x15, 0x86, 0x11, 0x8e, 0xd9, 0x84, + 0x1b, 0x31, 0x8f, 0x14, 0x0b, 0x4c, 0xb9, 0x7c, 0x8e, 0xef, 0xec, 0xb5, 0x88, 0x8c, 0xaf, 0x71, + 0x37, 0x7e, 0x02, 0xea, 0x70, 0x7d, 0x92, 0x94, 0x78, 0x32, 0x71, 0x35, 0x53, 0x86, 0x50, 0xe4, + 0x1e, 0xac, 0xf4, 0x69, 0xaf, 0xeb, 0x11, 0xf7, 0xd0, 0x36, 0xc9, 0xf8, 0x31, 0x29, 0x70, 0x2f, + 0xbf, 0xd2, 0xa7, 0xbd, 0x8e, 0x20, 0x49, 0x1d, 0x15, 0xfd, 0x03, 0x58, 0xea, 0x08, 0x85, 0x42, + 0x2b, 0xc8, 0x98, 0x79, 0x1b, 0x8a, 0x42, 0x99, 0xe9, 0x76, 0x93, 0x84, 0x53, 0xac, 0xa6, 0x0f, + 0xe0, 0xf2, 0xd8, 0x5c, 0x32, 0x51, 0x7e, 0x0d, 0x4a, 0xc6, 0x70, 0xd8, 0xb7, 0x89, 0x35, 0x7d, + 0xb6, 0x80, 0x72, 0xda, 0x74, 0x1f, 0xc0, 0x8d, 0x4e, 0x3c, 0x36, 0xc4, 0x8c, 0x17, 0xac, 0x71, + 0x56, 0x91, 0x4a, 0xff, 0x3a, 0x5c, 0x6b, 0x50, 0xea, 0x5a, 0xb6, 0xc3, 0x6e, 0xae, 0xfb, 0xc1, + 0x31, 0x09, 0xe6, 0x59, 0x86, 0xd2, 0x21, 0x71, 0xbd, 0xa0, 0x86, 0xce, 0xe1, 0x00, 0x64, 0x25, + 0xd5, 0x4a, 0x36, 0xa7, 0xb4, 0xcc, 0xff, 0x1e, 0x99, 0xd1, 0xeb, 0xb0, 0x14, 0x9e, 0x3d, 0x9f, + 0x9a, 0xb4, 0xdf, 0x0d, 0x94, 0x50, 0xb9, 0x5b, 0x5c, 0x0c, 0xce, 0x19, 0x1f, 0x7c, 0x4f, 0x8c, + 0xfd, 0xdf, 0xf8, 0xb6, 0xfe, 0x6f, 0x05, 0x2e, 0xd6, 0x2d, 0x2b, 0x5a, 0x60, 0x60, 0xcd, 0x97, + 0x41, 0x95, 0x3b, 0x75, 0x6c, 0xdc, 0x55, 0x6d, 0x0b, 0x2d, 0x25, 0x32, 0x9f, 0xf9, 0x30, 0xb5, + 0x19, 0x8b, 0x99, 0x59, 0xf9, 0x7d, 0x0d, 0xce, 0xdb, 0x5e, 0xd7, 0x21, 0x47, 0xdd, 0x28, 0x82, + 0x73, 0xbd, 0xcb, 0x78, 0xd1, 0xf6, 0xb6, 0xc8, 0x51, 0x34, 0x1d, 0xba, 0x01, 0x73, 0x07, 0xb2, + 0x4f, 0xc6, 0x2c, 0x24, 0xce, 0x1d, 0x04, 0xa8, 0x96, 0x95, 0x19, 0xc5, 0x8a, 0xd9, 0x51, 0xec, + 0x4f, 0x0a, 0x5c, 0xc6, 0x84, 0xdd, 0x55, 0x67, 0x5a, 0xfb, 0x32, 0x94, 0x4c, 0xc3, 0x33, 0x0d, + 0x8b, 0xc8, 0x8e, 0x46, 0x00, 0xb2, 0x11, 0x97, 0xcb, 0xb7, 0x64, 0x13, 0x26, 0x00, 0xd3, 0xcb, + 0xc8, 0x9f, 0x68, 0x19, 0x13, 0x52, 0x8d, 0xbf, 0xe4, 0xe0, 0x6a, 0xb4, 0x80, 0xb1, 0x33, 0x71, + 0xc6, 0x7b, 0x74, 0xd2, 0xce, 0x5e, 0xe1, 0xe7, 0xc5, 0x8d, 0x6d, 0x6a, 0x98, 0xfe, 0x9b, 0xf0, + 0x82, 0xcf, 0x6a, 0x85, 0xae, 0xef, 0xda, 0xbd, 0x1e, 0x53, 0xff, 0x90, 0x38, 0x89, 0xdc, 0xc2, + 0x3e, 0x41, 0x67, 0xe4, 0x3a, 0x97, 0xb1, 0x2b, 0x44, 0x6c, 0x30, 0x09, 0xf1, 0x1e, 0x49, 0xb6, + 0xd3, 0x14, 0xb2, 0x9d, 0xc6, 0x60, 0x79, 0x4a, 0x5c, 0x21, 0x97, 0x58, 0x34, 0xa5, 0x4f, 0x71, + 0x9a, 0x3e, 0x2b, 0x71, 0x7d, 0x58, 0x8d, 0x97, 0x50, 0x27, 0xb5, 0xa1, 0xa5, 0x13, 0x6d, 0x68, + 0x39, 0x7b, 0x43, 0xff, 0x9a, 0x83, 0x6b, 0x99, 0x1b, 0x3a, 0x9b, 0x6e, 0xc7, 0x5d, 0x28, 0xb0, + 0xfa, 0x2d, 0xc8, 0xae, 0x93, 0x6d, 0x98, 0x70, 0xb6, 0xa8, 0xda, 0x13, 0xd4, 0x41, 0x66, 0x93, + 0x3b, 0x49, 0x9f, 0xf5, 0x64, 0xb9, 0xd2, 0xab, 0x80, 0xf8, 0x46, 0x24, 0x29, 0x85, 0x97, 0x6b, + 0x6c, 0x24, 0xde, 0x06, 0x41, 0x4d, 0xa8, 0x04, 0x15, 0x0b, 0x2b, 0xef, 0x98, 0xea, 0x5f, 0xce, + 0x2c, 0x90, 0xc6, 0xca, 0x12, 0x1c, 0x31, 0x66, 0x6e, 0x43, 0x29, 0x73, 0x1b, 0xd0, 0x26, 0x2c, + 0xf2, 0xba, 0xa0, 0x1b, 0x4d, 0x5b, 0xe6, 0xd3, 0xbe, 0x98, 0xbc, 0x18, 0x32, 0x4b, 0x21, 0xbc, + 0xc0, 0x79, 0x83, 0x8a, 0xcb, 0xd3, 0xff, 0xae, 0xc2, 0x6a, 0xb4, 0xa9, 0x3b, 0xd4, 0xf3, 0x67, + 0x7d, 0x52, 0x4f, 0x74, 0xec, 0xd4, 0x33, 0x1e, 0xbb, 0xdb, 0x50, 0x12, 0xb5, 0x38, 0x3b, 0xf5, + 0xcc, 0x18, 0x97, 0xc7, 0xf6, 0x60, 0x60, 0xb4, 0x9c, 0xc7, 0x14, 0x07, 0x74, 0xe8, 0x0d, 0x98, + 0xe7, 0xdb, 0x1c, 0xf0, 0xe5, 0x8f, 0xe7, 0x9b, 0x63, 0xc4, 0x1d, 0xc9, 0x7b, 0x8a, 0x30, 0xf8, + 0xb1, 0x0a, 0x37, 0x26, 0x1a, 0x78, 0x36, 0x27, 0xe7, 0xb9, 0x58, 0xf8, 0x54, 0xe7, 0xec, 0x54, + 0x6d, 0x45, 0x88, 0xac, 0x9c, 0xe8, 0x65, 0x2b, 0xa9, 0x5e, 0xf6, 0x6a, 0x40, 0xb9, 0x65, 0x0c, + 0x82, 0xd2, 0x29, 0x86, 0x41, 0x37, 0xa1, 0xc8, 0xa3, 0x43, 0xe0, 0x02, 0x19, 0x6d, 0x22, 0xbe, + 0x93, 0x92, 0x4a, 0x6f, 0xc8, 0x87, 0x34, 0x3e, 0xf1, 0xe4, 0x87, 0xb4, 0x15, 0x49, 0x16, 0x9b, + 0x35, 0x42, 0xe8, 0xbf, 0x57, 0x01, 0x8d, 0x07, 0x27, 0x76, 0x4f, 0x4f, 0xd8, 0xc7, 0x84, 0xcd, + 0x55, 0xf9, 0x50, 0x17, 0x2c, 0x59, 0x4d, 0x2d, 0x39, 0xe8, 0x7b, 0xe5, 0x4e, 0xd0, 0xf7, 0x7a, + 0x1b, 0x34, 0x33, 0x28, 0x10, 0xbb, 0x5e, 0xd4, 0xd1, 0x9e, 0x52, 0x45, 0x2e, 0x9a, 0x71, 0x78, + 0xe4, 0x8d, 0xc7, 0xc8, 0x42, 0x46, 0x8c, 0x7c, 0x0d, 0xe6, 0xf6, 0xfa, 0xd4, 0x3c, 0x90, 0x75, + 0xac, 0xb8, 0xa5, 0x50, 0xf2, 0xec, 0x70, 0xf1, 0xb0, 0x17, 0x74, 0xc9, 0x49, 0xd8, 0xbb, 0x28, + 0x45, 0xbd, 0x0b, 0xfd, 0x97, 0x0a, 0x2c, 0x45, 0xc7, 0xa3, 0xd1, 0xa7, 0x1e, 0x99, 0x51, 0xdc, + 0x89, 0x65, 0x39, 0x6a, 0x32, 0xcb, 0x39, 0x45, 0x37, 0xf2, 0x63, 0x05, 0x2e, 0x8f, 0xa9, 0x37, + 0x9b, 0x53, 0xbb, 0x0c, 0x25, 0x6f, 0x64, 0x9a, 0xac, 0x32, 0x95, 0xfa, 0x49, 0xf0, 0x34, 0xfa, + 0xfd, 0x54, 0x01, 0x2d, 0x7a, 0x54, 0x11, 0x8e, 0x3d, 0x83, 0x37, 0xa9, 0xab, 0x50, 0x96, 0xee, + 0x2f, 0xae, 0xe3, 0x1c, 0x0e, 0xe1, 0xe3, 0x9e, 0x9b, 0xf4, 0xef, 0x41, 0x81, 0xd3, 0x4d, 0x79, + 0x97, 0x9e, 0xe4, 0xee, 0x2b, 0x50, 0xe9, 0x0c, 0xfb, 0x36, 0x0f, 0x44, 0x32, 0x35, 0x8d, 0x10, + 0xfa, 0x47, 0x2a, 0x5c, 0xc0, 0x74, 0xe4, 0x13, 0x2e, 0xaa, 0x6e, 0x0d, 0x6c, 0x8f, 0x57, 0x2c, + 0x35, 0xd0, 0x3a, 0x74, 0xe4, 0x9a, 0x24, 0x16, 0x1d, 0x44, 0x1d, 0x37, 0x86, 0x47, 0xeb, 0xb0, + 0x28, 0x70, 0xe9, 0x23, 0x9d, 0x46, 0x33, 0xa9, 0xa2, 0x1a, 0x89, 0x49, 0x15, 0x85, 0xcf, 0x18, + 0x9e, 0x49, 0x15, 0xb8, 0x48, 0x6a, 0x5e, 0x48, 0x4d, 0xa1, 0xd1, 0x5b, 0x50, 0x94, 0xbd, 0xd4, + 0x02, 0xdf, 0x93, 0x64, 0xaa, 0x90, 0xb1, 0xba, 0xe0, 0x71, 0x4b, 0xfc, 0xea, 0x0e, 0x2c, 0x04, + 0xd6, 0x12, 0xae, 0x75, 0x8c, 0xa5, 0xd7, 0x60, 0x6e, 0xbb, 0x6f, 0xa5, 0x8c, 0x1d, 0x47, 0x31, + 0x8a, 0x2d, 0x72, 0x94, 0xda, 0xcd, 0x38, 0x4a, 0xff, 0x4f, 0x0e, 0x0a, 0xe2, 0xf0, 0xae, 0x40, + 0xa5, 0xe5, 0xf1, 0x27, 0x2f, 0x59, 0xa4, 0x97, 0x71, 0x84, 0x60, 0x5a, 0xf0, 0xcf, 0xa8, 0xe9, + 0x2e, 0x41, 0x74, 0x0f, 0xe6, 0xc4, 0x67, 0x10, 0x9a, 0xc7, 0x3b, 0xd0, 0x69, 0x07, 0xc6, 0x71, + 0x0e, 0xf4, 0x10, 0xce, 0x6f, 0x11, 0x62, 0x35, 0x5d, 0x3a, 0x1c, 0x06, 0x14, 0x32, 0x4d, 0x9f, + 0x22, 0x66, 0x9c, 0x0f, 0xbd, 0x09, 0x8b, 0x0c, 0x59, 0xb7, 0xac, 0x50, 0x94, 0xe8, 0x89, 0xa1, + 0xf1, 0xd8, 0x8a, 0xd3, 0xa4, 0xa8, 0x01, 0x0b, 0xef, 0x0e, 0x2d, 0xc3, 0x27, 0xd2, 0x84, 0x41, + 0xc2, 0x77, 0x2d, 0x2b, 0x69, 0x90, 0x1b, 0x84, 0x53, 0x2c, 0xe9, 0xd7, 0xe6, 0xd2, 0xd8, 0x6b, + 0x33, 0xfa, 0x0a, 0x6f, 0x02, 0xf6, 0x08, 0x4f, 0xc4, 0x17, 0x52, 0x29, 0x49, 0xf0, 0xea, 0xd8, + 0x13, 0x0d, 0xc0, 0x1e, 0x41, 0xbb, 0x70, 0x31, 0xc3, 0x71, 0xbc, 0xe5, 0x0a, 0xd7, 0x6d, 0x6d, + 0x9a, 0x87, 0xe1, 0x4c, 0x6e, 0xfd, 0x47, 0x70, 0x31, 0xbc, 0x61, 0xe2, 0x0f, 0xe9, 0xa7, 0xb8, + 0xd9, 0xd6, 0x83, 0x66, 0xa6, 0x3a, 0xf1, 0x7a, 0x90, 0x3d, 0xcc, 0x8c, 0xa7, 0x49, 0xfd, 0x9f, + 0x0a, 0x3b, 0x55, 0x89, 0x3f, 0x62, 0x9c, 0x66, 0xf2, 0xac, 0xeb, 0x50, 0x9d, 0xc5, 0x75, 0x98, + 0xd5, 0x2a, 0xb8, 0x0d, 0x97, 0x44, 0xce, 0xe5, 0xd9, 0xcf, 0x48, 0x77, 0x48, 0xdc, 0xae, 0x47, + 0x4c, 0xea, 0x88, 0x72, 0x52, 0xc5, 0x88, 0x0f, 0x76, 0xec, 0x67, 0x64, 0x87, 0xb8, 0x1d, 0x3e, + 0x92, 0xf5, 0x62, 0xa4, 0xff, 0x4e, 0x01, 0x14, 0x7f, 0x69, 0x9e, 0xcd, 0x45, 0xf8, 0x0e, 0x54, + 0xf7, 0x22, 0xa1, 0xe1, 0x0b, 0xf2, 0x0b, 0xd9, 0xd9, 0x44, 0x7c, 0xfe, 0x24, 0x5f, 0xe6, 0x2e, + 0x59, 0x30, 0x1f, 0x4f, 0xff, 0x18, 0x8d, 0x6f, 0x87, 0x01, 0x98, 0x7f, 0x33, 0x9c, 0x43, 0xad, + 0x20, 0xd2, 0xf2, 0x6f, 0x86, 0x33, 0x03, 0x59, 0x15, 0xcc, 0xbf, 0x59, 0x10, 0x19, 0x88, 0xf7, + 0x48, 0x19, 0x3e, 0x03, 0x50, 0x7f, 0x1d, 0xe6, 0xd3, 0xaf, 0x20, 0xfb, 0x76, 0x6f, 0x5f, 0xfe, + 0x93, 0x83, 0x7f, 0x23, 0x0d, 0x72, 0x7d, 0x7a, 0x24, 0xc3, 0x0f, 0xfb, 0x64, 0xba, 0xc5, 0xcd, + 0x72, 0x32, 0x2e, 0xae, 0x6d, 0x14, 0xec, 0xf9, 0x37, 0xbb, 0xb4, 0x82, 0x9a, 0x59, 0xaa, 0x16, + 0xc2, 0xfa, 0xf7, 0xe1, 0xc6, 0x26, 0xed, 0xc5, 0x9a, 0x78, 0xd1, 0x23, 0xeb, 0x6c, 0x36, 0x50, + 0xff, 0x48, 0x81, 0xb5, 0xc9, 0x53, 0xcc, 0x26, 0x1b, 0x99, 0xf6, 0x82, 0xdc, 0x67, 0xb6, 0x24, + 0xe6, 0x81, 0x37, 0x1a, 0xb4, 0x89, 0x6f, 0xa0, 0xaf, 0x06, 0x67, 0x3b, 0x2b, 0xb7, 0x08, 0x28, + 0x13, 0x67, 0xbc, 0x06, 0x9a, 0x19, 0xc7, 0x77, 0xc8, 0x87, 0x72, 0x9e, 0x31, 0xbc, 0xfe, 0x73, + 0x05, 0x2e, 0xc5, 0xfe, 0xd3, 0x40, 0xfc, 0x40, 0x22, 0xba, 0x08, 0x05, 0xd1, 0xda, 0x16, 0x9b, + 0x28, 0x00, 0xe6, 0x39, 0x4f, 0xa8, 0xfb, 0x80, 0x6d, 0xae, 0xbc, 0x7e, 0x24, 0x88, 0x96, 0xa0, + 0xf8, 0x84, 0xba, 0x9b, 0xf4, 0x48, 0x9e, 0x5b, 0x09, 0x89, 0xec, 0x6b, 0xc0, 0x39, 0xf2, 0xb2, + 0x4d, 0x24, 0x40, 0xc6, 0xe1, 0x8d, 0x06, 0x8c, 0x43, 0x24, 0xbe, 0x12, 0x62, 0xa9, 0xe0, 0x5a, + 0xa6, 0x4e, 0x75, 0xf3, 0x60, 0x56, 0xbb, 0x70, 0x11, 0x0a, 0xf1, 0x1e, 0xb3, 0x00, 0x32, 0xff, + 0xb8, 0x21, 0xff, 0xdf, 0x95, 0x0f, 0xff, 0xdf, 0xa5, 0xff, 0x43, 0x01, 0x3d, 0x53, 0x3f, 0x71, + 0xff, 0xcc, 0x28, 0x98, 0x9c, 0x41, 0x43, 0xf4, 0x16, 0x94, 0x83, 0x9d, 0xe6, 0xb6, 0x4d, 0xff, + 0x3b, 0x28, 0x53, 0x7b, 0x1c, 0xf2, 0xd4, 0xae, 0x07, 0xc9, 0x13, 0xaa, 0x40, 0xe1, 0x91, 0x6b, + 0xfb, 0x44, 0x3b, 0x87, 0xca, 0x90, 0xdf, 0x31, 0x3c, 0x4f, 0x53, 0x6a, 0xeb, 0x22, 0x37, 0x8a, + 0x3d, 0x3e, 0x03, 0x14, 0x1b, 0x2e, 0x31, 0x38, 0x1d, 0x40, 0x51, 0x34, 0x55, 0x35, 0xa5, 0xd6, + 0x86, 0xf9, 0xf8, 0x9b, 0x33, 0x13, 0xb7, 0xdd, 0xad, 0x5b, 0x96, 0x76, 0x0e, 0xcd, 0x43, 0x79, + 0xbb, 0x1b, 0x10, 0x32, 0xa6, 0xed, 0x6e, 0x9b, 0x7d, 0xab, 0x68, 0x0e, 0x4a, 0xdb, 0x5d, 0x9e, + 0x8d, 0x6a, 0x39, 0x01, 0xf0, 0x16, 0x8b, 0x96, 0xaf, 0xdd, 0x85, 0xf9, 0xf8, 0x0b, 0x05, 0x13, + 0x57, 0xdf, 0x6c, 0xbd, 0xb7, 0x21, 0xc4, 0x35, 0x71, 0xbd, 0xb5, 0xd5, 0xda, 0x7a, 0x47, 0x53, + 0x18, 0xd4, 0xd9, 0xdd, 0xde, 0xd9, 0x61, 0x90, 0x5a, 0x7b, 0x03, 0x20, 0xba, 0xcc, 0xd9, 0x3a, + 0xb6, 0xb6, 0xb7, 0x18, 0xcf, 0x1c, 0x94, 0x1e, 0xd5, 0x5b, 0xbb, 0x82, 0x85, 0x01, 0x58, 0x00, + 0x2a, 0xa3, 0x69, 0x32, 0x9a, 0x5c, 0xed, 0xd5, 0x54, 0x8a, 0x8f, 0x4a, 0x90, 0xab, 0xf7, 0xfb, + 0xda, 0x39, 0x54, 0x04, 0xb5, 0x79, 0x5f, 0xa8, 0xbe, 0x45, 0xdd, 0x81, 0xd1, 0xd7, 0xd4, 0xda, + 0x3b, 0x70, 0x65, 0x62, 0x6a, 0xc9, 0xb5, 0x6d, 0xb6, 0x5b, 0xbb, 0x62, 0x66, 0xbc, 0xb1, 0xb9, + 0x51, 0xef, 0x6c, 0x68, 0x0a, 0x42, 0xb0, 0x20, 0x81, 0x6e, 0xa7, 0xf1, 0x60, 0xa3, 0x5d, 0xd7, + 0xd4, 0xda, 0x33, 0x58, 0x48, 0xde, 0x97, 0x5c, 0x3f, 0xea, 0x1e, 0xd8, 0x4e, 0x4f, 0xf0, 0x77, + 0x7c, 0x9e, 0x6e, 0x09, 0xcd, 0x85, 0x1d, 0x2d, 0x4d, 0x45, 0x1a, 0xcc, 0xb7, 0x1c, 0xdb, 0xb7, + 0x8d, 0xbe, 0xfd, 0x8c, 0xd1, 0xe6, 0x50, 0x15, 0x2a, 0x3b, 0x2e, 0x19, 0x1a, 0x2e, 0x03, 0xf3, + 0x68, 0x01, 0x80, 0x9b, 0x13, 0x13, 0xc3, 0x7a, 0xaa, 0x15, 0x18, 0xc3, 0x23, 0xc3, 0xf6, 0x6d, + 0xa7, 0x27, 0xac, 0x5c, 0xac, 0x7d, 0x0b, 0xaa, 0x89, 0xb8, 0x82, 0xce, 0x43, 0xf5, 0xdd, 0xad, + 0xd6, 0x56, 0x6b, 0xb7, 0x55, 0xdf, 0x6c, 0xbd, 0xbf, 0xd1, 0x14, 0xe6, 0x6e, 0xb7, 0x3a, 0xed, + 0xfa, 0x6e, 0xe3, 0x81, 0xa6, 0xb0, 0x95, 0x89, 0x4f, 0xf5, 0xfe, 0x5b, 0x7f, 0xfe, 0x7c, 0x55, + 0xf9, 0xe4, 0xf3, 0x55, 0xe5, 0xb3, 0xcf, 0x57, 0x95, 0x9f, 0x7d, 0xb1, 0x7a, 0xee, 0x93, 0x2f, + 0x56, 0xcf, 0x7d, 0xfa, 0xc5, 0xea, 0xb9, 0xf7, 0x5f, 0xea, 0xd9, 0xfe, 0xfe, 0x68, 0xef, 0xa6, + 0x49, 0x07, 0xb7, 0x86, 0xb6, 0xd3, 0x33, 0x8d, 0xe1, 0x2d, 0xdf, 0x36, 0x2d, 0xf3, 0x56, 0xcc, + 0x35, 0xf7, 0x8a, 0xfc, 0x0d, 0xe5, 0xb5, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0x6e, 0x77, 0x18, + 0x03, 0xa7, 0x2b, 0x00, 0x00, } func (m *TableSpan) Marshal() (dAtA []byte, err error) { @@ -5271,6 +5283,11 @@ func (m *NodeHeartbeat) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.LogServiceDispatcherCount != 0 { + i = encodeVarintHeartbeat(dAtA, i, uint64(m.LogServiceDispatcherCount)) + i-- + dAtA[i] = 0x28 + } if m.DispatcherDrainTargetEpoch != 0 { i = encodeVarintHeartbeat(dAtA, i, uint64(m.DispatcherDrainTargetEpoch)) i-- @@ -7549,6 +7566,9 @@ func (m *NodeHeartbeat) Size() (n int) { if m.DispatcherDrainTargetEpoch != 0 { n += 1 + sovHeartbeat(uint64(m.DispatcherDrainTargetEpoch)) } + if m.LogServiceDispatcherCount != 0 { + n += 1 + sovHeartbeat(uint64(m.LogServiceDispatcherCount)) + } return n } @@ -11320,6 +11340,25 @@ func (m *NodeHeartbeat) Unmarshal(dAtA []byte) error { break } } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LogServiceDispatcherCount", wireType) + } + m.LogServiceDispatcherCount = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LogServiceDispatcherCount |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipHeartbeat(dAtA[iNdEx:]) diff --git a/heartbeatpb/heartbeat.proto b/heartbeatpb/heartbeat.proto index b55a9117af..e9e322b204 100644 --- a/heartbeatpb/heartbeat.proto +++ b/heartbeatpb/heartbeat.proto @@ -206,6 +206,9 @@ message NodeHeartbeat { // currently applied on this node. Empty target means the drain target is clear. string dispatcher_drain_target_node_id = 3; uint64 dispatcher_drain_target_epoch = 4; + // log_service_dispatcher_count is the number of dispatchers currently + // reading from this node's event store. + uint32 log_service_dispatcher_count = 5; } // SetNodeLivenessRequest asks a node to transition its local liveness. diff --git a/logservice/eventstore/event_store.go b/logservice/eventstore/event_store.go index 07ec16e5e2..b068a5edc0 100644 --- a/logservice/eventstore/event_store.go +++ b/logservice/eventstore/event_store.go @@ -93,6 +93,9 @@ type EventStore interface { UpdateDispatcherCheckpointTs(dispatcherID common.DispatcherID, checkpointTs uint64) + // DispatcherCount returns the number of dispatchers currently reading from this event store. + DispatcherCount() int + // GetIterator returns an iterator for the requested range and resume cursor. GetIterator(dispatcherID common.DispatcherID, request ScanRequest) (EventIterator, error) @@ -720,6 +723,12 @@ func (e *eventStore) UnregisterDispatcher(changefeedID common.ChangeFeedID, disp zap.Stringer("dispatcherID", dispatcherID)) } +func (e *eventStore) DispatcherCount() int { + e.dispatcherMeta.RLock() + defer e.dispatcherMeta.RUnlock() + return len(e.dispatcherMeta.dispatcherStats) +} + func (e *eventStore) UpdateDispatcherCheckpointTs( dispatcherID common.DispatcherID, checkpointTs uint64, diff --git a/logservice/eventstore/event_store_test.go b/logservice/eventstore/event_store_test.go index 6b58725b18..ac41b62dd1 100644 --- a/logservice/eventstore/event_store_test.go +++ b/logservice/eventstore/event_store_test.go @@ -758,6 +758,7 @@ func TestEventStoreUnregisterDispatcherWithoutDataSharingRemovesSubscription(t * EndKey: []byte("h"), } require.True(t, store.RegisterDispatcher(cfID, dispatcherID, span, 100, func(uint64, uint64) {}, false, false)) + require.Equal(t, 1, store.DispatcherCount()) mockSubClient := subClient.(*mockSubscriptionClient) mockSubClient.mu.Lock() @@ -765,6 +766,7 @@ func TestEventStoreUnregisterDispatcherWithoutDataSharingRemovesSubscription(t * mockSubClient.mu.Unlock() store.UnregisterDispatcher(cfID, dispatcherID) + require.Zero(t, store.DispatcherCount()) mockSubClient.mu.Lock() require.Equal(t, 1, len(mockSubClient.subscriptions)) @@ -800,8 +802,10 @@ func TestEventStoreUnregisterDispatcherWithDataSharingKeepsSubscriptionForTTL(t EndKey: []byte("h"), } require.True(t, store.RegisterDispatcher(cfID, dispatcherID, span, 100, func(uint64, uint64) {}, false, false)) + require.Equal(t, 1, store.DispatcherCount()) store.UnregisterDispatcher(cfID, dispatcherID) + require.Zero(t, store.DispatcherCount()) mockSubClient := subClient.(*mockSubscriptionClient) mockSubClient.mu.Lock() diff --git a/maintainer/maintainer_manager_node.go b/maintainer/maintainer_manager_node.go index 3b67b3d5f1..3e368a474c 100644 --- a/maintainer/maintainer_manager_node.go +++ b/maintainer/maintainer_manager_node.go @@ -19,6 +19,7 @@ import ( "github.com/pingcap/log" "github.com/pingcap/ticdc/heartbeatpb" + appcontext "github.com/pingcap/ticdc/pkg/common/context" "github.com/pingcap/ticdc/pkg/liveness" "github.com/pingcap/ticdc/pkg/messaging" "github.com/pingcap/ticdc/pkg/node" @@ -29,6 +30,10 @@ import ( // Forced heartbeats bypass this throttle to acknowledge state changes immediately. const nodeHeartbeatInterval = 5 * time.Second +type logServiceDispatcherCounter interface { + DispatcherCount() int +} + // managerNodeState owns node-scoped state shared by all local maintainers. type managerNodeState struct { // liveness points to the server-wide node liveness state shared with other @@ -74,9 +79,10 @@ func newNodeEpoch() uint64 { return nodeEpoch } -// sendNodeHeartbeat reports node-scoped liveness and dispatcher drain target to -// coordinator. It is the authoritative acknowledgement channel for node-level -// drain state, including cases where no changefeed maintainer exists locally. +// sendNodeHeartbeat reports node-scoped liveness, dispatcher drain target, and +// the local log service dispatcher count to coordinator. It is the authoritative +// acknowledgement channel for node-level drain state, including cases where no +// changefeed maintainer exists locally. func (m *Manager) sendNodeHeartbeat(force bool) { if !m.isBootstrap() { return @@ -95,9 +101,14 @@ func (m *Manager) sendNodeHeartbeat(force bool) { currentLiveness = m.node.liveness.Load() } drainTarget, drainEpoch := m.getDispatcherDrainTarget() + logServiceDispatcherCount := 0 + if store, ok := appcontext.TryGetService[logServiceDispatcherCounter](appcontext.EventStore); ok { + logServiceDispatcherCount = store.DispatcherCount() + } hb := &heartbeatpb.NodeHeartbeat{ - Liveness: m.toNodeLivenessPB(currentLiveness), - NodeEpoch: m.node.nodeEpoch, + Liveness: m.toNodeLivenessPB(currentLiveness), + NodeEpoch: m.node.nodeEpoch, + LogServiceDispatcherCount: uint32(logServiceDispatcherCount), // Report the manager-level dispatcher drain target so coordinator can // confirm both activation and clearing even when no maintainers exist. DispatcherDrainTargetNodeId: drainTarget.String(), diff --git a/maintainer/node_liveness_test.go b/maintainer/node_liveness_test.go index b35e056dec..da9f86ea64 100644 --- a/maintainer/node_liveness_test.go +++ b/maintainer/node_liveness_test.go @@ -26,6 +26,14 @@ import ( "github.com/stretchr/testify/require" ) +type mockLogServiceDispatcherCounter struct { + count int +} + +func (m *mockLogServiceDispatcherCounter) DispatcherCount() int { + return m.count +} + func TestSetNodeLivenessRejectEpochMismatch(t *testing.T) { mc := messaging.NewMockMessageCenter() appcontext.SetService(appcontext.MessageCenter, mc) @@ -159,6 +167,11 @@ func TestSetDispatcherDrainTargetRejectStaleUpdate(t *testing.T) { func TestSetDispatcherDrainTargetSendsNodeHeartbeatAck(t *testing.T) { mc := messaging.NewMockMessageCenter() appcontext.SetService(appcontext.MessageCenter, mc) + logService := &mockLogServiceDispatcherCounter{count: 2} + appcontext.SetService(appcontext.EventStore, logService) + t.Cleanup(func() { + logService.count = 0 + }) var nodeLiveness liveness.Liveness m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness) @@ -185,6 +198,7 @@ func TestSetDispatcherDrainTargetSendsNodeHeartbeatAck(t *testing.T) { hb := apply("n2", 1) require.Equal(t, "n2", hb.DispatcherDrainTargetNodeId) require.Equal(t, uint64(1), hb.DispatcherDrainTargetEpoch) + require.Equal(t, uint32(2), hb.LogServiceDispatcherCount) hb = apply("", 1) require.Equal(t, "", hb.DispatcherDrainTargetNodeId) diff --git a/pkg/eventservice/event_service_test.go b/pkg/eventservice/event_service_test.go index fab9ac0db6..42f1628a16 100644 --- a/pkg/eventservice/event_service_test.go +++ b/pkg/eventservice/event_service_test.go @@ -287,6 +287,15 @@ func (m *mockEventStore) Close(ctx context.Context) error { func (m *mockEventStore) UpdateDispatcherCheckpointTs(dispatcherID common.DispatcherID, gcTS uint64) { } +func (m *mockEventStore) DispatcherCount() int { + count := 0 + m.dispatcherMap.Range(func(_, _ any) bool { + count++ + return true + }) + return count +} + func (m *mockEventStore) UnregisterDispatcher(changefeedID common.ChangeFeedID, dispatcherID common.DispatcherID) { span, ok := m.dispatcherMap.Load(dispatcherID) if ok { From ec0628612e7a65bd7eb58a9fe37f22dc433f3c50 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Wed, 12 Aug 2026 10:21:20 +0000 Subject: [PATCH 03/18] fmt Signed-off-by: wk989898 --- coordinator/controller_drain.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/coordinator/controller_drain.go b/coordinator/controller_drain.go index 0c0ef47e8e..677b4aa622 100644 --- a/coordinator/controller_drain.go +++ b/coordinator/controller_drain.go @@ -342,8 +342,7 @@ func (c *Controller) observeDrainNode(target node.ID, epoch uint64) drainNodeObs ) _, observation.drainingObserved, observation.stoppingObserved = c.drainController.GetStatus(target) - observation.logServiceDispatcherCount, observation.logServiceDispatcherCountObserved = - c.drainController.GetLogServiceDispatcherCount(target) + observation.logServiceDispatcherCount, observation.logServiceDispatcherCountObserved = c.drainController.GetLogServiceDispatcherCount(target) observation.nodeState = c.drainController.GetState(target) return observation } From f4adda14551f9a83d4468ee17dddbb172f668a9f Mon Sep 17 00:00:00 2001 From: wk989898 Date: Thu, 13 Aug 2026 07:33:08 +0000 Subject: [PATCH 04/18] update Signed-off-by: wk989898 --- .../eventcollector/event_collector.go | 35 +++++++++---- .../eventcollector/event_collector_test.go | 27 +++++++--- logservice/schemastore/schema_store.go | 4 ++ logservice/schemastore/schema_store_test.go | 32 ++++++++++++ maintainer/maintainer_manager_node.go | 8 +++ maintainer/node_liveness_test.go | 37 ++++++++++++++ pkg/common/event/dispatcher_heartbeat.go | 13 ++++- pkg/common/event/dispatcher_heartbeat_test.go | 14 ++---- pkg/config/capture.go | 11 ++-- pkg/eventservice/event_service.go | 18 +++++++ pkg/eventservice/event_service_test.go | 15 ++++++ pkg/messaging/message_center.go | 19 +++++-- pkg/messaging/mock_message_center.go | 18 ++++++- pkg/messaging/remote_target.go | 50 +++++++++++-------- pkg/messaging/target_test.go | 23 +++++++++ pkg/node/node.go | 33 +++++++----- pkg/node/node_test.go | 37 ++++++++++++++ server/server.go | 7 +-- server/server_prepare.go | 15 +++--- 19 files changed, 336 insertions(+), 80 deletions(-) create mode 100644 pkg/node/node_test.go diff --git a/downstreamadapter/eventcollector/event_collector.go b/downstreamadapter/eventcollector/event_collector.go index 6456f1ada2..03543c4fe5 100644 --- a/downstreamadapter/eventcollector/event_collector.go +++ b/downstreamadapter/eventcollector/event_collector.go @@ -415,7 +415,7 @@ func (c *EventCollector) groupHeartbeat() map[node.ID]*event.DispatcherHeartbeat group := func(target node.ID, dispatcherID common.DispatcherID, checkpointTs uint64, epoch uint64) { heartbeat, ok := groupedHeartbeats[target] if !ok { - heartbeat = event.NewDispatcherHeartbeat() + heartbeat = event.NewDispatcherHeartbeatWithVersion(c.eventServiceProtocolVersion(target)) groupedHeartbeats[target] = heartbeat } heartbeat.AddDispatcherProgress(dispatcherID, checkpointTs, epoch) @@ -439,6 +439,17 @@ func (c *EventCollector) groupHeartbeat() map[node.ID]*event.DispatcherHeartbeat return groupedHeartbeats } +func (c *EventCollector) eventServiceProtocolVersion(target node.ID) int { + if target == c.serverId { + return node.CurrentMessagingProtocolVersion + } + info := c.mc.GetNodeInfo(target) + if info != nil && info.MessagingProtocolVersion >= node.CurrentMessagingProtocolVersion { + return node.CurrentMessagingProtocolVersion + } + return node.LegacyMessagingProtocolVersion +} + func (c *EventCollector) processDSFeedback(ctx context.Context) error { log.Info("Start process feedback from dynamic stream") defer log.Info("Stop process feedback from dynamic stream") @@ -727,7 +738,8 @@ func (c *EventCollector) newCongestionControlMessages() map[node.ID]*event.Conge // build congestion control messages for each node result := make(map[node.ID]*event.CongestionControl) for nodeID, changefeedDispatchers := range nodeDispatcherMemory { - congestionControl := event.NewCongestionControlWithVersion(event.CongestionControlVersion2) + protocolVersion := c.eventServiceProtocolVersion(nodeID) + congestionControl := event.NewCongestionControlWithVersion(protocolVersion) for changefeedID, dispatcherMemory := range changefeedDispatchers { if len(dispatcherMemory) == 0 { @@ -739,13 +751,18 @@ func (c *EventCollector) newCongestionControlMessages() map[node.ID]*event.Conge if !ok { continue } - congestionControl.AddAvailableMemoryWithDispatchersAndUsageAndReleaseCount( - changefeedID.ID(), - totalAvailable, - changefeedUsageRatio[changefeedID], - dispatcherMemory, - getAndResetMemoryReleaseCount(changefeedID), - ) + if protocolVersion >= event.CongestionControlVersion2 { + congestionControl.AddAvailableMemoryWithDispatchersAndUsageAndReleaseCount( + changefeedID.ID(), + totalAvailable, + changefeedUsageRatio[changefeedID], + dispatcherMemory, + getAndResetMemoryReleaseCount(changefeedID), + ) + } else { + congestionControl.AddAvailableMemoryWithDispatchers( + changefeedID.ID(), totalAvailable, dispatcherMemory) + } } if len(congestionControl.GetAvailables()) > 0 { diff --git a/downstreamadapter/eventcollector/event_collector_test.go b/downstreamadapter/eventcollector/event_collector_test.go index 1d29b8568e..b508b21367 100644 --- a/downstreamadapter/eventcollector/event_collector_test.go +++ b/downstreamadapter/eventcollector/event_collector_test.go @@ -272,11 +272,8 @@ func TestRemoveLastDispatcher(t *testing.T) { } func TestGroupHeartbeatUsesEpochAndClamp(t *testing.T) { - ctx := context.Background() serverInfo := node.NewInfo("127.0.0.1:18300", "") - mc := messaging.NewMessageCenter(ctx, serverInfo.ID, config.NewDefaultMessageCenterConfig(serverInfo.AdvertiseAddr), nil) - mc.Run(ctx) - defer mc.Close() + mc := messaging.NewMockMessageCenter() appcontext.SetService(appcontext.MessageCenter, mc) c := New(serverInfo.ID) @@ -320,12 +317,28 @@ func TestGroupHeartbeatUsesEpochAndClamp(t *testing.T) { remoteHeartbeat := grouped[remoteID] require.NotNil(t, remoteHeartbeat) + require.Equal(t, commonEvent.DispatcherHeartbeatVersion1, remoteHeartbeat.Version) + require.Len(t, remoteHeartbeat.DispatcherProgressesLegacy, 1) + require.Equal(t, remoteDispatcher.id, remoteHeartbeat.DispatcherProgressesLegacy[0].DispatcherID) + require.Equal(t, uint64(210), remoteHeartbeat.DispatcherProgressesLegacy[0].CheckpointTs) + + controlMessages := c.newCongestionControlMessages() + require.Equal(t, commonEvent.CongestionControlVersion2, controlMessages[serverInfo.ID].GetVersion()) + require.Equal(t, commonEvent.CongestionControlVersion1, controlMessages[remoteID].GetVersion()) + + mc.OnNodeChanges(map[node.ID]*node.Info{ + remoteID: { + ID: remoteID, + MessagingProtocolVersion: node.CurrentMessagingProtocolVersion, + }, + }) + grouped = c.groupHeartbeat() + remoteHeartbeat = grouped[remoteID] require.Equal(t, commonEvent.DispatcherHeartbeatVersion2, remoteHeartbeat.Version) require.Len(t, remoteHeartbeat.DispatcherProgresses, 1) - require.Equal(t, uint8(commonEvent.DispatcherProgressVersion1), remoteHeartbeat.DispatcherProgresses[0].Version) - require.Equal(t, remoteDispatcher.id, remoteHeartbeat.DispatcherProgresses[0].DispatcherID) - require.Equal(t, uint64(210), remoteHeartbeat.DispatcherProgresses[0].CheckpointTs) require.Equal(t, uint64(5), remoteHeartbeat.DispatcherProgresses[0].Epoch) + controlMessages = c.newCongestionControlMessages() + require.Equal(t, commonEvent.CongestionControlVersion2, controlMessages[remoteID].GetVersion()) } func TestGroupHeartbeatResetThenHandshake(t *testing.T) { diff --git a/logservice/schemastore/schema_store.go b/logservice/schemastore/schema_store.go index 5273c93ca8..aaeccf2d58 100644 --- a/logservice/schemastore/schema_store.go +++ b/logservice/schemastore/schema_store.go @@ -74,6 +74,7 @@ type DDLEventState struct { type keyspaceSchemaStore struct { cancel context.CancelFunc + wg sync.WaitGroup ddlJobFetcher *ddlJobFetcher gcKeeper *schemaStoreGCKeeper pdClock pdutil.Clock @@ -349,6 +350,7 @@ func (s *schemaStore) Close(ctx context.Context) error { if store.cancel != nil { store.cancel() } + store.wg.Wait() if store.gcKeeper != nil { err := closeSchemaStoreGCKeeper(keyspaceID, store.gcKeeper) if err != nil { @@ -603,7 +605,9 @@ func (s *schemaStore) RegisterKeyspace( return store.resolvedTs.Load() }) + store.wg.Add(1) go func(ctx context.Context, schemaStore *keyspaceSchemaStore) { + defer schemaStore.wg.Done() ticker := time.NewTicker(50 * time.Millisecond) defer ticker.Stop() for { diff --git a/logservice/schemastore/schema_store_test.go b/logservice/schemastore/schema_store_test.go index 1c30d52f95..aacd1d3b31 100644 --- a/logservice/schemastore/schema_store_test.go +++ b/logservice/schemastore/schema_store_test.go @@ -152,6 +152,38 @@ func TestIgnoreDDLByCommitTs(t *testing.T) { require.NotContains(t, tableNames, "t2") } +func TestSchemaStoreCloseWaitsForWorkersBeforeClosingPebble(t *testing.T) { + pstorage := newPersistentStorageForTest(t.TempDir(), nil) + workerStarted := make(chan struct{}) + releaseWorker := make(chan struct{}) + store := &keyspaceSchemaStore{dataStorage: pstorage} + store.wg.Add(1) + go func() { + defer store.wg.Done() + close(workerStarted) + <-releaseWorker + }() + <-workerStarted + + schemaStore := &schemaStore{ + keyspaceSchemaStoreMap: map[uint32]*keyspaceSchemaStore{0: store}, + } + closeDone := make(chan error, 1) + go func() { + closeDone <- schemaStore.Close(context.Background()) + }() + + select { + case err := <-closeDone: + require.FailNow(t, "schema store closed before its worker exited", "error: %v", err) + case <-time.After(50 * time.Millisecond): + } + require.NoError(t, pstorage.db.Flush()) + + close(releaseWorker) + require.NoError(t, <-closeDone) +} + func TestTryUpdateResolvedTsRetryAfterDDLHandleFailure(t *testing.T) { mockPDClock := pdutil.NewClock4Test() appcontext.SetService(appcontext.DefaultPDClock, mockPDClock) diff --git a/maintainer/maintainer_manager_node.go b/maintainer/maintainer_manager_node.go index 3e368a474c..8b42600798 100644 --- a/maintainer/maintainer_manager_node.go +++ b/maintainer/maintainer_manager_node.go @@ -95,6 +95,14 @@ func (m *Manager) sendNodeHeartbeat(force bool) { // Update before sending so a transient send failure will not cause // frequent retries and log spam on the 200ms tick. m.node.lastNodeHeartbeatSentAt = now + // NodeHeartbeat uses an IO type that legacy coordinators do not recognize. + // Suppress it unless membership explicitly advertises current messaging + // support, so a new capture can safely bootstrap against an old coordinator. + coordinatorInfo := m.mc.GetNodeInfo(m.coordinatorID) + if coordinatorInfo == nil || + coordinatorInfo.MessagingProtocolVersion < node.CurrentMessagingProtocolVersion { + return + } currentLiveness := liveness.CaptureAlive if m.node.liveness != nil { diff --git a/maintainer/node_liveness_test.go b/maintainer/node_liveness_test.go index da9f86ea64..34097ecfa4 100644 --- a/maintainer/node_liveness_test.go +++ b/maintainer/node_liveness_test.go @@ -34,6 +34,17 @@ func (m *mockLogServiceDispatcherCounter) DispatcherCount() int { return m.count } +func registerCurrentCoordinator(mc interface { + OnNodeChanges(map[node.ID]*node.Info) +}, coordinatorID node.ID) { + mc.OnNodeChanges(map[node.ID]*node.Info{ + coordinatorID: { + ID: coordinatorID, + MessagingProtocolVersion: node.CurrentMessagingProtocolVersion, + }, + }) +} + func TestSetNodeLivenessRejectEpochMismatch(t *testing.T) { mc := messaging.NewMockMessageCenter() appcontext.SetService(appcontext.MessageCenter, mc) @@ -42,6 +53,7 @@ func TestSetNodeLivenessRejectEpochMismatch(t *testing.T) { m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness) m.coordinatorID = node.ID("coordinator") m.coordinatorVersion = 1 + registerCurrentCoordinator(mc, m.coordinatorID) req := &heartbeatpb.SetNodeLivenessRequest{ Target: heartbeatpb.NodeLiveness_DRAINING, @@ -68,6 +80,7 @@ func TestSetNodeLivenessApplyTransition(t *testing.T) { m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness) m.coordinatorID = node.ID("coordinator") m.coordinatorVersion = 1 + registerCurrentCoordinator(mc, m.coordinatorID) req := &heartbeatpb.SetNodeLivenessRequest{ Target: heartbeatpb.NodeLiveness_DRAINING, @@ -177,6 +190,7 @@ func TestSetDispatcherDrainTargetSendsNodeHeartbeatAck(t *testing.T) { m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness) m.coordinatorID = node.ID("coordinator") m.coordinatorVersion = 1 + registerCurrentCoordinator(mc, m.coordinatorID) apply := func(target string, epoch uint64) *heartbeatpb.NodeHeartbeat { msg := messaging.NewSingleTargetMessage( @@ -232,6 +246,29 @@ func TestCoordinatorBootstrapResponseIncludesDispatcherDrainTarget(t *testing.T) resp := out.Message[0].(*heartbeatpb.CoordinatorBootstrapResponse) require.Equal(t, "n2", resp.DispatcherDrainTargetNodeId) require.Equal(t, uint64(7), resp.DispatcherDrainTargetEpoch) + require.Empty(t, mc.GetMessageChannel()) +} + +func TestCoordinatorBootstrapSendsNodeHeartbeatToCurrentCoordinator(t *testing.T) { + mc := messaging.NewMockMessageCenter() + appcontext.SetService(appcontext.MessageCenter, mc) + coordinatorID := node.ID("coordinator") + registerCurrentCoordinator(mc, coordinatorID) + + var nodeLiveness liveness.Liveness + m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness) + req := messaging.NewSingleTargetMessage( + m.nodeInfo.ID, + messaging.MaintainerManagerTopic, + &heartbeatpb.CoordinatorBootstrapRequest{Version: 1}, + ) + req.From = coordinatorID + m.onCoordinatorBootstrapRequest(req) + + first := <-mc.GetMessageChannel() + second := <-mc.GetMessageChannel() + require.Equal(t, messaging.TypeCoordinatorBootstrapResponse, first.Type) + require.Equal(t, messaging.TypeNodeHeartbeatRequest, second.Type) } func TestAddMaintainerIgnoreInvalidConfig(t *testing.T) { diff --git a/pkg/common/event/dispatcher_heartbeat.go b/pkg/common/event/dispatcher_heartbeat.go index 793149bc85..0a9dd2ff52 100644 --- a/pkg/common/event/dispatcher_heartbeat.go +++ b/pkg/common/event/dispatcher_heartbeat.go @@ -107,8 +107,12 @@ type DispatcherHeartbeat struct { } func NewDispatcherHeartbeat() *DispatcherHeartbeat { + return NewDispatcherHeartbeatWithVersion(DispatcherHeartbeatVersion2) +} + +func NewDispatcherHeartbeatWithVersion(version int) *DispatcherHeartbeat { return &DispatcherHeartbeat{ - Version: DispatcherHeartbeatVersion2, + Version: version, // TODO: Pass a real clusterID when we support 1 TiCDC cluster subscribe multiple TiDB clusters ClusterID: 0, DispatcherProgressesLegacy: make([]DispatcherProgressLegacy, 0), @@ -118,6 +122,13 @@ func NewDispatcherHeartbeat() *DispatcherHeartbeat { func (d *DispatcherHeartbeat) AddDispatcherProgress(dispatcherID common.DispatcherID, checkpointTs uint64, epoch uint64) { d.DispatcherCount++ + if d.Version == DispatcherHeartbeatVersion1 { + d.DispatcherProgressesLegacy = append(d.DispatcherProgressesLegacy, DispatcherProgressLegacy{ + DispatcherID: dispatcherID, + CheckpointTs: checkpointTs, + }) + return + } d.DispatcherProgresses = append(d.DispatcherProgresses, DispatcherProgress{ Version: DispatcherProgressVersion1, DispatcherID: dispatcherID, diff --git a/pkg/common/event/dispatcher_heartbeat_test.go b/pkg/common/event/dispatcher_heartbeat_test.go index 649b707367..cf4811a0da 100644 --- a/pkg/common/event/dispatcher_heartbeat_test.go +++ b/pkg/common/event/dispatcher_heartbeat_test.go @@ -151,17 +151,9 @@ func TestDispatcherHeartbeatBackwardCompatibleV1(t *testing.T) { t.Parallel() dispatcherID := common.NewDispatcherID() - heartbeat := &DispatcherHeartbeat{ - Version: DispatcherHeartbeatVersion1, - ClusterID: 123, - DispatcherProgressesLegacy: []DispatcherProgressLegacy{ - { - DispatcherID: dispatcherID, - CheckpointTs: 456, - }, - }, - DispatcherCount: 1, - } + heartbeat := NewDispatcherHeartbeatWithVersion(DispatcherHeartbeatVersion1) + heartbeat.ClusterID = 123 + heartbeat.AddDispatcherProgress(dispatcherID, 456, 789) data, err := heartbeat.Marshal() require.NoError(t, err) diff --git a/pkg/config/capture.go b/pkg/config/capture.go index 65bd9d433f..a5114936f6 100644 --- a/pkg/config/capture.go +++ b/pkg/config/capture.go @@ -45,11 +45,12 @@ type CaptureInfo struct { ID CaptureID `json:"id"` AdvertiseAddr string `json:"address"` - Version string `json:"version"` - GitHash string `json:"git-hash"` - DeployPath string `json:"deploy-path"` - StartTimestamp int64 `json:"start-timestamp"` - IsNewArch bool `json:"is-new-arch"` + Version string `json:"version"` + GitHash string `json:"git-hash"` + DeployPath string `json:"deploy-path"` + StartTimestamp int64 `json:"start-timestamp"` + IsNewArch bool `json:"is-new-arch"` + MessagingProtocolVersion int `json:"messaging-protocol-version,omitempty"` } // Marshal using json.Marshal. diff --git a/pkg/eventservice/event_service.go b/pkg/eventservice/event_service.go index 5f4656742a..254b481cfb 100644 --- a/pkg/eventservice/event_service.go +++ b/pkg/eventservice/event_service.go @@ -15,6 +15,7 @@ package eventservice import ( "context" + "sync" "time" "github.com/pingcap/log" @@ -80,6 +81,9 @@ type eventService struct { // TODO: use a better way to cache the acceptorInfos dispatcherInfoChan chan DispatcherInfo dispatcherHeartbeat chan *DispatcherHeartBeatWithServerID + handlerMu sync.Mutex + handlerWG sync.WaitGroup + closing bool tz *time.Location } @@ -160,6 +164,11 @@ func (s *eventService) Run(ctx context.Context) error { func (s *eventService) Close(_ context.Context) error { log.Info("event service is closing") + s.handlerMu.Lock() + s.closing = true + s.handlerMu.Unlock() + s.mc.DeRegisterHandler(messaging.EventServiceTopic) + s.handlerWG.Wait() for _, c := range s.brokers { c.close() } @@ -168,6 +177,15 @@ func (s *eventService) Close(_ context.Context) error { } func (s *eventService) handleMessage(ctx context.Context, msg *messaging.TargetMessage) error { + s.handlerMu.Lock() + if s.closing { + s.handlerMu.Unlock() + return nil + } + s.handlerWG.Add(1) + s.handlerMu.Unlock() + defer s.handlerWG.Done() + switch msg.Type { case messaging.TypeDispatcherRequest: infos := msgToDispatcherInfo(msg) diff --git a/pkg/eventservice/event_service_test.go b/pkg/eventservice/event_service_test.go index 25864521b0..56b4ca1aed 100644 --- a/pkg/eventservice/event_service_test.go +++ b/pkg/eventservice/event_service_test.go @@ -201,6 +201,21 @@ func TestHandleMessageIgnoresInvalidSingleMessagePayloads(t *testing.T) { }) } +func TestHandleMessageIgnoredAfterClose(t *testing.T) { + mc := messaging.NewMockMessageCenter() + es := &eventService{ + mc: mc, + brokers: make(map[uint64]*eventBroker), + } + require.NoError(t, es.Close(context.Background())) + + heartbeat := commonEvent.NewDispatcherHeartbeat() + require.NoError(t, es.handleMessage(context.Background(), &messaging.TargetMessage{ + Type: messaging.TypeDispatcherHeartbeat, + Message: []messaging.IOTypeT{heartbeat}, + })) +} + var _ eventstore.EventStore = &mockEventStore{} // mockEventStore is a mock implementation of the EventStore interface diff --git a/pkg/messaging/message_center.go b/pkg/messaging/message_center.go index 0a11c31dab..0361ea945f 100644 --- a/pkg/messaging/message_center.go +++ b/pkg/messaging/message_center.go @@ -43,6 +43,7 @@ const ( type MessageCenter interface { MessageSender MessageReceiver + GetNodeInfo(node.ID) *node.Info // OnNodeChanges is called when the nodes in the cluster are changed. The message center should update the target list. OnNodeChanges(map[node.ID]*node.Info) Close() @@ -87,8 +88,11 @@ type messageCenter struct { m map[node.ID]*remoteMessageTarget } - remoteNodeInfos map[node.ID]*node.Info - notifyCh chan map[node.ID]*node.Info + nodeInfos struct { + sync.RWMutex + m map[node.ID]*node.Info + } + notifyCh chan map[node.ID]*node.Info grpcServer *grpc.Server router *router @@ -122,6 +126,7 @@ func NewMessageCenter( notifyCh: make(chan map[node.ID]*node.Info, 128), } mc.remoteTargets.m = make(map[node.ID]*remoteMessageTarget) + mc.nodeInfos.m = make(map[node.ID]*node.Info) log.Info("create message center success.", zap.Stringer("id", id), zap.String("addr", cfg.Addr)) @@ -195,6 +200,12 @@ func (mc *messageCenter) DeRegisterHandler(topic string) { mc.router.deRegisterHandler(topic) } +func (mc *messageCenter) GetNodeInfo(id node.ID) *node.Info { + mc.nodeInfos.RLock() + defer mc.nodeInfos.RUnlock() + return mc.nodeInfos.m[id] +} + func (mc *messageCenter) OnNodeChanges(activeNode map[node.ID]*node.Info) { mc.notifyCh <- activeNode log.Info("notify node changes", zap.Any("activeNode", activeNode)) @@ -205,7 +216,9 @@ func (mc *messageCenter) handleNodeChanges(activeNode map[node.ID]*node.Info) { for id, node := range activeNode { infosMap[id] = node } - mc.remoteNodeInfos = infosMap + mc.nodeInfos.Lock() + mc.nodeInfos.m = infosMap + mc.nodeInfos.Unlock() currentTargets := make(map[node.ID]bool) currentTargets[mc.id] = true diff --git a/pkg/messaging/mock_message_center.go b/pkg/messaging/mock_message_center.go index 6425224260..25436c66c8 100644 --- a/pkg/messaging/mock_message_center.go +++ b/pkg/messaging/mock_message_center.go @@ -13,13 +13,18 @@ package messaging -import "github.com/pingcap/ticdc/pkg/node" +import ( + "sync" + + "github.com/pingcap/ticdc/pkg/node" +) var _ MessageCenter = &mockMessageCenter{} // mockMessageCenter is a mock implementation of the MessageCenter interface type mockMessageCenter struct { messageCh chan *TargetMessage + nodeInfos sync.Map } func NewMockMessageCenter() *mockMessageCenter { @@ -33,6 +38,17 @@ func (m *mockMessageCenter) GetMessageChannel() chan *TargetMessage { } func (m *mockMessageCenter) OnNodeChanges(nodeInfos map[node.ID]*node.Info) { + for id, info := range nodeInfos { + m.nodeInfos.Store(id, info) + } +} + +func (m *mockMessageCenter) GetNodeInfo(id node.ID) *node.Info { + info, ok := m.nodeInfos.Load(id) + if !ok { + return nil + } + return info.(*node.Info) } func (m *mockMessageCenter) SendEvent(event *TargetMessage) error { diff --git a/pkg/messaging/remote_target.go b/pkg/messaging/remote_target.go index 852ebe6336..3a108f893b 100644 --- a/pkg/messaging/remote_target.go +++ b/pkg/messaging/remote_target.go @@ -619,31 +619,39 @@ func (s *remoteMessageTarget) handleIncomingMessage(ctx context.Context, stream return err } - mt := IOType(message.Type) - - targetMsg := &TargetMessage{ - From: node.ID(message.From), - To: node.ID(message.To), - Topic: message.Topic, - Type: mt, + targetMsg := s.decodeMessage(message) + if targetMsg != nil { + ch <- targetMsg } + } +} - for _, payload := range message.Payload { - msg, err := decodeIOType(mt, payload) - if err != nil { - log.Error("Failed to decode message", - zap.Error(err), - zap.Stringer("localID", s.messageCenterID), - zap.String("localAddr", s.localAddr), - zap.Stringer("remoteID", s.targetId), - zap.String("remoteAddr", s.targetAddr)) - continue - } - targetMsg.Message = append(targetMsg.Message, msg) - } +func (s *remoteMessageTarget) decodeMessage(message *proto.Message) *TargetMessage { + mt := IOType(message.Type) + targetMsg := &TargetMessage{ + From: node.ID(message.From), + To: node.ID(message.To), + Topic: message.Topic, + Type: mt, + } - ch <- targetMsg + for _, payload := range message.Payload { + msg, err := decodeIOType(mt, payload) + if err != nil { + log.Error("Failed to decode message", + zap.Error(err), + zap.Stringer("localID", s.messageCenterID), + zap.String("localAddr", s.localAddr), + zap.Stringer("remoteID", s.targetId), + zap.String("remoteAddr", s.targetAddr)) + continue + } + targetMsg.Message = append(targetMsg.Message, msg) + } + if len(targetMsg.Message) == 0 { + return nil } + return targetMsg } // Create a new protocol message from target messages diff --git a/pkg/messaging/target_test.go b/pkg/messaging/target_test.go index 6cffb6b114..7b76e3fbc5 100644 --- a/pkg/messaging/target_test.go +++ b/pkg/messaging/target_test.go @@ -18,7 +18,9 @@ import ( "testing" "github.com/pingcap/log" + commonEvent "github.com/pingcap/ticdc/pkg/common/event" "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/ticdc/pkg/messaging/proto" "github.com/pingcap/ticdc/pkg/node" "github.com/stretchr/testify/require" "go.uber.org/zap" @@ -34,6 +36,27 @@ func newRemoteMessageTargetForTest() *remoteMessageTarget { return rt } +func TestDecodeMessageDropsEmptyDecodedPayload(t *testing.T) { + rt := newRemoteMessageTargetForTest() + defer rt.close() + + received := rt.decodeMessage(&proto.Message{ + Type: int32(TypeDispatcherHeartbeat), + Payload: [][]byte{{0}}, + }) + require.Nil(t, received) + + heartbeat := commonEvent.NewDispatcherHeartbeat() + payload, err := heartbeat.Marshal() + require.NoError(t, err) + received = rt.decodeMessage(&proto.Message{ + Type: int32(TypeDispatcherHeartbeat), + Payload: [][]byte{payload}, + }) + require.NotNil(t, received) + require.Len(t, received.Message, 1) +} + func TestRemoteTargetNewMessage(t *testing.T) { rt := newRemoteMessageTargetForTest() defer rt.close() diff --git a/pkg/node/node.go b/pkg/node/node.go index 148c87f926..a4e8a66289 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -26,6 +26,11 @@ import ( type ID string +const ( + LegacyMessagingProtocolVersion = 1 + CurrentMessagingProtocolVersion = 2 +) + func (s ID) String() string { return string(s) } @@ -52,18 +57,21 @@ type Info struct { DeployPath string `json:"deploy-path"` StartTimestamp int64 `json:"start-timestamp"` + MessagingProtocolVersion int `json:"messaging-protocol-version,omitempty"` + // Epoch represents how many times the node has been restarted. Epoch uint64 `json:"epoch"` } func NewInfo(addr string, deployPath string) *Info { return &Info{ - ID: NewID(), - AdvertiseAddr: addr, - Version: version.ReleaseVersion, - GitHash: version.GitHash, - DeployPath: deployPath, - StartTimestamp: time.Now().Unix(), + ID: NewID(), + AdvertiseAddr: addr, + Version: version.ReleaseVersion, + GitHash: version.GitHash, + DeployPath: deployPath, + StartTimestamp: time.Now().Unix(), + MessagingProtocolVersion: CurrentMessagingProtocolVersion, } } @@ -91,12 +99,13 @@ func (c *Info) Unmarshal(data []byte) error { func CaptureInfoToNodeInfo(captureInfo *config.CaptureInfo) *Info { return &Info{ - ID: ID(captureInfo.ID), - AdvertiseAddr: captureInfo.AdvertiseAddr, - Version: captureInfo.Version, - GitHash: captureInfo.GitHash, - DeployPath: captureInfo.DeployPath, - StartTimestamp: captureInfo.StartTimestamp, + ID: ID(captureInfo.ID), + AdvertiseAddr: captureInfo.AdvertiseAddr, + Version: captureInfo.Version, + GitHash: captureInfo.GitHash, + DeployPath: captureInfo.DeployPath, + StartTimestamp: captureInfo.StartTimestamp, + MessagingProtocolVersion: captureInfo.MessagingProtocolVersion, } } diff --git a/pkg/node/node_test.go b/pkg/node/node_test.go new file mode 100644 index 0000000000..f3d1db73bb --- /dev/null +++ b/pkg/node/node_test.go @@ -0,0 +1,37 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package node + +import ( + "testing" + + "github.com/pingcap/ticdc/pkg/config" + "github.com/stretchr/testify/require" +) + +func TestMessagingProtocolVersion(t *testing.T) { + current := NewInfo("127.0.0.1:8300", "") + require.Equal(t, CurrentMessagingProtocolVersion, current.MessagingProtocolVersion) + + legacy := CaptureInfoToNodeInfo(&config.CaptureInfo{ + ID: config.CaptureID("legacy-node"), + }) + require.Zero(t, legacy.MessagingProtocolVersion) + + capable := CaptureInfoToNodeInfo(&config.CaptureInfo{ + ID: config.CaptureID("capable-node"), + MessagingProtocolVersion: CurrentMessagingProtocolVersion, + }) + require.Equal(t, CurrentMessagingProtocolVersion, capable.MessagingProtocolVersion) +} diff --git a/server/server.go b/server/server.go index 8a604d35cc..eb61e1d50e 100644 --- a/server/server.go +++ b/server/server.go @@ -645,9 +645,10 @@ func (c *server) GetCoordinatorInfo(ctx context.Context) (*node.Info, error) { ID: node.ID(captureInfo.ID), AdvertiseAddr: captureInfo.AdvertiseAddr, - Version: captureInfo.Version, - DeployPath: captureInfo.DeployPath, - StartTimestamp: captureInfo.StartTimestamp, + Version: captureInfo.Version, + DeployPath: captureInfo.DeployPath, + StartTimestamp: captureInfo.StartTimestamp, + MessagingProtocolVersion: captureInfo.MessagingProtocolVersion, // Epoch is now not used in TiCDC, so we just set it to 0. Epoch: 0, diff --git a/server/server_prepare.go b/server/server_prepare.go index 60621e4dc7..60f46fd15b 100644 --- a/server/server_prepare.go +++ b/server/server_prepare.go @@ -201,13 +201,14 @@ func (c *server) setUpDir() { // registerNodeToEtcd the server by put the server's information in etcd func (c *server) registerNodeToEtcd(ctx context.Context) error { cInfo := &config.CaptureInfo{ - ID: config.CaptureID(c.info.ID), - AdvertiseAddr: c.info.AdvertiseAddr, - Version: c.info.Version, - GitHash: c.info.GitHash, - DeployPath: c.info.DeployPath, - StartTimestamp: c.info.StartTimestamp, - IsNewArch: true, + ID: config.CaptureID(c.info.ID), + AdvertiseAddr: c.info.AdvertiseAddr, + Version: c.info.Version, + GitHash: c.info.GitHash, + DeployPath: c.info.DeployPath, + StartTimestamp: c.info.StartTimestamp, + IsNewArch: true, + MessagingProtocolVersion: c.info.MessagingProtocolVersion, } err := c.EtcdClient.PutCaptureInfo(ctx, cInfo, c.session.Lease()) if err != nil { From de8b863e37d74237d108c8de28f1accb858ac64a Mon Sep 17 00:00:00 2001 From: wk989898 Date: Thu, 13 Aug 2026 08:57:14 +0000 Subject: [PATCH 05/18] update Signed-off-by: wk989898 --- coordinator/controller.go | 1 + coordinator/controller_drain.go | 12 +++++- coordinator/controller_drain_test.go | 22 +++++++++++ .../eventcollector/event_collector.go | 14 ++++--- .../eventcollector/event_collector_test.go | 5 ++- heartbeatpb/drain_protocol.go | 11 +++++- maintainer/maintainer_manager_node.go | 9 +++-- maintainer/node_liveness_test.go | 18 +++++---- pkg/config/capture.go | 11 +++--- pkg/node/node.go | 33 ++++++----------- pkg/node/node_test.go | 37 ------------------- server/server.go | 7 ++-- server/server_prepare.go | 15 ++++---- 13 files changed, 97 insertions(+), 98 deletions(-) delete mode 100644 pkg/node/node_test.go diff --git a/coordinator/controller.go b/coordinator/controller.go index 682e7e995f..be254f7654 100644 --- a/coordinator/controller.go +++ b/coordinator/controller.go @@ -1165,6 +1165,7 @@ func (c *Controller) RemoveNode(id node.ID) { observation.remaining, observation.logServiceDispatcherCount, observation.logServiceDispatcherCountObserved, + observation.logServiceDispatcherCountRequired, ) } diff --git a/coordinator/controller_drain.go b/coordinator/controller_drain.go index 677b4aa622..8cfbd01bfa 100644 --- a/coordinator/controller_drain.go +++ b/coordinator/controller_drain.go @@ -207,6 +207,7 @@ func (c *Controller) DrainNode(ctx context.Context, target node.ID) (int, error) observation.remaining, observation.logServiceDispatcherCount, observation.logServiceDispatcherCountObserved, + observation.logServiceDispatcherCountRequired, ) if completionObserved { @@ -229,6 +230,7 @@ func (c *Controller) DrainNode(ctx context.Context, target node.ID) (int, error) zap.Int("pendingStatusCount", observation.pendingStatusCount), zap.Uint32("logServiceDispatcherCount", observation.logServiceDispatcherCount), zap.Bool("logServiceDispatcherCountObserved", observation.logServiceDispatcherCountObserved), + zap.Bool("logServiceDispatcherCountRequired", observation.logServiceDispatcherCountRequired), zap.Int("remaining", observation.remaining)) return ensureDrainRemainingNonZero(observation.remaining), nil } @@ -246,6 +248,7 @@ func (c *Controller) observeRemovedActiveDrainTarget(target node.ID, epoch uint6 observation.remaining, observation.logServiceDispatcherCount, observation.logServiceDispatcherCountObserved, + observation.logServiceDispatcherCountRequired, ) if completionObserved { log.Info("drain completion observed for removed active target", @@ -324,6 +327,7 @@ type drainNodeObservation struct { stoppingObserved bool logServiceDispatcherCount uint32 logServiceDispatcherCountObserved bool + logServiceDispatcherCountRequired bool } func (c *Controller) observeDrainNode(target node.ID, epoch uint64) drainNodeObservation { @@ -343,6 +347,9 @@ func (c *Controller) observeDrainNode(target node.ID, epoch uint64) drainNodeObs _, observation.drainingObserved, observation.stoppingObserved = c.drainController.GetStatus(target) observation.logServiceDispatcherCount, observation.logServiceDispatcherCountObserved = c.drainController.GetLogServiceDispatcherCount(target) + drainProtocolVersion, drainProtocolObserved := c.drainController.GetDrainProtocolVersion(target) + observation.logServiceDispatcherCountRequired = !drainProtocolObserved || + heartbeatpb.SupportsLogServiceDispatcherCount(drainProtocolVersion) observation.nodeState = c.drainController.GetState(target) return observation } @@ -1095,14 +1102,15 @@ func isBestEffortDrainComplete( remaining int, logServiceDispatcherCount uint32, logServiceDispatcherCountObserved bool, + logServiceDispatcherCountRequired bool, ) bool { if nodeState == drain.StateUnknown || !drainingObserved { return false } return stoppingObserved && remaining == 0 && - logServiceDispatcherCountObserved && - logServiceDispatcherCount == 0 + (!logServiceDispatcherCountRequired || + logServiceDispatcherCountObserved && logServiceDispatcherCount == 0) } // drainRemainingEstimate uses the larger workload dimension to avoid obvious double counting. diff --git a/coordinator/controller_drain_test.go b/coordinator/controller_drain_test.go index b9f84c7b97..be451e45c3 100644 --- a/coordinator/controller_drain_test.go +++ b/coordinator/controller_drain_test.go @@ -187,6 +187,28 @@ func TestDrainNodeWaitsForLogServiceDispatchers(t *testing.T) { require.Equal(t, 0, remaining) } +func TestDrainNodeVersion1IgnoresMissingLogServiceDispatcherCount(t *testing.T) { + c, drainController, target := newDrainTestController(t) + setDrainProtocolVersion(c, target, heartbeatpb.DrainProtocolVersion1) + cf := addRunningChangefeed(c, "cf1", node.ID("other"), 100) + + remaining, err := c.DrainNode(context.Background(), target) + require.NoError(t, err) + require.Equal(t, 1, remaining) + + _, epoch, ok := c.getDispatcherDrainTarget() + require.True(t, ok) + setChangefeedDrainStatus(cf, target, epoch, 0, 0) + drainController.ObserveSetNodeLivenessResponse(target, &heartbeatpb.SetNodeLivenessResponse{ + Applied: heartbeatpb.NodeLiveness_STOPPING, + NodeEpoch: 1, + }) + + remaining, err = c.DrainNode(context.Background(), target) + require.NoError(t, err) + require.Equal(t, 0, remaining) +} + func TestDrainNodeDispatcherCountBlocksCompletion(t *testing.T) { c, drainController, target := newDrainTestController(t) setDrainProtocolVersion(c, target, heartbeatpb.CurrentDrainProtocolVersion) diff --git a/downstreamadapter/eventcollector/event_collector.go b/downstreamadapter/eventcollector/event_collector.go index 03543c4fe5..42fac0667d 100644 --- a/downstreamadapter/eventcollector/event_collector.go +++ b/downstreamadapter/eventcollector/event_collector.go @@ -441,13 +441,17 @@ func (c *EventCollector) groupHeartbeat() map[node.ID]*event.DispatcherHeartbeat func (c *EventCollector) eventServiceProtocolVersion(target node.ID) int { if target == c.serverId { - return node.CurrentMessagingProtocolVersion + return event.DispatcherHeartbeatVersion2 } info := c.mc.GetNodeInfo(target) - if info != nil && info.MessagingProtocolVersion >= node.CurrentMessagingProtocolVersion { - return node.CurrentMessagingProtocolVersion - } - return node.LegacyMessagingProtocolVersion + localInfo := c.mc.GetNodeInfo(c.serverId) + // A matching non-empty Git hash proves both endpoints run the same message + // implementation without introducing a separate capability in membership. + // During a rolling upgrade, conservatively use the legacy wire format. + if info != nil && localInfo != nil && localInfo.GitHash != "" && info.GitHash == localInfo.GitHash { + return event.DispatcherHeartbeatVersion2 + } + return event.DispatcherHeartbeatVersion1 } func (c *EventCollector) processDSFeedback(ctx context.Context) error { diff --git a/downstreamadapter/eventcollector/event_collector_test.go b/downstreamadapter/eventcollector/event_collector_test.go index b508b21367..1691d759d1 100644 --- a/downstreamadapter/eventcollector/event_collector_test.go +++ b/downstreamadapter/eventcollector/event_collector_test.go @@ -274,6 +274,7 @@ func TestRemoveLastDispatcher(t *testing.T) { func TestGroupHeartbeatUsesEpochAndClamp(t *testing.T) { serverInfo := node.NewInfo("127.0.0.1:18300", "") mc := messaging.NewMockMessageCenter() + mc.OnNodeChanges(map[node.ID]*node.Info{serverInfo.ID: serverInfo}) appcontext.SetService(appcontext.MessageCenter, mc) c := New(serverInfo.ID) @@ -328,8 +329,8 @@ func TestGroupHeartbeatUsesEpochAndClamp(t *testing.T) { mc.OnNodeChanges(map[node.ID]*node.Info{ remoteID: { - ID: remoteID, - MessagingProtocolVersion: node.CurrentMessagingProtocolVersion, + ID: remoteID, + GitHash: serverInfo.GitHash, }, }) grouped = c.groupHeartbeat() diff --git a/heartbeatpb/drain_protocol.go b/heartbeatpb/drain_protocol.go index cd1b0b0143..74b7c3b8b5 100644 --- a/heartbeatpb/drain_protocol.go +++ b/heartbeatpb/drain_protocol.go @@ -16,11 +16,20 @@ package heartbeatpb const ( // LegacyDrainProtocolVersion means the node only supports legacy hard-restart drain. LegacyDrainProtocolVersion uint32 = 0 + // DrainProtocolVersion1 supports coordinator-driven drain but does not report + // the log service dispatcher count in node heartbeats. + DrainProtocolVersion1 uint32 = 1 // CurrentDrainProtocolVersion is the coordinator-driven drain protocol version. - CurrentDrainProtocolVersion uint32 = 1 + CurrentDrainProtocolVersion uint32 = 2 ) // SupportsCoordinatorDrivenDrain returns whether the node can run coordinator-driven drain. func SupportsCoordinatorDrivenDrain(version uint32) bool { return version != LegacyDrainProtocolVersion } + +// SupportsLogServiceDispatcherCount reports whether STOPPING heartbeats include +// the log service dispatcher count used as a drain-completion gate. +func SupportsLogServiceDispatcherCount(version uint32) bool { + return version >= CurrentDrainProtocolVersion +} diff --git a/maintainer/maintainer_manager_node.go b/maintainer/maintainer_manager_node.go index 8b42600798..f6b3cb8f2e 100644 --- a/maintainer/maintainer_manager_node.go +++ b/maintainer/maintainer_manager_node.go @@ -96,11 +96,12 @@ func (m *Manager) sendNodeHeartbeat(force bool) { // frequent retries and log spam on the 200ms tick. m.node.lastNodeHeartbeatSentAt = now // NodeHeartbeat uses an IO type that legacy coordinators do not recognize. - // Suppress it unless membership explicitly advertises current messaging - // support, so a new capture can safely bootstrap against an old coordinator. + // A matching non-empty Git hash proves the coordinator runs the same message + // implementation. During a rolling upgrade, suppress this new IO type until + // the coordinator has been upgraded to the same build. coordinatorInfo := m.mc.GetNodeInfo(m.coordinatorID) - if coordinatorInfo == nil || - coordinatorInfo.MessagingProtocolVersion < node.CurrentMessagingProtocolVersion { + if coordinatorInfo == nil || m.nodeInfo.GitHash == "" || + coordinatorInfo.GitHash != m.nodeInfo.GitHash { return } diff --git a/maintainer/node_liveness_test.go b/maintainer/node_liveness_test.go index 34097ecfa4..4dcca8e194 100644 --- a/maintainer/node_liveness_test.go +++ b/maintainer/node_liveness_test.go @@ -36,11 +36,12 @@ func (m *mockLogServiceDispatcherCounter) DispatcherCount() int { func registerCurrentCoordinator(mc interface { OnNodeChanges(map[node.ID]*node.Info) -}, coordinatorID node.ID) { +}, manager *Manager) { + manager.nodeInfo.GitHash = "current-build" mc.OnNodeChanges(map[node.ID]*node.Info{ - coordinatorID: { - ID: coordinatorID, - MessagingProtocolVersion: node.CurrentMessagingProtocolVersion, + manager.coordinatorID: { + ID: manager.coordinatorID, + GitHash: manager.nodeInfo.GitHash, }, }) } @@ -53,7 +54,7 @@ func TestSetNodeLivenessRejectEpochMismatch(t *testing.T) { m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness) m.coordinatorID = node.ID("coordinator") m.coordinatorVersion = 1 - registerCurrentCoordinator(mc, m.coordinatorID) + registerCurrentCoordinator(mc, m) req := &heartbeatpb.SetNodeLivenessRequest{ Target: heartbeatpb.NodeLiveness_DRAINING, @@ -80,7 +81,7 @@ func TestSetNodeLivenessApplyTransition(t *testing.T) { m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness) m.coordinatorID = node.ID("coordinator") m.coordinatorVersion = 1 - registerCurrentCoordinator(mc, m.coordinatorID) + registerCurrentCoordinator(mc, m) req := &heartbeatpb.SetNodeLivenessRequest{ Target: heartbeatpb.NodeLiveness_DRAINING, @@ -190,7 +191,7 @@ func TestSetDispatcherDrainTargetSendsNodeHeartbeatAck(t *testing.T) { m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness) m.coordinatorID = node.ID("coordinator") m.coordinatorVersion = 1 - registerCurrentCoordinator(mc, m.coordinatorID) + registerCurrentCoordinator(mc, m) apply := func(target string, epoch uint64) *heartbeatpb.NodeHeartbeat { msg := messaging.NewSingleTargetMessage( @@ -253,10 +254,11 @@ func TestCoordinatorBootstrapSendsNodeHeartbeatToCurrentCoordinator(t *testing.T mc := messaging.NewMockMessageCenter() appcontext.SetService(appcontext.MessageCenter, mc) coordinatorID := node.ID("coordinator") - registerCurrentCoordinator(mc, coordinatorID) var nodeLiveness liveness.Liveness m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness) + m.coordinatorID = coordinatorID + registerCurrentCoordinator(mc, m) req := messaging.NewSingleTargetMessage( m.nodeInfo.ID, messaging.MaintainerManagerTopic, diff --git a/pkg/config/capture.go b/pkg/config/capture.go index a5114936f6..65bd9d433f 100644 --- a/pkg/config/capture.go +++ b/pkg/config/capture.go @@ -45,12 +45,11 @@ type CaptureInfo struct { ID CaptureID `json:"id"` AdvertiseAddr string `json:"address"` - Version string `json:"version"` - GitHash string `json:"git-hash"` - DeployPath string `json:"deploy-path"` - StartTimestamp int64 `json:"start-timestamp"` - IsNewArch bool `json:"is-new-arch"` - MessagingProtocolVersion int `json:"messaging-protocol-version,omitempty"` + Version string `json:"version"` + GitHash string `json:"git-hash"` + DeployPath string `json:"deploy-path"` + StartTimestamp int64 `json:"start-timestamp"` + IsNewArch bool `json:"is-new-arch"` } // Marshal using json.Marshal. diff --git a/pkg/node/node.go b/pkg/node/node.go index a4e8a66289..148c87f926 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -26,11 +26,6 @@ import ( type ID string -const ( - LegacyMessagingProtocolVersion = 1 - CurrentMessagingProtocolVersion = 2 -) - func (s ID) String() string { return string(s) } @@ -57,21 +52,18 @@ type Info struct { DeployPath string `json:"deploy-path"` StartTimestamp int64 `json:"start-timestamp"` - MessagingProtocolVersion int `json:"messaging-protocol-version,omitempty"` - // Epoch represents how many times the node has been restarted. Epoch uint64 `json:"epoch"` } func NewInfo(addr string, deployPath string) *Info { return &Info{ - ID: NewID(), - AdvertiseAddr: addr, - Version: version.ReleaseVersion, - GitHash: version.GitHash, - DeployPath: deployPath, - StartTimestamp: time.Now().Unix(), - MessagingProtocolVersion: CurrentMessagingProtocolVersion, + ID: NewID(), + AdvertiseAddr: addr, + Version: version.ReleaseVersion, + GitHash: version.GitHash, + DeployPath: deployPath, + StartTimestamp: time.Now().Unix(), } } @@ -99,13 +91,12 @@ func (c *Info) Unmarshal(data []byte) error { func CaptureInfoToNodeInfo(captureInfo *config.CaptureInfo) *Info { return &Info{ - ID: ID(captureInfo.ID), - AdvertiseAddr: captureInfo.AdvertiseAddr, - Version: captureInfo.Version, - GitHash: captureInfo.GitHash, - DeployPath: captureInfo.DeployPath, - StartTimestamp: captureInfo.StartTimestamp, - MessagingProtocolVersion: captureInfo.MessagingProtocolVersion, + ID: ID(captureInfo.ID), + AdvertiseAddr: captureInfo.AdvertiseAddr, + Version: captureInfo.Version, + GitHash: captureInfo.GitHash, + DeployPath: captureInfo.DeployPath, + StartTimestamp: captureInfo.StartTimestamp, } } diff --git a/pkg/node/node_test.go b/pkg/node/node_test.go deleted file mode 100644 index f3d1db73bb..0000000000 --- a/pkg/node/node_test.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package node - -import ( - "testing" - - "github.com/pingcap/ticdc/pkg/config" - "github.com/stretchr/testify/require" -) - -func TestMessagingProtocolVersion(t *testing.T) { - current := NewInfo("127.0.0.1:8300", "") - require.Equal(t, CurrentMessagingProtocolVersion, current.MessagingProtocolVersion) - - legacy := CaptureInfoToNodeInfo(&config.CaptureInfo{ - ID: config.CaptureID("legacy-node"), - }) - require.Zero(t, legacy.MessagingProtocolVersion) - - capable := CaptureInfoToNodeInfo(&config.CaptureInfo{ - ID: config.CaptureID("capable-node"), - MessagingProtocolVersion: CurrentMessagingProtocolVersion, - }) - require.Equal(t, CurrentMessagingProtocolVersion, capable.MessagingProtocolVersion) -} diff --git a/server/server.go b/server/server.go index eb61e1d50e..8a604d35cc 100644 --- a/server/server.go +++ b/server/server.go @@ -645,10 +645,9 @@ func (c *server) GetCoordinatorInfo(ctx context.Context) (*node.Info, error) { ID: node.ID(captureInfo.ID), AdvertiseAddr: captureInfo.AdvertiseAddr, - Version: captureInfo.Version, - DeployPath: captureInfo.DeployPath, - StartTimestamp: captureInfo.StartTimestamp, - MessagingProtocolVersion: captureInfo.MessagingProtocolVersion, + Version: captureInfo.Version, + DeployPath: captureInfo.DeployPath, + StartTimestamp: captureInfo.StartTimestamp, // Epoch is now not used in TiCDC, so we just set it to 0. Epoch: 0, diff --git a/server/server_prepare.go b/server/server_prepare.go index 60f46fd15b..60621e4dc7 100644 --- a/server/server_prepare.go +++ b/server/server_prepare.go @@ -201,14 +201,13 @@ func (c *server) setUpDir() { // registerNodeToEtcd the server by put the server's information in etcd func (c *server) registerNodeToEtcd(ctx context.Context) error { cInfo := &config.CaptureInfo{ - ID: config.CaptureID(c.info.ID), - AdvertiseAddr: c.info.AdvertiseAddr, - Version: c.info.Version, - GitHash: c.info.GitHash, - DeployPath: c.info.DeployPath, - StartTimestamp: c.info.StartTimestamp, - IsNewArch: true, - MessagingProtocolVersion: c.info.MessagingProtocolVersion, + ID: config.CaptureID(c.info.ID), + AdvertiseAddr: c.info.AdvertiseAddr, + Version: c.info.Version, + GitHash: c.info.GitHash, + DeployPath: c.info.DeployPath, + StartTimestamp: c.info.StartTimestamp, + IsNewArch: true, } err := c.EtcdClient.PutCaptureInfo(ctx, cInfo, c.session.Lease()) if err != nil { From a484b4ad818c2991f39b05fefaa6dd279f358a4d Mon Sep 17 00:00:00 2001 From: wk989898 Date: Thu, 13 Aug 2026 09:18:44 +0000 Subject: [PATCH 06/18] fmt Signed-off-by: wk989898 --- maintainer/node_liveness_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/maintainer/node_liveness_test.go b/maintainer/node_liveness_test.go index 4dcca8e194..f4abf64b35 100644 --- a/maintainer/node_liveness_test.go +++ b/maintainer/node_liveness_test.go @@ -36,7 +36,8 @@ func (m *mockLogServiceDispatcherCounter) DispatcherCount() int { func registerCurrentCoordinator(mc interface { OnNodeChanges(map[node.ID]*node.Info) -}, manager *Manager) { +}, manager *Manager, +) { manager.nodeInfo.GitHash = "current-build" mc.OnNodeChanges(map[node.ID]*node.Info{ manager.coordinatorID: { From 4ef16741820f49f5aed385b55e7ec486ce0e9644 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Fri, 14 Aug 2026 02:53:03 +0000 Subject: [PATCH 07/18] update Signed-off-by: wk989898 --- coordinator/controller.go | 2 - coordinator/controller_drain.go | 29 ++++---------- coordinator/controller_drain_test.go | 29 +++++--------- .../eventcollector/event_collector.go | 39 +++++------------- .../eventcollector/event_collector_test.go | 23 ++--------- heartbeatpb/drain_protocol.go | 11 +---- maintainer/maintainer_manager_node.go | 9 ----- maintainer/node_liveness_test.go | 40 ------------------- pkg/common/event/dispatcher_heartbeat.go | 13 +----- pkg/common/event/dispatcher_heartbeat_test.go | 14 +++++-- pkg/messaging/message_center.go | 19 ++------- pkg/messaging/mock_message_center.go | 18 +-------- 12 files changed, 47 insertions(+), 199 deletions(-) diff --git a/coordinator/controller.go b/coordinator/controller.go index be254f7654..c49a2b355d 100644 --- a/coordinator/controller.go +++ b/coordinator/controller.go @@ -1164,8 +1164,6 @@ func (c *Controller) RemoveNode(id node.ID) { observation.stoppingObserved, observation.remaining, observation.logServiceDispatcherCount, - observation.logServiceDispatcherCountObserved, - observation.logServiceDispatcherCountRequired, ) } diff --git a/coordinator/controller_drain.go b/coordinator/controller_drain.go index 8cfbd01bfa..7785d43b9a 100644 --- a/coordinator/controller_drain.go +++ b/coordinator/controller_drain.go @@ -206,8 +206,6 @@ func (c *Controller) DrainNode(ctx context.Context, target node.ID) (int, error) observation.stoppingObserved, observation.remaining, observation.logServiceDispatcherCount, - observation.logServiceDispatcherCountObserved, - observation.logServiceDispatcherCountRequired, ) if completionObserved { @@ -229,8 +227,6 @@ func (c *Controller) DrainNode(ctx context.Context, target node.ID) (int, error) zap.Int("targetInflightDrainMoveCount", observation.targetInflightDrainMoveCount), zap.Int("pendingStatusCount", observation.pendingStatusCount), zap.Uint32("logServiceDispatcherCount", observation.logServiceDispatcherCount), - zap.Bool("logServiceDispatcherCountObserved", observation.logServiceDispatcherCountObserved), - zap.Bool("logServiceDispatcherCountRequired", observation.logServiceDispatcherCountRequired), zap.Int("remaining", observation.remaining)) return ensureDrainRemainingNonZero(observation.remaining), nil } @@ -247,8 +243,6 @@ func (c *Controller) observeRemovedActiveDrainTarget(target node.ID, epoch uint6 observation.stoppingObserved, observation.remaining, observation.logServiceDispatcherCount, - observation.logServiceDispatcherCountObserved, - observation.logServiceDispatcherCountRequired, ) if completionObserved { log.Info("drain completion observed for removed active target", @@ -269,7 +263,6 @@ func (c *Controller) observeRemovedActiveDrainTarget(target node.ID, epoch uint6 zap.Int("targetInflightDrainMoveCount", observation.targetInflightDrainMoveCount), zap.Int("pendingStatusCount", observation.pendingStatusCount), zap.Uint32("logServiceDispatcherCount", observation.logServiceDispatcherCount), - zap.Bool("logServiceDispatcherCountObserved", observation.logServiceDispatcherCountObserved), zap.Int("remaining", observation.remaining)) return ensureDrainRemainingNonZero(observation.remaining) } @@ -321,13 +314,11 @@ type drainNodeObservation struct { // pendingStatusCount is the number of running changefeeds not converged to the active target epoch. pendingStatusCount int // remaining is the max of all workload dimensions used by drain completion gating. - remaining int - nodeState drain.State - drainingObserved bool - stoppingObserved bool - logServiceDispatcherCount uint32 - logServiceDispatcherCountObserved bool - logServiceDispatcherCountRequired bool + remaining int + nodeState drain.State + drainingObserved bool + stoppingObserved bool + logServiceDispatcherCount uint32 } func (c *Controller) observeDrainNode(target node.ID, epoch uint64) drainNodeObservation { @@ -346,10 +337,7 @@ func (c *Controller) observeDrainNode(target node.ID, epoch uint64) drainNodeObs ) _, observation.drainingObserved, observation.stoppingObserved = c.drainController.GetStatus(target) - observation.logServiceDispatcherCount, observation.logServiceDispatcherCountObserved = c.drainController.GetLogServiceDispatcherCount(target) - drainProtocolVersion, drainProtocolObserved := c.drainController.GetDrainProtocolVersion(target) - observation.logServiceDispatcherCountRequired = !drainProtocolObserved || - heartbeatpb.SupportsLogServiceDispatcherCount(drainProtocolVersion) + observation.logServiceDispatcherCount, _ = c.drainController.GetLogServiceDispatcherCount(target) observation.nodeState = c.drainController.GetState(target) return observation } @@ -1101,16 +1089,13 @@ func isBestEffortDrainComplete( stoppingObserved bool, remaining int, logServiceDispatcherCount uint32, - logServiceDispatcherCountObserved bool, - logServiceDispatcherCountRequired bool, ) bool { if nodeState == drain.StateUnknown || !drainingObserved { return false } return stoppingObserved && remaining == 0 && - (!logServiceDispatcherCountRequired || - logServiceDispatcherCountObserved && logServiceDispatcherCount == 0) + logServiceDispatcherCount == 0 } // drainRemainingEstimate uses the larger workload dimension to avoid obvious double counting. diff --git a/coordinator/controller_drain_test.go b/coordinator/controller_drain_test.go index be451e45c3..0bf25aebb0 100644 --- a/coordinator/controller_drain_test.go +++ b/coordinator/controller_drain_test.go @@ -154,7 +154,7 @@ func TestDrainNodeCompletesAfterCompletionObserved(t *testing.T) { require.Equal(t, epoch, c.drainSession.epoch) } -func TestDrainNodeWaitsForLogServiceDispatchers(t *testing.T) { +func TestDrainNodeUsesDefaultLogServiceDispatcherCount(t *testing.T) { c, drainController, target := newDrainTestController(t) setDrainProtocolVersion(c, target, heartbeatpb.CurrentDrainProtocolVersion) cf := addRunningChangefeed(c, "cf1", node.ID("other"), 100) @@ -167,29 +167,20 @@ func TestDrainNodeWaitsForLogServiceDispatchers(t *testing.T) { require.True(t, ok) setChangefeedDrainStatus(cf, target, epoch, 0, 0) - // A STOPPING response alone does not contain the log service dispatcher count. + // Old nodes do not report the log service dispatcher count. Its protobuf + // default is zero, so the missing field must not block drain completion. drainController.ObserveSetNodeLivenessResponse(target, &heartbeatpb.SetNodeLivenessResponse{ Applied: heartbeatpb.NodeLiveness_STOPPING, NodeEpoch: 1, }) remaining, err = c.DrainNode(context.Background(), target) require.NoError(t, err) - require.Equal(t, 1, remaining) - - setTargetStoppingHeartbeat(drainController, target, 2) - remaining, err = c.DrainNode(context.Background(), target) - require.NoError(t, err) - require.Equal(t, 1, remaining) - - setTargetStoppingHeartbeat(drainController, target, 0) - remaining, err = c.DrainNode(context.Background(), target) - require.NoError(t, err) require.Equal(t, 0, remaining) } -func TestDrainNodeVersion1IgnoresMissingLogServiceDispatcherCount(t *testing.T) { +func TestDrainNodeWaitsForReportedLogServiceDispatchers(t *testing.T) { c, drainController, target := newDrainTestController(t) - setDrainProtocolVersion(c, target, heartbeatpb.DrainProtocolVersion1) + setDrainProtocolVersion(c, target, heartbeatpb.CurrentDrainProtocolVersion) cf := addRunningChangefeed(c, "cf1", node.ID("other"), 100) remaining, err := c.DrainNode(context.Background(), target) @@ -199,11 +190,13 @@ func TestDrainNodeVersion1IgnoresMissingLogServiceDispatcherCount(t *testing.T) _, epoch, ok := c.getDispatcherDrainTarget() require.True(t, ok) setChangefeedDrainStatus(cf, target, epoch, 0, 0) - drainController.ObserveSetNodeLivenessResponse(target, &heartbeatpb.SetNodeLivenessResponse{ - Applied: heartbeatpb.NodeLiveness_STOPPING, - NodeEpoch: 1, - }) + setTargetStoppingHeartbeat(drainController, target, 2) + remaining, err = c.DrainNode(context.Background(), target) + require.NoError(t, err) + require.Equal(t, 1, remaining) + + setTargetStoppingHeartbeat(drainController, target, 0) remaining, err = c.DrainNode(context.Background(), target) require.NoError(t, err) require.Equal(t, 0, remaining) diff --git a/downstreamadapter/eventcollector/event_collector.go b/downstreamadapter/eventcollector/event_collector.go index 42fac0667d..6456f1ada2 100644 --- a/downstreamadapter/eventcollector/event_collector.go +++ b/downstreamadapter/eventcollector/event_collector.go @@ -415,7 +415,7 @@ func (c *EventCollector) groupHeartbeat() map[node.ID]*event.DispatcherHeartbeat group := func(target node.ID, dispatcherID common.DispatcherID, checkpointTs uint64, epoch uint64) { heartbeat, ok := groupedHeartbeats[target] if !ok { - heartbeat = event.NewDispatcherHeartbeatWithVersion(c.eventServiceProtocolVersion(target)) + heartbeat = event.NewDispatcherHeartbeat() groupedHeartbeats[target] = heartbeat } heartbeat.AddDispatcherProgress(dispatcherID, checkpointTs, epoch) @@ -439,21 +439,6 @@ func (c *EventCollector) groupHeartbeat() map[node.ID]*event.DispatcherHeartbeat return groupedHeartbeats } -func (c *EventCollector) eventServiceProtocolVersion(target node.ID) int { - if target == c.serverId { - return event.DispatcherHeartbeatVersion2 - } - info := c.mc.GetNodeInfo(target) - localInfo := c.mc.GetNodeInfo(c.serverId) - // A matching non-empty Git hash proves both endpoints run the same message - // implementation without introducing a separate capability in membership. - // During a rolling upgrade, conservatively use the legacy wire format. - if info != nil && localInfo != nil && localInfo.GitHash != "" && info.GitHash == localInfo.GitHash { - return event.DispatcherHeartbeatVersion2 - } - return event.DispatcherHeartbeatVersion1 -} - func (c *EventCollector) processDSFeedback(ctx context.Context) error { log.Info("Start process feedback from dynamic stream") defer log.Info("Stop process feedback from dynamic stream") @@ -742,8 +727,7 @@ func (c *EventCollector) newCongestionControlMessages() map[node.ID]*event.Conge // build congestion control messages for each node result := make(map[node.ID]*event.CongestionControl) for nodeID, changefeedDispatchers := range nodeDispatcherMemory { - protocolVersion := c.eventServiceProtocolVersion(nodeID) - congestionControl := event.NewCongestionControlWithVersion(protocolVersion) + congestionControl := event.NewCongestionControlWithVersion(event.CongestionControlVersion2) for changefeedID, dispatcherMemory := range changefeedDispatchers { if len(dispatcherMemory) == 0 { @@ -755,18 +739,13 @@ func (c *EventCollector) newCongestionControlMessages() map[node.ID]*event.Conge if !ok { continue } - if protocolVersion >= event.CongestionControlVersion2 { - congestionControl.AddAvailableMemoryWithDispatchersAndUsageAndReleaseCount( - changefeedID.ID(), - totalAvailable, - changefeedUsageRatio[changefeedID], - dispatcherMemory, - getAndResetMemoryReleaseCount(changefeedID), - ) - } else { - congestionControl.AddAvailableMemoryWithDispatchers( - changefeedID.ID(), totalAvailable, dispatcherMemory) - } + congestionControl.AddAvailableMemoryWithDispatchersAndUsageAndReleaseCount( + changefeedID.ID(), + totalAvailable, + changefeedUsageRatio[changefeedID], + dispatcherMemory, + getAndResetMemoryReleaseCount(changefeedID), + ) } if len(congestionControl.GetAvailables()) > 0 { diff --git a/downstreamadapter/eventcollector/event_collector_test.go b/downstreamadapter/eventcollector/event_collector_test.go index 1691d759d1..0fa5c75273 100644 --- a/downstreamadapter/eventcollector/event_collector_test.go +++ b/downstreamadapter/eventcollector/event_collector_test.go @@ -274,7 +274,6 @@ func TestRemoveLastDispatcher(t *testing.T) { func TestGroupHeartbeatUsesEpochAndClamp(t *testing.T) { serverInfo := node.NewInfo("127.0.0.1:18300", "") mc := messaging.NewMockMessageCenter() - mc.OnNodeChanges(map[node.ID]*node.Info{serverInfo.ID: serverInfo}) appcontext.SetService(appcontext.MessageCenter, mc) c := New(serverInfo.ID) @@ -318,28 +317,12 @@ func TestGroupHeartbeatUsesEpochAndClamp(t *testing.T) { remoteHeartbeat := grouped[remoteID] require.NotNil(t, remoteHeartbeat) - require.Equal(t, commonEvent.DispatcherHeartbeatVersion1, remoteHeartbeat.Version) - require.Len(t, remoteHeartbeat.DispatcherProgressesLegacy, 1) - require.Equal(t, remoteDispatcher.id, remoteHeartbeat.DispatcherProgressesLegacy[0].DispatcherID) - require.Equal(t, uint64(210), remoteHeartbeat.DispatcherProgressesLegacy[0].CheckpointTs) - - controlMessages := c.newCongestionControlMessages() - require.Equal(t, commonEvent.CongestionControlVersion2, controlMessages[serverInfo.ID].GetVersion()) - require.Equal(t, commonEvent.CongestionControlVersion1, controlMessages[remoteID].GetVersion()) - - mc.OnNodeChanges(map[node.ID]*node.Info{ - remoteID: { - ID: remoteID, - GitHash: serverInfo.GitHash, - }, - }) - grouped = c.groupHeartbeat() - remoteHeartbeat = grouped[remoteID] require.Equal(t, commonEvent.DispatcherHeartbeatVersion2, remoteHeartbeat.Version) require.Len(t, remoteHeartbeat.DispatcherProgresses, 1) + require.Equal(t, uint8(commonEvent.DispatcherProgressVersion1), remoteHeartbeat.DispatcherProgresses[0].Version) + require.Equal(t, remoteDispatcher.id, remoteHeartbeat.DispatcherProgresses[0].DispatcherID) + require.Equal(t, uint64(210), remoteHeartbeat.DispatcherProgresses[0].CheckpointTs) require.Equal(t, uint64(5), remoteHeartbeat.DispatcherProgresses[0].Epoch) - controlMessages = c.newCongestionControlMessages() - require.Equal(t, commonEvent.CongestionControlVersion2, controlMessages[remoteID].GetVersion()) } func TestGroupHeartbeatResetThenHandshake(t *testing.T) { diff --git a/heartbeatpb/drain_protocol.go b/heartbeatpb/drain_protocol.go index 74b7c3b8b5..cd1b0b0143 100644 --- a/heartbeatpb/drain_protocol.go +++ b/heartbeatpb/drain_protocol.go @@ -16,20 +16,11 @@ package heartbeatpb const ( // LegacyDrainProtocolVersion means the node only supports legacy hard-restart drain. LegacyDrainProtocolVersion uint32 = 0 - // DrainProtocolVersion1 supports coordinator-driven drain but does not report - // the log service dispatcher count in node heartbeats. - DrainProtocolVersion1 uint32 = 1 // CurrentDrainProtocolVersion is the coordinator-driven drain protocol version. - CurrentDrainProtocolVersion uint32 = 2 + CurrentDrainProtocolVersion uint32 = 1 ) // SupportsCoordinatorDrivenDrain returns whether the node can run coordinator-driven drain. func SupportsCoordinatorDrivenDrain(version uint32) bool { return version != LegacyDrainProtocolVersion } - -// SupportsLogServiceDispatcherCount reports whether STOPPING heartbeats include -// the log service dispatcher count used as a drain-completion gate. -func SupportsLogServiceDispatcherCount(version uint32) bool { - return version >= CurrentDrainProtocolVersion -} diff --git a/maintainer/maintainer_manager_node.go b/maintainer/maintainer_manager_node.go index f6b3cb8f2e..3e368a474c 100644 --- a/maintainer/maintainer_manager_node.go +++ b/maintainer/maintainer_manager_node.go @@ -95,15 +95,6 @@ func (m *Manager) sendNodeHeartbeat(force bool) { // Update before sending so a transient send failure will not cause // frequent retries and log spam on the 200ms tick. m.node.lastNodeHeartbeatSentAt = now - // NodeHeartbeat uses an IO type that legacy coordinators do not recognize. - // A matching non-empty Git hash proves the coordinator runs the same message - // implementation. During a rolling upgrade, suppress this new IO type until - // the coordinator has been upgraded to the same build. - coordinatorInfo := m.mc.GetNodeInfo(m.coordinatorID) - if coordinatorInfo == nil || m.nodeInfo.GitHash == "" || - coordinatorInfo.GitHash != m.nodeInfo.GitHash { - return - } currentLiveness := liveness.CaptureAlive if m.node.liveness != nil { diff --git a/maintainer/node_liveness_test.go b/maintainer/node_liveness_test.go index f4abf64b35..da9f86ea64 100644 --- a/maintainer/node_liveness_test.go +++ b/maintainer/node_liveness_test.go @@ -34,19 +34,6 @@ func (m *mockLogServiceDispatcherCounter) DispatcherCount() int { return m.count } -func registerCurrentCoordinator(mc interface { - OnNodeChanges(map[node.ID]*node.Info) -}, manager *Manager, -) { - manager.nodeInfo.GitHash = "current-build" - mc.OnNodeChanges(map[node.ID]*node.Info{ - manager.coordinatorID: { - ID: manager.coordinatorID, - GitHash: manager.nodeInfo.GitHash, - }, - }) -} - func TestSetNodeLivenessRejectEpochMismatch(t *testing.T) { mc := messaging.NewMockMessageCenter() appcontext.SetService(appcontext.MessageCenter, mc) @@ -55,7 +42,6 @@ func TestSetNodeLivenessRejectEpochMismatch(t *testing.T) { m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness) m.coordinatorID = node.ID("coordinator") m.coordinatorVersion = 1 - registerCurrentCoordinator(mc, m) req := &heartbeatpb.SetNodeLivenessRequest{ Target: heartbeatpb.NodeLiveness_DRAINING, @@ -82,7 +68,6 @@ func TestSetNodeLivenessApplyTransition(t *testing.T) { m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness) m.coordinatorID = node.ID("coordinator") m.coordinatorVersion = 1 - registerCurrentCoordinator(mc, m) req := &heartbeatpb.SetNodeLivenessRequest{ Target: heartbeatpb.NodeLiveness_DRAINING, @@ -192,7 +177,6 @@ func TestSetDispatcherDrainTargetSendsNodeHeartbeatAck(t *testing.T) { m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness) m.coordinatorID = node.ID("coordinator") m.coordinatorVersion = 1 - registerCurrentCoordinator(mc, m) apply := func(target string, epoch uint64) *heartbeatpb.NodeHeartbeat { msg := messaging.NewSingleTargetMessage( @@ -248,30 +232,6 @@ func TestCoordinatorBootstrapResponseIncludesDispatcherDrainTarget(t *testing.T) resp := out.Message[0].(*heartbeatpb.CoordinatorBootstrapResponse) require.Equal(t, "n2", resp.DispatcherDrainTargetNodeId) require.Equal(t, uint64(7), resp.DispatcherDrainTargetEpoch) - require.Empty(t, mc.GetMessageChannel()) -} - -func TestCoordinatorBootstrapSendsNodeHeartbeatToCurrentCoordinator(t *testing.T) { - mc := messaging.NewMockMessageCenter() - appcontext.SetService(appcontext.MessageCenter, mc) - coordinatorID := node.ID("coordinator") - - var nodeLiveness liveness.Liveness - m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness) - m.coordinatorID = coordinatorID - registerCurrentCoordinator(mc, m) - req := messaging.NewSingleTargetMessage( - m.nodeInfo.ID, - messaging.MaintainerManagerTopic, - &heartbeatpb.CoordinatorBootstrapRequest{Version: 1}, - ) - req.From = coordinatorID - m.onCoordinatorBootstrapRequest(req) - - first := <-mc.GetMessageChannel() - second := <-mc.GetMessageChannel() - require.Equal(t, messaging.TypeCoordinatorBootstrapResponse, first.Type) - require.Equal(t, messaging.TypeNodeHeartbeatRequest, second.Type) } func TestAddMaintainerIgnoreInvalidConfig(t *testing.T) { diff --git a/pkg/common/event/dispatcher_heartbeat.go b/pkg/common/event/dispatcher_heartbeat.go index 0a9dd2ff52..793149bc85 100644 --- a/pkg/common/event/dispatcher_heartbeat.go +++ b/pkg/common/event/dispatcher_heartbeat.go @@ -107,12 +107,8 @@ type DispatcherHeartbeat struct { } func NewDispatcherHeartbeat() *DispatcherHeartbeat { - return NewDispatcherHeartbeatWithVersion(DispatcherHeartbeatVersion2) -} - -func NewDispatcherHeartbeatWithVersion(version int) *DispatcherHeartbeat { return &DispatcherHeartbeat{ - Version: version, + Version: DispatcherHeartbeatVersion2, // TODO: Pass a real clusterID when we support 1 TiCDC cluster subscribe multiple TiDB clusters ClusterID: 0, DispatcherProgressesLegacy: make([]DispatcherProgressLegacy, 0), @@ -122,13 +118,6 @@ func NewDispatcherHeartbeatWithVersion(version int) *DispatcherHeartbeat { func (d *DispatcherHeartbeat) AddDispatcherProgress(dispatcherID common.DispatcherID, checkpointTs uint64, epoch uint64) { d.DispatcherCount++ - if d.Version == DispatcherHeartbeatVersion1 { - d.DispatcherProgressesLegacy = append(d.DispatcherProgressesLegacy, DispatcherProgressLegacy{ - DispatcherID: dispatcherID, - CheckpointTs: checkpointTs, - }) - return - } d.DispatcherProgresses = append(d.DispatcherProgresses, DispatcherProgress{ Version: DispatcherProgressVersion1, DispatcherID: dispatcherID, diff --git a/pkg/common/event/dispatcher_heartbeat_test.go b/pkg/common/event/dispatcher_heartbeat_test.go index cf4811a0da..649b707367 100644 --- a/pkg/common/event/dispatcher_heartbeat_test.go +++ b/pkg/common/event/dispatcher_heartbeat_test.go @@ -151,9 +151,17 @@ func TestDispatcherHeartbeatBackwardCompatibleV1(t *testing.T) { t.Parallel() dispatcherID := common.NewDispatcherID() - heartbeat := NewDispatcherHeartbeatWithVersion(DispatcherHeartbeatVersion1) - heartbeat.ClusterID = 123 - heartbeat.AddDispatcherProgress(dispatcherID, 456, 789) + heartbeat := &DispatcherHeartbeat{ + Version: DispatcherHeartbeatVersion1, + ClusterID: 123, + DispatcherProgressesLegacy: []DispatcherProgressLegacy{ + { + DispatcherID: dispatcherID, + CheckpointTs: 456, + }, + }, + DispatcherCount: 1, + } data, err := heartbeat.Marshal() require.NoError(t, err) diff --git a/pkg/messaging/message_center.go b/pkg/messaging/message_center.go index 0361ea945f..0a11c31dab 100644 --- a/pkg/messaging/message_center.go +++ b/pkg/messaging/message_center.go @@ -43,7 +43,6 @@ const ( type MessageCenter interface { MessageSender MessageReceiver - GetNodeInfo(node.ID) *node.Info // OnNodeChanges is called when the nodes in the cluster are changed. The message center should update the target list. OnNodeChanges(map[node.ID]*node.Info) Close() @@ -88,11 +87,8 @@ type messageCenter struct { m map[node.ID]*remoteMessageTarget } - nodeInfos struct { - sync.RWMutex - m map[node.ID]*node.Info - } - notifyCh chan map[node.ID]*node.Info + remoteNodeInfos map[node.ID]*node.Info + notifyCh chan map[node.ID]*node.Info grpcServer *grpc.Server router *router @@ -126,7 +122,6 @@ func NewMessageCenter( notifyCh: make(chan map[node.ID]*node.Info, 128), } mc.remoteTargets.m = make(map[node.ID]*remoteMessageTarget) - mc.nodeInfos.m = make(map[node.ID]*node.Info) log.Info("create message center success.", zap.Stringer("id", id), zap.String("addr", cfg.Addr)) @@ -200,12 +195,6 @@ func (mc *messageCenter) DeRegisterHandler(topic string) { mc.router.deRegisterHandler(topic) } -func (mc *messageCenter) GetNodeInfo(id node.ID) *node.Info { - mc.nodeInfos.RLock() - defer mc.nodeInfos.RUnlock() - return mc.nodeInfos.m[id] -} - func (mc *messageCenter) OnNodeChanges(activeNode map[node.ID]*node.Info) { mc.notifyCh <- activeNode log.Info("notify node changes", zap.Any("activeNode", activeNode)) @@ -216,9 +205,7 @@ func (mc *messageCenter) handleNodeChanges(activeNode map[node.ID]*node.Info) { for id, node := range activeNode { infosMap[id] = node } - mc.nodeInfos.Lock() - mc.nodeInfos.m = infosMap - mc.nodeInfos.Unlock() + mc.remoteNodeInfos = infosMap currentTargets := make(map[node.ID]bool) currentTargets[mc.id] = true diff --git a/pkg/messaging/mock_message_center.go b/pkg/messaging/mock_message_center.go index 25436c66c8..6425224260 100644 --- a/pkg/messaging/mock_message_center.go +++ b/pkg/messaging/mock_message_center.go @@ -13,18 +13,13 @@ package messaging -import ( - "sync" - - "github.com/pingcap/ticdc/pkg/node" -) +import "github.com/pingcap/ticdc/pkg/node" var _ MessageCenter = &mockMessageCenter{} // mockMessageCenter is a mock implementation of the MessageCenter interface type mockMessageCenter struct { messageCh chan *TargetMessage - nodeInfos sync.Map } func NewMockMessageCenter() *mockMessageCenter { @@ -38,17 +33,6 @@ func (m *mockMessageCenter) GetMessageChannel() chan *TargetMessage { } func (m *mockMessageCenter) OnNodeChanges(nodeInfos map[node.ID]*node.Info) { - for id, info := range nodeInfos { - m.nodeInfos.Store(id, info) - } -} - -func (m *mockMessageCenter) GetNodeInfo(id node.ID) *node.Info { - info, ok := m.nodeInfos.Load(id) - if !ok { - return nil - } - return info.(*node.Info) } func (m *mockMessageCenter) SendEvent(event *TargetMessage) error { From 98add57eee84ce3c75fdeb2c5ec423e3e8c37b82 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Fri, 14 Aug 2026 03:05:07 +0000 Subject: [PATCH 08/18] update Signed-off-by: wk989898 --- coordinator/drain/controller.go | 6 ++---- maintainer/maintainer_manager_node.go | 7 ++----- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/coordinator/drain/controller.go b/coordinator/drain/controller.go index 4301bbfa3d..156791bb8c 100644 --- a/coordinator/drain/controller.go +++ b/coordinator/drain/controller.go @@ -67,8 +67,7 @@ type nodeState struct { // logServiceDispatcherCount is valid only after a STOPPING heartbeat has // been observed for the current node epoch. - logServiceDispatcherCount uint32 - logServiceDispatcherCountObserved bool + logServiceDispatcherCount uint32 } type drainTargetSchedulerGate struct { @@ -193,7 +192,6 @@ func (c *Controller) ObserveHeartbeat(nodeID node.ID, hb *heartbeatpb.NodeHeartb st := c.ensureNodeStateLocked(nodeID) if hb.NodeEpoch == st.nodeEpoch && hb.Liveness == heartbeatpb.NodeLiveness_STOPPING { st.logServiceDispatcherCount = hb.GetLogServiceDispatcherCount() - st.logServiceDispatcherCountObserved = true } c.observeTargetSchedulerAckLocked(nodeID, hb) } @@ -432,7 +430,7 @@ func (c *Controller) GetLogServiceDispatcherCount(nodeID node.ID) (uint32, bool) defer c.mu.Unlock() st, ok := c.nodes[nodeID] - if !ok || !st.logServiceDispatcherCountObserved { + if !ok { return 0, false } return st.logServiceDispatcherCount, true diff --git a/maintainer/maintainer_manager_node.go b/maintainer/maintainer_manager_node.go index 3e368a474c..0cd4361fa5 100644 --- a/maintainer/maintainer_manager_node.go +++ b/maintainer/maintainer_manager_node.go @@ -19,6 +19,7 @@ import ( "github.com/pingcap/log" "github.com/pingcap/ticdc/heartbeatpb" + "github.com/pingcap/ticdc/logservice/eventstore" appcontext "github.com/pingcap/ticdc/pkg/common/context" "github.com/pingcap/ticdc/pkg/liveness" "github.com/pingcap/ticdc/pkg/messaging" @@ -30,10 +31,6 @@ import ( // Forced heartbeats bypass this throttle to acknowledge state changes immediately. const nodeHeartbeatInterval = 5 * time.Second -type logServiceDispatcherCounter interface { - DispatcherCount() int -} - // managerNodeState owns node-scoped state shared by all local maintainers. type managerNodeState struct { // liveness points to the server-wide node liveness state shared with other @@ -102,7 +99,7 @@ func (m *Manager) sendNodeHeartbeat(force bool) { } drainTarget, drainEpoch := m.getDispatcherDrainTarget() logServiceDispatcherCount := 0 - if store, ok := appcontext.TryGetService[logServiceDispatcherCounter](appcontext.EventStore); ok { + if store, ok := appcontext.TryGetService[eventstore.EventStore](appcontext.EventStore); ok { logServiceDispatcherCount = store.DispatcherCount() } hb := &heartbeatpb.NodeHeartbeat{ From 520fdf34cc44c054de9ef91a1585970dcdf66822 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Fri, 14 Aug 2026 03:14:09 +0000 Subject: [PATCH 09/18] update Signed-off-by: wk989898 --- coordinator/controller.go | 1 - coordinator/controller_drain.go | 18 ++++++++---------- coordinator/controller_drain_test.go | 2 +- coordinator/drain/controller.go | 13 +++++++------ coordinator/drain/controller_test.go | 13 ++++--------- 5 files changed, 20 insertions(+), 27 deletions(-) diff --git a/coordinator/controller.go b/coordinator/controller.go index c49a2b355d..39114c7a7d 100644 --- a/coordinator/controller.go +++ b/coordinator/controller.go @@ -1163,7 +1163,6 @@ func (c *Controller) RemoveNode(id node.ID) { observation.drainingObserved, observation.stoppingObserved, observation.remaining, - observation.logServiceDispatcherCount, ) } diff --git a/coordinator/controller_drain.go b/coordinator/controller_drain.go index 7785d43b9a..1de4facd83 100644 --- a/coordinator/controller_drain.go +++ b/coordinator/controller_drain.go @@ -205,7 +205,6 @@ func (c *Controller) DrainNode(ctx context.Context, target node.ID) (int, error) observation.drainingObserved, observation.stoppingObserved, observation.remaining, - observation.logServiceDispatcherCount, ) if completionObserved { @@ -226,7 +225,7 @@ func (c *Controller) DrainNode(ctx context.Context, target node.ID) (int, error) zap.Int("dispatcherCountOnTarget", observation.dispatcherCountOnTarget), zap.Int("targetInflightDrainMoveCount", observation.targetInflightDrainMoveCount), zap.Int("pendingStatusCount", observation.pendingStatusCount), - zap.Uint32("logServiceDispatcherCount", observation.logServiceDispatcherCount), + zap.Int("logServiceDispatcherCount", observation.logServiceDispatcherCount), zap.Int("remaining", observation.remaining)) return ensureDrainRemainingNonZero(observation.remaining), nil } @@ -242,7 +241,6 @@ func (c *Controller) observeRemovedActiveDrainTarget(target node.ID, epoch uint6 observation.drainingObserved, observation.stoppingObserved, observation.remaining, - observation.logServiceDispatcherCount, ) if completionObserved { log.Info("drain completion observed for removed active target", @@ -262,7 +260,7 @@ func (c *Controller) observeRemovedActiveDrainTarget(target node.ID, epoch uint6 zap.Int("dispatcherCountOnTarget", observation.dispatcherCountOnTarget), zap.Int("targetInflightDrainMoveCount", observation.targetInflightDrainMoveCount), zap.Int("pendingStatusCount", observation.pendingStatusCount), - zap.Uint32("logServiceDispatcherCount", observation.logServiceDispatcherCount), + zap.Int("logServiceDispatcherCount", observation.logServiceDispatcherCount), zap.Int("remaining", observation.remaining)) return ensureDrainRemainingNonZero(observation.remaining) } @@ -318,7 +316,7 @@ type drainNodeObservation struct { nodeState drain.State drainingObserved bool stoppingObserved bool - logServiceDispatcherCount uint32 + logServiceDispatcherCount int } func (c *Controller) observeDrainNode(target node.ID, epoch uint64) drainNodeObservation { @@ -328,16 +326,17 @@ func (c *Controller) observeDrainNode(target node.ID, epoch uint64) drainNodeObs } observation.dispatcherCountOnTarget, observation.targetInflightDrainMoveCount = c.aggregateDrainTargetProgress(target, epoch) observation.pendingStatusCount = c.collectDrainPendingStatus(target, epoch) + observation.logServiceDispatcherCount = c.drainController.GetLogServiceDispatcherCount(target) observation.remaining = drainRemainingEstimate( observation.maintainersOnTarget, observation.inflightOpsInvolvingTarget, observation.dispatcherCountOnTarget, observation.targetInflightDrainMoveCount, observation.pendingStatusCount, + observation.logServiceDispatcherCount, ) _, observation.drainingObserved, observation.stoppingObserved = c.drainController.GetStatus(target) - observation.logServiceDispatcherCount, _ = c.drainController.GetLogServiceDispatcherCount(target) observation.nodeState = c.drainController.GetState(target) return observation } @@ -1088,14 +1087,11 @@ func isBestEffortDrainComplete( drainingObserved bool, stoppingObserved bool, remaining int, - logServiceDispatcherCount uint32, ) bool { if nodeState == drain.StateUnknown || !drainingObserved { return false } - return stoppingObserved && - remaining == 0 && - logServiceDispatcherCount == 0 + return stoppingObserved && remaining == 0 } // drainRemainingEstimate uses the larger workload dimension to avoid obvious double counting. @@ -1105,6 +1101,7 @@ func drainRemainingEstimate( dispatcherCountOnTarget int, targetInflightDrainMoveCount int, pendingStatusCount int, + logServiceDispatcherCount int, ) int { return max( maintainersOnTarget, @@ -1112,6 +1109,7 @@ func drainRemainingEstimate( dispatcherCountOnTarget, targetInflightDrainMoveCount, pendingStatusCount, + logServiceDispatcherCount, ) } diff --git a/coordinator/controller_drain_test.go b/coordinator/controller_drain_test.go index 0bf25aebb0..a7bc008a8e 100644 --- a/coordinator/controller_drain_test.go +++ b/coordinator/controller_drain_test.go @@ -194,7 +194,7 @@ func TestDrainNodeWaitsForReportedLogServiceDispatchers(t *testing.T) { setTargetStoppingHeartbeat(drainController, target, 2) remaining, err = c.DrainNode(context.Background(), target) require.NoError(t, err) - require.Equal(t, 1, remaining) + require.Equal(t, 2, remaining) setTargetStoppingHeartbeat(drainController, target, 0) remaining, err = c.DrainNode(context.Background(), target) diff --git a/coordinator/drain/controller.go b/coordinator/drain/controller.go index 156791bb8c..997aa24ce5 100644 --- a/coordinator/drain/controller.go +++ b/coordinator/drain/controller.go @@ -67,7 +67,7 @@ type nodeState struct { // logServiceDispatcherCount is valid only after a STOPPING heartbeat has // been observed for the current node epoch. - logServiceDispatcherCount uint32 + logServiceDispatcherCount int } type drainTargetSchedulerGate struct { @@ -191,7 +191,7 @@ func (c *Controller) ObserveHeartbeat(nodeID node.ID, hb *heartbeatpb.NodeHeartb c.observeLivenessLocked(nodeID, hb.NodeEpoch, hb.Liveness) st := c.ensureNodeStateLocked(nodeID) if hb.NodeEpoch == st.nodeEpoch && hb.Liveness == heartbeatpb.NodeLiveness_STOPPING { - st.logServiceDispatcherCount = hb.GetLogServiceDispatcherCount() + st.logServiceDispatcherCount = int(hb.GetLogServiceDispatcherCount()) } c.observeTargetSchedulerAckLocked(nodeID, hb) } @@ -424,16 +424,17 @@ func (c *Controller) GetStatus(nodeID node.ID) (drainRequested, drainingObserved } // GetLogServiceDispatcherCount returns the dispatcher count reported by a -// STOPPING heartbeat for the current node epoch. -func (c *Controller) GetLogServiceDispatcherCount(nodeID node.ID) (uint32, bool) { +// STOPPING heartbeat for the current node epoch. Nodes that do not report the +// field use the protobuf default value zero. +func (c *Controller) GetLogServiceDispatcherCount(nodeID node.ID) int { c.mu.Lock() defer c.mu.Unlock() st, ok := c.nodes[nodeID] if !ok { - return 0, false + return 0 } - return st.logServiceDispatcherCount, true + return st.logServiceDispatcherCount } // GetDrainProtocolVersion returns the bootstrap-observed drain capability for a node. diff --git a/coordinator/drain/controller_test.go b/coordinator/drain/controller_test.go index b6c72f13a0..47c5b9bb58 100644 --- a/coordinator/drain/controller_test.go +++ b/coordinator/drain/controller_test.go @@ -134,24 +134,20 @@ func TestDrainControllerTracksStoppingLogServiceDispatcherCountByEpoch(t *testin Applied: heartbeatpb.NodeLiveness_STOPPING, NodeEpoch: 42, }) - _, observed := c.GetLogServiceDispatcherCount(target) - require.False(t, observed) + require.Zero(t, c.GetLogServiceDispatcherCount(target)) c.ObserveHeartbeat(target, &heartbeatpb.NodeHeartbeat{ Liveness: heartbeatpb.NodeLiveness_STOPPING, NodeEpoch: 42, LogServiceDispatcherCount: 2, }) - count, observed := c.GetLogServiceDispatcherCount(target) - require.True(t, observed) - require.Equal(t, uint32(2), count) + require.Equal(t, 2, c.GetLogServiceDispatcherCount(target)) c.ObserveHeartbeat(target, &heartbeatpb.NodeHeartbeat{ Liveness: heartbeatpb.NodeLiveness_ALIVE, NodeEpoch: 43, }) - _, observed = c.GetLogServiceDispatcherCount(target) - require.False(t, observed) + require.Zero(t, c.GetLogServiceDispatcherCount(target)) // A delayed heartbeat from the old process must not satisfy the new epoch. c.ObserveHeartbeat(target, &heartbeatpb.NodeHeartbeat{ @@ -159,8 +155,7 @@ func TestDrainControllerTracksStoppingLogServiceDispatcherCountByEpoch(t *testin NodeEpoch: 42, LogServiceDispatcherCount: 0, }) - _, observed = c.GetLogServiceDispatcherCount(target) - require.False(t, observed) + require.Zero(t, c.GetLogServiceDispatcherCount(target)) } func TestDrainControllerSkipStoppingForNewEpochWithoutDraining(t *testing.T) { From ce8aa797c7bc7843ffe44d69c5d6cf7bceafbd6e Mon Sep 17 00:00:00 2001 From: wk989898 Date: Fri, 14 Aug 2026 03:39:51 +0000 Subject: [PATCH 10/18] fix ut Signed-off-by: wk989898 --- maintainer/node_liveness_test.go | 17 +++++++++++++---- pkg/common/context/app_context.go | 1 + 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/maintainer/node_liveness_test.go b/maintainer/node_liveness_test.go index da9f86ea64..13178ec631 100644 --- a/maintainer/node_liveness_test.go +++ b/maintainer/node_liveness_test.go @@ -17,6 +17,7 @@ import ( "testing" "github.com/pingcap/ticdc/heartbeatpb" + "github.com/pingcap/ticdc/logservice/eventstore" "github.com/pingcap/ticdc/pkg/common" appcontext "github.com/pingcap/ticdc/pkg/common/context" "github.com/pingcap/ticdc/pkg/config" @@ -26,11 +27,14 @@ import ( "github.com/stretchr/testify/require" ) -type mockLogServiceDispatcherCounter struct { +type mockEventStore struct { + eventstore.EventStore count int } -func (m *mockLogServiceDispatcherCounter) DispatcherCount() int { +var _ eventstore.EventStore = (*mockEventStore)(nil) + +func (m *mockEventStore) DispatcherCount() int { return m.count } @@ -167,10 +171,15 @@ func TestSetDispatcherDrainTargetRejectStaleUpdate(t *testing.T) { func TestSetDispatcherDrainTargetSendsNodeHeartbeatAck(t *testing.T) { mc := messaging.NewMockMessageCenter() appcontext.SetService(appcontext.MessageCenter, mc) - logService := &mockLogServiceDispatcherCounter{count: 2} + previousEventStore, hadPreviousEventStore := appcontext.TryGetService[any](appcontext.EventStore) + logService := &mockEventStore{count: 2} appcontext.SetService(appcontext.EventStore, logService) t.Cleanup(func() { - logService.count = 0 + if hadPreviousEventStore { + appcontext.SetService(appcontext.EventStore, previousEventStore) + } else { + appcontext.DeleteService(appcontext.EventStore) + } }) var nodeLiveness liveness.Liveness diff --git a/pkg/common/context/app_context.go b/pkg/common/context/app_context.go index 84bca4d842..c9f5d4816c 100644 --- a/pkg/common/context/app_context.go +++ b/pkg/common/context/app_context.go @@ -60,6 +60,7 @@ func SetID(id string) { GetGlobalContext().id = id } func GetID() string { return GetGlobalContext().id } func SetService[T any](name string, t T) { GetGlobalContext().serviceMap.Store(name, t) } +func DeleteService(name string) { GetGlobalContext().serviceMap.Delete(name) } func GetService[T any](name string) T { v, _ := GetGlobalContext().serviceMap.Load(name) return v.(T) From c07b0cdac233f4b04b34d8598a0042f4771c55fc Mon Sep 17 00:00:00 2001 From: wk989898 Date: Fri, 14 Aug 2026 05:44:37 +0000 Subject: [PATCH 11/18] fmt Signed-off-by: wk989898 --- logservice/schemastore/schema_store_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/logservice/schemastore/schema_store_test.go b/logservice/schemastore/schema_store_test.go index aacd1d3b31..1d7cabfca5 100644 --- a/logservice/schemastore/schema_store_test.go +++ b/logservice/schemastore/schema_store_test.go @@ -157,12 +157,10 @@ func TestSchemaStoreCloseWaitsForWorkersBeforeClosingPebble(t *testing.T) { workerStarted := make(chan struct{}) releaseWorker := make(chan struct{}) store := &keyspaceSchemaStore{dataStorage: pstorage} - store.wg.Add(1) - go func() { - defer store.wg.Done() + store.wg.Go(func() { close(workerStarted) <-releaseWorker - }() + }) <-workerStarted schemaStore := &schemaStore{ From f2d945b0ca0d375b3f4b7138ccc4e956515c7f95 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Mon, 17 Aug 2026 05:23:59 +0000 Subject: [PATCH 12/18] add resolved ts for ready event Signed-off-by: wk989898 --- .../eventcollector/dispatcher_session.go | 47 +++++++++++++------ .../eventcollector/dispatcher_stat_test.go | 33 +++++++++++++ pkg/common/event/ready_event.go | 33 ++++++++++--- pkg/common/event/ready_event_test.go | 41 ++++++++++++++-- pkg/eventservice/event_broker.go | 2 +- pkg/eventservice/event_broker_test.go | 3 ++ 6 files changed, 132 insertions(+), 27 deletions(-) diff --git a/downstreamadapter/eventcollector/dispatcher_session.go b/downstreamadapter/eventcollector/dispatcher_session.go index ec1a600cfa..d0ae2a0ca4 100644 --- a/downstreamadapter/eventcollector/dispatcher_session.go +++ b/downstreamadapter/eventcollector/dispatcher_session.go @@ -48,8 +48,9 @@ import ( // pendingRemoteEventServiceID="" // // This means local registration and remote probing can overlap. A remote service -// may serve data first, but a later local ready still moves the dispatcher back -// to local and cleans up remote registrations. +// may serve data first, but after the local service catches up to the dispatcher +// checkpoint, a later local ready moves the dispatcher back to local and cleans +// up remote registrations. type dispatcherConnState struct { sync.RWMutex // removed marks the session as terminal after removal starts. New @@ -60,7 +61,8 @@ type dispatcherConnState struct { currentEventServiceID node.ID // localReadyPending means the local register request has been sent but local // ready has not been accepted. It may be true while a remote service is - // already current; local ready wins when it arrives later. + // already current; local ready wins after its resolved ts covers the current + // dispatcher checkpoint. localReadyPending bool // pendingRemoteEventServiceID is the remote EventService currently being // probed for reuse. It waits for either ready or not reusable, and only one @@ -124,11 +126,17 @@ type readyDecision struct { // acceptReady enforces the ready acceptance rules: // 1. once local is already serving, any later remote ready is stale and should // only trigger cleanup; -// 2. local ready can be accepted while local registration is still pending, and -// local wins over any remote that started serving earlier; +// 2. local ready can be accepted while local registration is still pending. If +// a remote is already serving, local only wins after its resolved ts covers +// the dispatcher's current checkpoint; // 3. remote ready is accepted only from the single remote candidate currently // being probed. -func (d *dispatcherConnState) acceptReady(from node.ID, localServerID node.ID) readyDecision { +func (d *dispatcherConnState) acceptReady( + from node.ID, + localServerID node.ID, + readyResolvedTs uint64, + requiredCheckpointTs uint64, +) readyDecision { d.Lock() defer d.Unlock() if d.removed { @@ -139,6 +147,11 @@ func (d *dispatcherConnState) acceptReady(from node.ID, localServerID node.ID) r if !d.localReadyPending { return readyDecision{} } + if !d.currentEventServiceID.IsEmpty() && + d.currentEventServiceID != localServerID && + readyResolvedTs < requiredCheckpointTs { + return readyDecision{} + } decision := readyDecision{ commitTarget: localServerID, @@ -170,8 +183,8 @@ func (d *dispatcherConnState) acceptReady(from node.ID, localServerID node.ID) r } d.currentEventServiceID = from - // Keep localReadyPending unchanged: local ready may still arrive later and - // move the dispatcher back to local. + // Keep localReadyPending unchanged: after the local service catches up, a + // later local ready may move the dispatcher back to local. d.pendingRemoteEventServiceID = "" d.remoteCandidates = nil return readyDecision{ @@ -440,7 +453,7 @@ func (s *dispatcherSession) handleSignalEvent(event dispatcher.DispatcherEvent) from := *event.From switch event.GetType() { case commonEvent.TypeReadyEvent: - s.handleReadyEvent(from) + s.handleReadyEvent(from, uint64(event.GetCommitTs())) case commonEvent.TypeNotReusableEvent: if from == s.localServerID { log.Panic("should not happen: local event service should not send not reusable event") @@ -459,12 +472,18 @@ func (s *dispatcherSession) handleSignalEvent(event dispatcher.DispatcherEvent) // handleReadyEvent applies the ready decision produced by connState: clean up // any stale registrations, then commit whichever target won the ready race. -func (s *dispatcherSession) handleReadyEvent(from node.ID) { +func (s *dispatcherSession) handleReadyEvent(from node.ID, readyResolvedTs uint64) { s.requestMu.Lock() defer s.requestMu.Unlock() // connState decides whether this ready should be accepted and which stale // registrations must be cleaned up. Session only applies the side effects. - accepted := s.connState.acceptReady(from, s.localServerID) + requiredCheckpointTs := s.target.GetCheckpointTs() + accepted := s.connState.acceptReady( + from, + s.localServerID, + readyResolvedTs, + requiredCheckpointTs, + ) for _, target := range accepted.cleanupTargets { s.removeFromLocked(target) } @@ -472,13 +491,13 @@ func (s *dispatcherSession) handleReadyEvent(from node.ID) { return } if accepted.commitTarget == s.localServerID { - s.handleAcceptedLocalReadyLocked() + s.handleAcceptedLocalReadyLocked(requiredCheckpointTs) return } s.handleAcceptedRemoteReadyLocked(accepted.commitTarget) } -func (s *dispatcherSession) handleAcceptedLocalReadyLocked() { +func (s *dispatcherSession) handleAcceptedLocalReadyLocked(resetTs uint64) { if s.readyCallback != nil { // This path is used during the initial add flow before the dispatcher is // committed. Local is still authoritative, so any speculative remote @@ -491,7 +510,7 @@ func (s *dispatcherSession) handleAcceptedLocalReadyLocked() { log.Info("received ready signal from local event service, prepare to reset the dispatcher", zap.Stringer("changefeedID", s.target.GetChangefeedID()), zap.Stringer("dispatcher", s.target.GetId())) - s.doResetLocked(s.localServerID, s.target.GetCheckpointTs()) + s.doResetLocked(s.localServerID, resetTs) } func (s *dispatcherSession) handleAcceptedRemoteReadyLocked(serverID node.ID) { diff --git a/downstreamadapter/eventcollector/dispatcher_stat_test.go b/downstreamadapter/eventcollector/dispatcher_stat_test.go index 76ec14c4f1..389c2adf47 100644 --- a/downstreamadapter/eventcollector/dispatcher_stat_test.go +++ b/downstreamadapter/eventcollector/dispatcher_stat_test.go @@ -847,6 +847,39 @@ func TestHandleLocalReadyEventCleansUpRemoteRegistrations(t *testing.T) { requireNoDispatcherRequest(t, mockEventCollector) }) + t.Run("local ready waits until its resolved ts covers the current checkpoint", func(t *testing.T) { + mockDisp := newMockDispatcher(dispatcherID, 100) + mockDisp.checkPointTs = 200 + mockEventCollector := newTestEventCollector(localServerID) + stat := newDispatcherStat(mockDisp, mockEventCollector, nil) + setSessionState(stat.session, remoteServerID, true, "") + newLocalReadyEvent := func(resolvedTs uint64) dispatcher.DispatcherEvent { + ready := commonEvent.NewReadyEventWithResolvedTs(dispatcherID, resolvedTs) + return dispatcher.DispatcherEvent{ + From: &localServerID, + Event: &ready, + } + } + + stat.handleSignalEvent(newLocalReadyEvent(199)) + + currentEventServiceID, localReadyPending, pendingRemoteTarget := sessionState(stat.session) + require.Equal(t, remoteServerID, currentEventServiceID) + require.True(t, localReadyPending) + require.Empty(t, pendingRemoteTarget) + requireNoDispatcherRequest(t, mockEventCollector) + + stat.handleSignalEvent(newLocalReadyEvent(200)) + + requireDispatcherRequests( + t, + readDispatcherRequests(t, mockEventCollector, 2), + dispatcherRequestRecord{to: remoteServerID, action: eventpb.ActionType_ACTION_TYPE_REMOVE}, + dispatcherRequestRecord{to: localServerID, action: eventpb.ActionType_ACTION_TYPE_RESET}, + ) + requireNoDispatcherRequest(t, mockEventCollector) + }) + t.Run("local ready with callback still removes speculative remote register", func(t *testing.T) { mockDisp := newMockDispatcher(dispatcherID, 0) mockEventCollector := newTestEventCollector(localServerID) diff --git a/pkg/common/event/ready_event.go b/pkg/common/event/ready_event.go index 5908c953d3..efa019a9f5 100644 --- a/pkg/common/event/ready_event.go +++ b/pkg/common/event/ready_event.go @@ -14,6 +14,7 @@ package event import ( + "encoding/binary" "fmt" "github.com/pingcap/ticdc/pkg/common" @@ -28,17 +29,26 @@ var _ Event = &ReadyEvent{} type ReadyEvent struct { Version int DispatcherID common.DispatcherID + // ResolvedTs is the highest timestamp the EventService can currently serve. + // It is appended to the version 1 payload so old decoders can ignore it, while + // new decoders still accept legacy payloads that only contain DispatcherID. + ResolvedTs uint64 } func NewReadyEvent(dispatcherID common.DispatcherID) ReadyEvent { + return NewReadyEventWithResolvedTs(dispatcherID, 0) +} + +func NewReadyEventWithResolvedTs(dispatcherID common.DispatcherID, resolvedTs uint64) ReadyEvent { return ReadyEvent{ Version: ReadyEventVersion1, DispatcherID: dispatcherID, + ResolvedTs: resolvedTs, } } func (e *ReadyEvent) String() string { - return fmt.Sprintf("ReadyEvent{Version: %d, DispatcherID: %s}", e.Version, e.DispatcherID) + return fmt.Sprintf("ReadyEvent{Version: %d, DispatcherID: %s, ResolvedTs: %d}", e.Version, e.DispatcherID, e.ResolvedTs) } // GetType returns the event type @@ -64,8 +74,7 @@ func (e *ReadyEvent) GetDispatcherID() common.DispatcherID { // GetCommitTs returns the commit timestamp func (e *ReadyEvent) GetCommitTs() common.Ts { - // not used - return 0 + return common.Ts(e.ResolvedTs) } // GetStartTs returns the start timestamp @@ -77,8 +86,8 @@ func (e *ReadyEvent) GetStartTs() common.Ts { // GetSize returns the approximate size of the event in bytes func (e *ReadyEvent) GetSize() int64 { // Size does not include header or version (those are only for serialization) - // Only business data: dispatcherID - return int64(e.DispatcherID.GetSize()) + // Only business data: dispatcherID + resolvedTs + return int64(e.DispatcherID.GetSize() + 8) } func (e *ReadyEvent) IsPaused() bool { @@ -129,13 +138,16 @@ func (e *ReadyEvent) Unmarshal(data []byte) error { func (e ReadyEvent) encodeV1() ([]byte, error) { // Note: version is now handled in the header by Marshal(), not here - // payload: dispatcherID - payloadSize := e.DispatcherID.GetSize() + // payload: dispatcherID + optional resolvedTs + payloadSize := e.DispatcherID.GetSize() + 8 data := make([]byte, payloadSize) offset := 0 // DispatcherID copy(data[offset:], e.DispatcherID.Marshal()) + offset += e.DispatcherID.GetSize() + + binary.BigEndian.PutUint64(data[offset:], e.ResolvedTs) return data, nil } @@ -149,6 +161,13 @@ func (e *ReadyEvent) decodeV1(data []byte) error { if err != nil { return err } + offset += e.DispatcherID.GetSize() + + // ResolvedTs was appended to the version 1 payload. Treat its absence as + // zero so messages produced by older EventServices remain decodable. + if len(data) >= offset+8 { + e.ResolvedTs = binary.BigEndian.Uint64(data[offset:]) + } return nil } diff --git a/pkg/common/event/ready_event_test.go b/pkg/common/event/ready_event_test.go index e8fde74e46..0dd871e6b8 100644 --- a/pkg/common/event/ready_event_test.go +++ b/pkg/common/event/ready_event_test.go @@ -23,7 +23,7 @@ import ( func TestReadyEvent(t *testing.T) { did := common.NewDispatcherID() - e := NewReadyEvent(did) + e := NewReadyEventWithResolvedTs(did, 123) data, err := e.Marshal() require.NoError(t, err) require.Len(t, data, int(e.GetSize())+int(GetEventHeaderSize())) @@ -33,11 +33,12 @@ func TestReadyEvent(t *testing.T) { require.NoError(t, err) require.Equal(t, e.Version, e2.Version) require.Equal(t, e.DispatcherID, e2.DispatcherID) + require.Equal(t, e.ResolvedTs, e2.ResolvedTs) } func TestReadyEventMethods(t *testing.T) { did := common.NewDispatcherID() - e := NewReadyEvent(did) + e := NewReadyEventWithResolvedTs(did, 123) // Test GetType require.Equal(t, TypeReadyEvent, e.GetType()) @@ -52,7 +53,7 @@ func TestReadyEventMethods(t *testing.T) { require.Equal(t, did, e.GetDispatcherID()) // Test GetCommitTs - require.Equal(t, common.Ts(0), e.GetCommitTs()) + require.Equal(t, common.Ts(123), e.GetCommitTs()) // Test GetStartTs require.Equal(t, common.Ts(0), e.GetStartTs()) @@ -65,7 +66,7 @@ func TestReadyEventMethods(t *testing.T) { } func TestReadyEventMarshalUnmarshal(t *testing.T) { - normalEvent := NewReadyEvent(common.NewDispatcherID()) + normalEvent := NewReadyEventWithResolvedTs(common.NewDispatcherID(), 123) testCases := []struct { name string event *ReadyEvent @@ -108,10 +109,40 @@ func TestReadyEventMarshalUnmarshal(t *testing.T) { require.NoError(t, err) require.Equal(t, tc.event.Version, e2.Version) require.Equal(t, tc.event.DispatcherID, e2.DispatcherID) + require.Equal(t, tc.event.ResolvedTs, e2.ResolvedTs) }) } } +func TestReadyEventDecodeLegacyPayload(t *testing.T) { + did := common.NewDispatcherID() + data, err := MarshalEventWithHeader(TypeReadyEvent, ReadyEventVersion1, did.Marshal()) + require.NoError(t, err) + + var event ReadyEvent + require.NoError(t, event.Unmarshal(data)) + require.Equal(t, did, event.DispatcherID) + require.Zero(t, event.ResolvedTs) +} + +func TestReadyEventLegacyDecoderIgnoresResolvedTs(t *testing.T) { + did := common.NewDispatcherID() + event := NewReadyEventWithResolvedTs(did, 123) + data, err := event.Marshal() + require.NoError(t, err) + + payload, version, err := ValidateAndExtractPayload(data, TypeReadyEvent) + require.NoError(t, err) + require.Equal(t, ReadyEventVersion1, version) + + // ReadyEvent's legacy version 1 decoder only unmarshals DispatcherID. Keep + // this assertion to guarantee that appending ResolvedTs remains compatible + // with older nodes during a rolling upgrade. + var legacyDispatcherID common.DispatcherID + require.NoError(t, legacyDispatcherID.Unmarshal(payload)) + require.Equal(t, did, legacyDispatcherID) +} + // TestReadyEventHeader verifies the unified header format func TestReadyEventHeader(t *testing.T) { did := common.NewDispatcherID() @@ -201,7 +232,7 @@ func TestReadyEventSize(t *testing.T) { e := NewReadyEvent(did) // GetSize should only return business data size, not including header - expectedSize := int64(did.GetSize()) + expectedSize := int64(did.GetSize() + 8) require.Equal(t, expectedSize, e.GetSize()) // Marshaled data should include header diff --git a/pkg/eventservice/event_broker.go b/pkg/eventservice/event_broker.go index 542556b88f..e2e077b5cb 100644 --- a/pkg/eventservice/event_broker.go +++ b/pkg/eventservice/event_broker.go @@ -595,7 +595,7 @@ func (c *eventBroker) checkAndSendReady(task scanTask) bool { return false } remoteID := node.ID(task.info.GetServerID()) - event := event.NewReadyEvent(task.info.GetID()) + event := event.NewReadyEventWithResolvedTs(task.info.GetID(), task.receivedResolvedTs.Load()) wrapEvent := newWrapReadyEvent(remoteID, event) c.getMessageCh(task.messageWorkerIndex, common.IsRedoMode(task.info.GetMode())) <- wrapEvent log.Debug("send ready event to dispatcher", diff --git a/pkg/eventservice/event_broker_test.go b/pkg/eventservice/event_broker_test.go index 99498796a7..b7c94b56c0 100644 --- a/pkg/eventservice/event_broker_test.go +++ b/pkg/eventservice/event_broker_test.go @@ -92,6 +92,9 @@ func TestScanRequestCoalescing(t *testing.T) { require.False(t, disp.isScanBusy()) e := <-broker.messageCh[0] require.Equal(t, event.TypeReadyEvent, e.msgType) + ready, ok := e.e.(*event.ReadyEvent) + require.True(t, ok) + require.Equal(t, uint64(102), ready.ResolvedTs) } type scanLifecycleTrackingContext struct { From d3c09ea5cf9f9eefde826bf44768b9f0b4aaf69d Mon Sep 17 00:00:00 2001 From: wk989898 Date: Tue, 18 Aug 2026 07:14:28 +0000 Subject: [PATCH 13/18] update Signed-off-by: wk989898 --- .../eventcollector/dispatcher_session.go | 40 ++++++++++--- .../dispatcher_session_test_helper_test.go | 12 ++++ .../eventcollector/dispatcher_stat_test.go | 57 +++++++++++++++++++ 3 files changed, 102 insertions(+), 7 deletions(-) diff --git a/downstreamadapter/eventcollector/dispatcher_session.go b/downstreamadapter/eventcollector/dispatcher_session.go index d0ae2a0ca4..7cc8fb9798 100644 --- a/downstreamadapter/eventcollector/dispatcher_session.go +++ b/downstreamadapter/eventcollector/dispatcher_session.go @@ -48,9 +48,9 @@ import ( // pendingRemoteEventServiceID="" // // This means local registration and remote probing can overlap. A remote service -// may serve data first, but after the local service catches up to the dispatcher -// checkpoint, a later local ready moves the dispatcher back to local and cleans -// up remote registrations. +// may serve data first, but after the local service catches up to the progress +// the remote reported (and the dispatcher checkpoint), a later local ready moves +// the dispatcher back to local and cleans up remote registrations. type dispatcherConnState struct { sync.RWMutex // removed marks the session as terminal after removal starts. New @@ -71,6 +71,14 @@ type dispatcherConnState struct { // remoteCandidates are the remaining remote EventServices to probe after the // current pending remote reports not reusable. remoteCandidates []string + // remoteResolvedTs is the resolved ts reported by the remote EventService + // currently serving. It is only meaningful while currentEventServiceID is a + // remote service and is cleared when the local service wins back. The local + // ready must cover it before taking over: the sink checkpoint alone is not a + // safe baseline before the remote delivers its first events because it still + // equals the dispatcher start ts, which would let a barely started local + // subscription win the race and stall the table while catching up from TiKV. + remoteResolvedTs uint64 } // Registration state transitions. @@ -128,7 +136,8 @@ type readyDecision struct { // only trigger cleanup; // 2. local ready can be accepted while local registration is still pending. If // a remote is already serving, local only wins after its resolved ts covers -// the dispatcher's current checkpoint; +// both the current dispatcher checkpoint and the progress the remote +// reported; // 3. remote ready is accepted only from the single remote candidate currently // being probed. func (d *dispatcherConnState) acceptReady( @@ -148,9 +157,20 @@ func (d *dispatcherConnState) acceptReady( return readyDecision{} } if !d.currentEventServiceID.IsEmpty() && - d.currentEventServiceID != localServerID && - readyResolvedTs < requiredCheckpointTs { - return readyDecision{} + d.currentEventServiceID != localServerID { + // The sink checkpoint alone is not a safe baseline before the remote + // delivers its first events: it still equals the dispatcher start ts, + // so a local subscription that has barely started could win the race + // and stall the table while catching up from TiKV. The remote + // service's reported resolved ts captures where the remote actually + // is, so the local service must cover that first. + baseline := requiredCheckpointTs + if d.remoteResolvedTs > baseline { + baseline = d.remoteResolvedTs + } + if readyResolvedTs < baseline { + return readyDecision{} + } } decision := readyDecision{ @@ -164,6 +184,7 @@ func (d *dispatcherConnState) acceptReady( d.localReadyPending = false d.pendingRemoteEventServiceID = "" d.remoteCandidates = nil + d.remoteResolvedTs = 0 return decision } @@ -187,6 +208,10 @@ func (d *dispatcherConnState) acceptReady( // later local ready may move the dispatcher back to local. d.pendingRemoteEventServiceID = "" d.remoteCandidates = nil + // Record the remote progress the local service must catch up to before it + // may win back. This is more meaningful than the sink checkpoint, which + // still equals the start ts before the remote delivers its first events. + d.remoteResolvedTs = readyResolvedTs return readyDecision{ commitTarget: from, } @@ -207,6 +232,7 @@ func (d *dispatcherConnState) beginRemove(localServerID node.ID) ([]node.ID, boo d.localReadyPending = false d.pendingRemoteEventServiceID = "" d.remoteCandidates = nil + d.remoteResolvedTs = 0 return []node.ID(targets), false } diff --git a/downstreamadapter/eventcollector/dispatcher_session_test_helper_test.go b/downstreamadapter/eventcollector/dispatcher_session_test_helper_test.go index 53bc40c314..d5148e805b 100644 --- a/downstreamadapter/eventcollector/dispatcher_session_test_helper_test.go +++ b/downstreamadapter/eventcollector/dispatcher_session_test_helper_test.go @@ -34,6 +34,12 @@ func setSessionRemoteCandidates(session *dispatcherSession, nodes []string) { session.connState.remoteCandidates = nodes } +func setSessionRemoteResolvedTs(session *dispatcherSession, resolvedTs uint64) { + session.connState.Lock() + defer session.connState.Unlock() + session.connState.remoteResolvedTs = resolvedTs +} + func setSessionReadyCallback(session *dispatcherSession, readyCallback func()) { session.readyCallback = readyCallback } @@ -57,3 +63,9 @@ func sessionState(session *dispatcherSession) (node.ID, bool, node.ID) { defer session.connState.RUnlock() return session.connState.currentEventServiceID, session.connState.localReadyPending, session.connState.pendingRemoteEventServiceID } + +func sessionRemoteResolvedTs(session *dispatcherSession) uint64 { + session.connState.RLock() + defer session.connState.RUnlock() + return session.connState.remoteResolvedTs +} diff --git a/downstreamadapter/eventcollector/dispatcher_stat_test.go b/downstreamadapter/eventcollector/dispatcher_stat_test.go index 389c2adf47..67eede2dc3 100644 --- a/downstreamadapter/eventcollector/dispatcher_stat_test.go +++ b/downstreamadapter/eventcollector/dispatcher_stat_test.go @@ -898,6 +898,63 @@ func TestHandleLocalReadyEventCleansUpRemoteRegistrations(t *testing.T) { }) } +func TestLocalReadyGatedByRemoteServedResolvedTs(t *testing.T) { + localServerID := node.ID("local-server") + remoteServerID := node.ID("remote-server") + dispatcherID := common.NewDispatcherID() + + // The sink checkpoint stays at the start ts because the remote has not + // delivered its first events yet. The local ready must still cover the + // resolved ts the remote reported when it was accepted. + mockDisp := newMockDispatcher(dispatcherID, 100) + mockDisp.checkPointTs = 100 + mockEventCollector := newTestEventCollector(localServerID) + stat := newDispatcherStat(mockDisp, mockEventCollector, nil) + setSessionState(stat.session, "", true, remoteServerID) + + // Accept a remote ready that reports the remote is already serving at 300. + remoteReady := commonEvent.NewReadyEventWithResolvedTs(dispatcherID, 300) + stat.handleSignalEvent(dispatcher.DispatcherEvent{ + From: &remoteServerID, + Event: &remoteReady, + }) + requireDispatcherRequests( + t, + readDispatcherRequests(t, mockEventCollector, 1), + dispatcherRequestRecord{to: remoteServerID, action: eventpb.ActionType_ACTION_TYPE_RESET}, + ) + require.Equal(t, uint64(300), sessionRemoteResolvedTs(stat.session)) + + newLocalReadyEvent := func(resolvedTs uint64) dispatcher.DispatcherEvent { + ready := commonEvent.NewReadyEventWithResolvedTs(dispatcherID, resolvedTs) + return dispatcher.DispatcherEvent{ + From: &localServerID, + Event: &ready, + } + } + + // Local ready above the sink checkpoint but below the remote progress is + // still rejected, otherwise the fresh local subscription would stall the + // table while catching up from TiKV. + stat.handleSignalEvent(newLocalReadyEvent(200)) + currentEventServiceID, localReadyPending, pendingRemoteTarget := sessionState(stat.session) + require.Equal(t, remoteServerID, currentEventServiceID) + require.True(t, localReadyPending) + require.Empty(t, pendingRemoteTarget) + requireNoDispatcherRequest(t, mockEventCollector) + + // Once local catches up to the remote progress it wins back and clears it. + stat.handleSignalEvent(newLocalReadyEvent(300)) + requireDispatcherRequests( + t, + readDispatcherRequests(t, mockEventCollector, 2), + dispatcherRequestRecord{to: remoteServerID, action: eventpb.ActionType_ACTION_TYPE_REMOVE}, + dispatcherRequestRecord{to: localServerID, action: eventpb.ActionType_ACTION_TYPE_RESET}, + ) + require.Zero(t, sessionRemoteResolvedTs(stat.session)) + requireNoDispatcherRequest(t, mockEventCollector) +} + func TestInitialLocalReadyCallbackIsOneShot(t *testing.T) { localServerID := node.ID("local-server") dispatcherID := common.NewDispatcherID() From cf1a3e73cd429f00a8511bc10e138f05a3a027b2 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Tue, 18 Aug 2026 11:15:15 +0000 Subject: [PATCH 14/18] make remote read first Signed-off-by: wk989898 --- .../eventcollector/dispatcher_session.go | 116 +++++++++-- .../dispatcher_session_test_helper_test.go | 12 +- .../eventcollector/dispatcher_stat.go | 8 + .../eventcollector/dispatcher_stat_test.go | 185 ++++++++++++++++-- .../eventcollector/event_collector.go | 12 ++ 5 files changed, 301 insertions(+), 32 deletions(-) diff --git a/downstreamadapter/eventcollector/dispatcher_session.go b/downstreamadapter/eventcollector/dispatcher_session.go index 7cc8fb9798..ccdac4a715 100644 --- a/downstreamadapter/eventcollector/dispatcher_session.go +++ b/downstreamadapter/eventcollector/dispatcher_session.go @@ -16,6 +16,7 @@ package eventcollector import ( "slices" "sync" + "time" "github.com/pingcap/log" "github.com/pingcap/ticdc/downstreamadapter/dispatcher" @@ -27,6 +28,13 @@ import ( "go.uber.org/zap" ) +// remoteProbeTimeout bounds how long a single remote reuse probe may stay in +// flight without a ready or not-reusable response before the dispatcher gives +// up on that candidate and falls back to the local event service. Without it, a +// silent remote would leave the dispatcher permanently waiting because the +// local ready is held while the probe is in flight. +const remoteProbeTimeout = 10 * time.Second + // dispatcherConnState owns the EventService registration state for one dispatcher. // It does not send messages. Its job is to apply atomic state transitions and // return the side-effect inputs that dispatcherSession should execute. @@ -50,7 +58,11 @@ import ( // This means local registration and remote probing can overlap. A remote service // may serve data first, but after the local service catches up to the progress // the remote reported (and the dispatcher checkpoint), a later local ready moves -// the dispatcher back to local and cleans up remote registrations. +// the dispatcher back to local and cleans up remote registrations. While a +// remote reuse probe is still in flight the local ready is held, so the remote +// (which already serves the span) wins over a local subscription that has not +// served data yet; the probe expires after remoteProbeTimeout to unblock the +// local fallback. type dispatcherConnState struct { sync.RWMutex // removed marks the session as terminal after removal starts. New @@ -79,6 +91,10 @@ type dispatcherConnState struct { // equals the dispatcher start ts, which would let a barely started local // subscription win the race and stall the table while catching up from TiKV. remoteResolvedTs uint64 + // remoteProbeStartAt records when the current remote reuse probe was sent, + // so a silent remote (no ready and no not-reusable response) can be + // abandoned after remoteProbeTimeout instead of blocking the dispatcher. + remoteProbeStartAt time.Time } // Registration state transitions. @@ -101,6 +117,7 @@ func (d *dispatcherConnState) beginRegisterToRemote(serverID node.ID) bool { return false } d.pendingRemoteEventServiceID = serverID + d.remoteProbeStartAt = time.Now() return true } @@ -134,10 +151,12 @@ type readyDecision struct { // acceptReady enforces the ready acceptance rules: // 1. once local is already serving, any later remote ready is stale and should // only trigger cleanup; -// 2. local ready can be accepted while local registration is still pending. If -// a remote is already serving, local only wins after its resolved ts covers -// both the current dispatcher checkpoint and the progress the remote -// reported; +// 2. local ready can be accepted while local registration is still pending, +// but only after any in-flight remote reuse probe has concluded (the remote +// already serves the span and should win over a local subscription that has +// not served data yet) and, when a remote is already serving, after the +// local resolved ts covers both the current dispatcher checkpoint and the +// progress the remote reported; // 3. remote ready is accepted only from the single remote candidate currently // being probed. func (d *dispatcherConnState) acceptReady( @@ -145,6 +164,8 @@ func (d *dispatcherConnState) acceptReady( localServerID node.ID, readyResolvedTs uint64, requiredCheckpointTs uint64, + remoteDeliveredTs uint64, + dispatcherStartTs uint64, ) readyDecision { d.Lock() defer d.Unlock() @@ -158,19 +179,31 @@ func (d *dispatcherConnState) acceptReady( } if !d.currentEventServiceID.IsEmpty() && d.currentEventServiceID != localServerID { - // The sink checkpoint alone is not a safe baseline before the remote - // delivers its first events: it still equals the dispatcher start ts, - // so a local subscription that has barely started could win the race - // and stall the table while catching up from TiKV. The remote - // service's reported resolved ts captures where the remote actually - // is, so the local service must cover that first. + // The local event service may only win back once its resolved ts + // covers the position the current remote has actually delivered to + // the sink, so the switch never stalls the table. Before the remote + // delivers its first events the sink position still equals the + // dispatcher start ts, which would let a barely started local + // subscription win the race; hold on the remote's reported progress + // until data actually flows. baseline := requiredCheckpointTs - if d.remoteResolvedTs > baseline { + if remoteDeliveredTs > baseline { + baseline = remoteDeliveredTs + } + if baseline <= dispatcherStartTs && d.remoteResolvedTs > baseline { baseline = d.remoteResolvedTs } if readyResolvedTs < baseline { return readyDecision{} } + } else if !d.pendingRemoteEventServiceID.IsEmpty() { + // A remote reuse probe is in flight. A local ready only means the + // local registration is complete, not that the local subscription + // has served any data yet: its resolved ts still starts at the move + // checkpoint. Prefer the remote, which already serves the span, over + // letting the local win by arriving first. The probe expires after + // remoteProbeTimeout so a silent remote cannot block the fallback. + return readyDecision{} } decision := readyDecision{ @@ -185,6 +218,7 @@ func (d *dispatcherConnState) acceptReady( d.pendingRemoteEventServiceID = "" d.remoteCandidates = nil d.remoteResolvedTs = 0 + d.remoteProbeStartAt = time.Time{} return decision } @@ -208,6 +242,7 @@ func (d *dispatcherConnState) acceptReady( // later local ready may move the dispatcher back to local. d.pendingRemoteEventServiceID = "" d.remoteCandidates = nil + d.remoteProbeStartAt = time.Time{} // Record the remote progress the local service must catch up to before it // may win back. This is more meaningful than the sink checkpoint, which // still equals the start ts before the remote delivers its first events. @@ -233,6 +268,7 @@ func (d *dispatcherConnState) beginRemove(localServerID node.ID) ([]node.ID, boo d.pendingRemoteEventServiceID = "" d.remoteCandidates = nil d.remoteResolvedTs = 0 + d.remoteProbeStartAt = time.Time{} return []node.ID(targets), false } @@ -277,6 +313,7 @@ func (d *dispatcherConnState) beginRemoteProbing(nodes []string) (node.ID, bool) } candidate := node.ID(nodes[0]) d.pendingRemoteEventServiceID = candidate + d.remoteProbeStartAt = time.Now() d.remoteCandidates = nodes[1:] return candidate, true } @@ -297,6 +334,31 @@ func (d *dispatcherConnState) advanceRemoteProbeAfterNotReusable(from node.ID) ( candidate := node.ID(d.remoteCandidates[0]) d.remoteCandidates = d.remoteCandidates[1:] d.pendingRemoteEventServiceID = candidate + d.remoteProbeStartAt = time.Now() + return candidate, true +} + +// expireRemoteProbeLocked abandons a remote reuse probe that has been in flight +// longer than timeout without a ready or not-reusable response. It advances to +// the next candidate when one remains, otherwise it clears the pending probe so +// the dispatcher can fall back to the local event service. +func (d *dispatcherConnState) expireRemoteProbeLocked(now time.Time, timeout time.Duration) (node.ID, bool) { + d.Lock() + defer d.Unlock() + if d.removed || d.pendingRemoteEventServiceID.IsEmpty() { + return "", false + } + if d.remoteProbeStartAt.IsZero() || now.Sub(d.remoteProbeStartAt) < timeout { + return "", false + } + if len(d.remoteCandidates) == 0 { + d.pendingRemoteEventServiceID = "" + return "", true + } + candidate := node.ID(d.remoteCandidates[0]) + d.remoteCandidates = d.remoteCandidates[1:] + d.pendingRemoteEventServiceID = candidate + d.remoteProbeStartAt = now return candidate, true } @@ -504,11 +566,18 @@ func (s *dispatcherSession) handleReadyEvent(from node.ID, readyResolvedTs uint6 // connState decides whether this ready should be accepted and which stale // registrations must be cleaned up. Session only applies the side effects. requiredCheckpointTs := s.target.GetCheckpointTs() + // remoteDeliveredTs is the highest resolved ts the sink has received from + // the current event service, i.e. the position the remote has actually + // delivered. The local service must catch up to it before winning back so + // the switch never stalls the table while the local catches up from TiKV. + remoteDeliveredTs := s.target.GetResolvedTs() accepted := s.connState.acceptReady( from, s.localServerID, readyResolvedTs, requiredCheckpointTs, + remoteDeliveredTs, + s.target.GetStartTs(), ) for _, target := range accepted.cleanupTargets { s.removeFromLocked(target) @@ -651,6 +720,29 @@ func (s *dispatcherSession) startRemoteProbing(nodes []string) { s.sendRegisterRequest(candidate) } +// expireStaleRemoteProbe abandons a remote reuse probe that has been waiting +// too long, advancing to the next candidate or clearing the pending probe so +// the local event service can take over. It is called periodically so a silent +// remote cannot block the dispatcher forever. +func (s *dispatcherSession) expireStaleRemoteProbe() { + s.requestMu.Lock() + defer s.requestMu.Unlock() + if s.connState.isRemoved() { + return + } + nextCandidate, advanced := s.connState.expireRemoteProbeLocked(time.Now(), remoteProbeTimeout) + if !advanced { + return + } + log.Info("remote reuse probe timed out", + zap.Stringer("changefeedID", s.target.GetChangefeedID()), + zap.Stringer("dispatcherID", s.target.GetId()), + zap.Stringer("nextCandidate", nextCandidate)) + if !nextCandidate.IsEmpty() { + s.sendRegisterRequest(nextCandidate) + } +} + // Read-only session queries. func (s *dispatcherSession) getEventServiceID() node.ID { return s.connState.getCurrentEventServiceID() diff --git a/downstreamadapter/eventcollector/dispatcher_session_test_helper_test.go b/downstreamadapter/eventcollector/dispatcher_session_test_helper_test.go index d5148e805b..7a04236f4b 100644 --- a/downstreamadapter/eventcollector/dispatcher_session_test_helper_test.go +++ b/downstreamadapter/eventcollector/dispatcher_session_test_helper_test.go @@ -13,7 +13,11 @@ package eventcollector -import "github.com/pingcap/ticdc/pkg/node" +import ( + "time" + + "github.com/pingcap/ticdc/pkg/node" +) func setSessionState( session *dispatcherSession, @@ -40,6 +44,12 @@ func setSessionRemoteResolvedTs(session *dispatcherSession, resolvedTs uint64) { session.connState.remoteResolvedTs = resolvedTs } +func setSessionRemoteProbeStartAt(session *dispatcherSession, startAt time.Time) { + session.connState.Lock() + defer session.connState.Unlock() + session.connState.remoteProbeStartAt = startAt +} + func setSessionReadyCallback(session *dispatcherSession, readyCallback func()) { session.readyCallback = readyCallback } diff --git a/downstreamadapter/eventcollector/dispatcher_stat.go b/downstreamadapter/eventcollector/dispatcher_stat.go index 694c67b2af..204a170cb3 100644 --- a/downstreamadapter/eventcollector/dispatcher_stat.go +++ b/downstreamadapter/eventcollector/dispatcher_stat.go @@ -133,6 +133,14 @@ func (d *dispatcherStat) startRemoteProbing(nodes []string) { d.session.startRemoteProbing(nodes) } +// expireStaleRemoteProbe is called periodically by the event collector to +// abandon remote reuse probes that have been waiting too long, so a silent +// remote cannot block the dispatcher from falling back to the local event +// service. +func (d *dispatcherStat) expireStaleRemoteProbe() { + d.session.expireStaleRemoteProbe() +} + func (d *dispatcherStat) advanceEpochForReset(resetTs uint64) uint64 { for { currentState := d.loadCurrentEpochState() diff --git a/downstreamadapter/eventcollector/dispatcher_stat_test.go b/downstreamadapter/eventcollector/dispatcher_stat_test.go index 67eede2dc3..2ab6ed9d00 100644 --- a/downstreamadapter/eventcollector/dispatcher_stat_test.go +++ b/downstreamadapter/eventcollector/dispatcher_stat_test.go @@ -47,6 +47,7 @@ type mockDispatcher struct { handleError func(err error) events []dispatcher.DispatcherEvent checkPointTs uint64 + resolvedTs uint64 tableSpan *heartbeatpb.TableSpan lowLatencyMode bool @@ -128,7 +129,10 @@ func (m *mockDispatcher) GetSkipSyncpointAtStartTs() bool { } func (m *mockDispatcher) GetResolvedTs() uint64 { - return m.startTs + if m.resolvedTs == 0 { + return m.startTs + } + return m.resolvedTs } func (m *mockDispatcher) GetCheckpointTs() uint64 { @@ -636,7 +640,7 @@ func TestHandleSignalEvent(t *testing.T) { expectedPendingRemoteTarget: "", }, { - name: "handle ready event from local server without callback", + name: "hold local ready while remote probe is in flight", event: dispatcher.DispatcherEvent{ From: &localServerID, Event: &mockEvent{ @@ -646,10 +650,13 @@ func TestHandleSignalEvent(t *testing.T) { initialState: func(stat *dispatcherStat) { setSessionState(stat.session, "", true, remoteServerID) }, - expectedEventServiceID: localServerID, - expectedReceivingData: true, - expectedAwaitingLocalReady: false, - expectedPendingRemoteTarget: "", + // The local ready only means the local registration is complete; the + // remote probe is still in flight and may serve the span, so the + // local must not win by arriving first. + expectedEventServiceID: "", + expectedReceivingData: false, + expectedAwaitingLocalReady: true, + expectedPendingRemoteTarget: remoteServerID, }, { name: "handle ready event from remote server", @@ -812,7 +819,7 @@ func TestHandleLocalReadyEventCleansUpRemoteRegistrations(t *testing.T) { } } - t.Run("local ready removes pending remote register and resets local", func(t *testing.T) { + t.Run("local ready is held while remote probe is in flight", func(t *testing.T) { mockDisp := newMockDispatcher(dispatcherID, 0) mockEventCollector := newTestEventCollector(localServerID) stat := newDispatcherStat(mockDisp, mockEventCollector, nil) @@ -820,10 +827,31 @@ func TestHandleLocalReadyEventCleansUpRemoteRegistrations(t *testing.T) { stat.handleSignalEvent(newReadyEvent(localServerID)) + currentEventServiceID, localReadyPending, pendingRemoteTarget := sessionState(stat.session) + require.Empty(t, currentEventServiceID) + require.True(t, localReadyPending) + require.Equal(t, remoteServerID, pendingRemoteTarget) + requireNoDispatcherRequest(t, mockEventCollector) + }) + + t.Run("local ready wins after remote probe reports not reusable", func(t *testing.T) { + mockDisp := newMockDispatcher(dispatcherID, 0) + mockEventCollector := newTestEventCollector(localServerID) + stat := newDispatcherStat(mockDisp, mockEventCollector, nil) + setSessionState(stat.session, "", true, remoteServerID) + + // The only candidate reports not reusable, so the probe concludes with + // no remote to serve the span. The next local ready is accepted. + stat.handleSignalEvent(dispatcher.DispatcherEvent{ + From: &remoteServerID, + Event: &mockEvent{eventType: commonEvent.TypeNotReusableEvent}, + }) + requireNoDispatcherRequest(t, mockEventCollector) + + stat.handleSignalEvent(newReadyEvent(localServerID)) requireDispatcherRequests( t, - readDispatcherRequests(t, mockEventCollector, 2), - dispatcherRequestRecord{to: remoteServerID, action: eventpb.ActionType_ACTION_TYPE_REMOVE}, + readDispatcherRequests(t, mockEventCollector, 1), dispatcherRequestRecord{to: localServerID, action: eventpb.ActionType_ACTION_TYPE_RESET}, ) requireNoDispatcherRequest(t, mockEventCollector) @@ -880,20 +908,27 @@ func TestHandleLocalReadyEventCleansUpRemoteRegistrations(t *testing.T) { requireNoDispatcherRequest(t, mockEventCollector) }) - t.Run("local ready with callback still removes speculative remote register", func(t *testing.T) { + t.Run("local ready with callback is held while remote probe is in flight", func(t *testing.T) { mockDisp := newMockDispatcher(dispatcherID, 0) mockEventCollector := newTestEventCollector(localServerID) stat := newDispatcherStat(mockDisp, mockEventCollector, nil) setSessionState(stat.session, "", true, remoteServerID) - setSessionReadyCallback(stat.session, func() {}) + callbackFired := false + setSessionReadyCallback(stat.session, func() { callbackFired = true }) stat.handleSignalEvent(newReadyEvent(localServerID)) - requireDispatcherRequests( - t, - readDispatcherRequests(t, mockEventCollector, 1), - dispatcherRequestRecord{to: remoteServerID, action: eventpb.ActionType_ACTION_TYPE_REMOVE}, - ) + require.False(t, callbackFired) + requireNoDispatcherRequest(t, mockEventCollector) + + // Once the probe concludes without a reusable remote, the local ready + // fires the callback and no speculative remote register remains. + stat.handleSignalEvent(dispatcher.DispatcherEvent{ + From: &remoteServerID, + Event: &mockEvent{eventType: commonEvent.TypeNotReusableEvent}, + }) + stat.handleSignalEvent(newReadyEvent(localServerID)) + require.True(t, callbackFired) requireNoDispatcherRequest(t, mockEventCollector) }) } @@ -903,9 +938,11 @@ func TestLocalReadyGatedByRemoteServedResolvedTs(t *testing.T) { remoteServerID := node.ID("remote-server") dispatcherID := common.NewDispatcherID() - // The sink checkpoint stays at the start ts because the remote has not - // delivered its first events yet. The local ready must still cover the - // resolved ts the remote reported when it was accepted. + // The remote has been accepted but has not delivered its first events yet, + // so the sink checkpoint and resolved ts both still equal the dispatcher + // start ts. The local ready must still cover the resolved ts the remote + // reported, otherwise the fresh local subscription would win the race and + // stall the table while catching up from TiKV. mockDisp := newMockDispatcher(dispatcherID, 100) mockDisp.checkPointTs = 100 mockEventCollector := newTestEventCollector(localServerID) @@ -955,6 +992,116 @@ func TestLocalReadyGatedByRemoteServedResolvedTs(t *testing.T) { requireNoDispatcherRequest(t, mockEventCollector) } +func TestLocalReadyGatedByRemoteDeliveredResolvedTs(t *testing.T) { + localServerID := node.ID("local-server") + remoteServerID := node.ID("remote-server") + dispatcherID := common.NewDispatcherID() + + // Once the remote delivers data the sink resolved ts reflects the position + // it actually delivered (250), which is the catch-up target. The remote's + // reported frontier (300) must not inflate the baseline, otherwise the + // local would wait on the remote's pull progress it does not need. + mockDisp := newMockDispatcher(dispatcherID, 100) + mockDisp.checkPointTs = 100 + mockDisp.resolvedTs = 250 + mockEventCollector := newTestEventCollector(localServerID) + stat := newDispatcherStat(mockDisp, mockEventCollector, nil) + setSessionState(stat.session, "", true, remoteServerID) + + remoteReady := commonEvent.NewReadyEventWithResolvedTs(dispatcherID, 300) + stat.handleSignalEvent(dispatcher.DispatcherEvent{ + From: &remoteServerID, + Event: &remoteReady, + }) + requireDispatcherRequests( + t, + readDispatcherRequests(t, mockEventCollector, 1), + dispatcherRequestRecord{to: remoteServerID, action: eventpb.ActionType_ACTION_TYPE_RESET}, + ) + require.Equal(t, uint64(300), sessionRemoteResolvedTs(stat.session)) + + newLocalReadyEvent := func(resolvedTs uint64) dispatcher.DispatcherEvent { + ready := commonEvent.NewReadyEventWithResolvedTs(dispatcherID, resolvedTs) + return dispatcher.DispatcherEvent{ + From: &localServerID, + Event: &ready, + } + } + + // The local must cover what the remote delivered (250), not its frontier. + stat.handleSignalEvent(newLocalReadyEvent(249)) + currentEventServiceID, localReadyPending, pendingRemoteTarget := sessionState(stat.session) + require.Equal(t, remoteServerID, currentEventServiceID) + require.True(t, localReadyPending) + require.Empty(t, pendingRemoteTarget) + requireNoDispatcherRequest(t, mockEventCollector) + + stat.handleSignalEvent(newLocalReadyEvent(250)) + requireDispatcherRequests( + t, + readDispatcherRequests(t, mockEventCollector, 2), + dispatcherRequestRecord{to: remoteServerID, action: eventpb.ActionType_ACTION_TYPE_REMOVE}, + dispatcherRequestRecord{to: localServerID, action: eventpb.ActionType_ACTION_TYPE_RESET}, + ) + require.Zero(t, sessionRemoteResolvedTs(stat.session)) + requireNoDispatcherRequest(t, mockEventCollector) +} + +func TestRemoteProbeExpiryFallsBackToLocal(t *testing.T) { + localServerID := node.ID("local-server") + remoteServerID := node.ID("remote-server") + dispatcherID := common.NewDispatcherID() + + newLocalReadyEvent := func(resolvedTs uint64) dispatcher.DispatcherEvent { + ready := commonEvent.NewReadyEventWithResolvedTs(dispatcherID, resolvedTs) + return dispatcher.DispatcherEvent{ + From: &localServerID, + Event: &ready, + } + } + + t.Run("silent remote probe expires and local ready is accepted", func(t *testing.T) { + mockDisp := newMockDispatcher(dispatcherID, 100) + mockEventCollector := newTestEventCollector(localServerID) + stat := newDispatcherStat(mockDisp, mockEventCollector, nil) + setSessionState(stat.session, "", true, remoteServerID) + + // While the probe is still young the local ready is held. + stat.handleSignalEvent(newLocalReadyEvent(100)) + requireNoDispatcherRequest(t, mockEventCollector) + + // Once the probe has been in flight past the timeout, the periodic + // expiry clears it so the next local ready wins. + setSessionRemoteProbeStartAt(stat.session, time.Now().Add(-remoteProbeTimeout-time.Second)) + stat.expireStaleRemoteProbe() + currentEventServiceID, localReadyPending, pendingRemoteTarget := sessionState(stat.session) + require.Empty(t, currentEventServiceID) + require.True(t, localReadyPending) + require.Empty(t, pendingRemoteTarget) + + stat.handleSignalEvent(newLocalReadyEvent(100)) + requireDispatcherRequests( + t, + readDispatcherRequests(t, mockEventCollector, 1), + dispatcherRequestRecord{to: localServerID, action: eventpb.ActionType_ACTION_TYPE_RESET}, + ) + requireNoDispatcherRequest(t, mockEventCollector) + }) + + t.Run("young remote probe is not expired", func(t *testing.T) { + mockDisp := newMockDispatcher(dispatcherID, 100) + mockEventCollector := newTestEventCollector(localServerID) + stat := newDispatcherStat(mockDisp, mockEventCollector, nil) + setSessionState(stat.session, "", true, remoteServerID) + + stat.expireStaleRemoteProbe() + currentEventServiceID, _, pendingRemoteTarget := sessionState(stat.session) + require.Empty(t, currentEventServiceID) + require.Equal(t, remoteServerID, pendingRemoteTarget) + requireNoDispatcherRequest(t, mockEventCollector) + }) +} + func TestInitialLocalReadyCallbackIsOneShot(t *testing.T) { localServerID := node.ID("local-server") dispatcherID := common.NewDispatcherID() diff --git a/downstreamadapter/eventcollector/event_collector.go b/downstreamadapter/eventcollector/event_collector.go index 6456f1ada2..ccc5f9340e 100644 --- a/downstreamadapter/eventcollector/event_collector.go +++ b/downstreamadapter/eventcollector/event_collector.go @@ -396,11 +396,23 @@ func (c *EventCollector) sendEventServiceHeartbeats(ctx context.Context) error { case <-ctx.Done(): return context.Cause(ctx) case <-ticker.C: + c.expireStaleRemoteProbes() c.sendDispatcherHeartbeat() } } } +// expireStaleRemoteProbes abandons remote reuse probes that have been waiting +// too long so the affected dispatchers can fall back to their local event +// service instead of blocking on a silent remote. +func (c *EventCollector) expireStaleRemoteProbes() { + c.dispatcherMap.Range(func(_, value any) bool { + stat := value.(*dispatcherStat) + stat.expireStaleRemoteProbe() + return true + }) +} + func (c *EventCollector) sendDispatcherHeartbeat() { groupedHeartbeats := c.groupHeartbeat() for serverID, heartbeat := range groupedHeartbeats { From f92e0af55cf6bc2bc52a3f8af570b15c487ed609 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Wed, 19 Aug 2026 04:15:51 +0000 Subject: [PATCH 15/18] hold local ready Signed-off-by: wk989898 --- .../eventcollector/dispatcher_session.go | 68 +++++++++++++++---- .../dispatcher_session_test_helper_test.go | 3 + .../eventcollector/dispatcher_stat.go | 6 ++ .../eventcollector/dispatcher_stat_test.go | 48 +++++++++++++ .../eventcollector/event_collector.go | 8 +++ .../eventcollector/event_collector_test.go | 16 +++++ 6 files changed, 136 insertions(+), 13 deletions(-) diff --git a/downstreamadapter/eventcollector/dispatcher_session.go b/downstreamadapter/eventcollector/dispatcher_session.go index ccdac4a715..c6a3f537ea 100644 --- a/downstreamadapter/eventcollector/dispatcher_session.go +++ b/downstreamadapter/eventcollector/dispatcher_session.go @@ -91,9 +91,12 @@ type dispatcherConnState struct { // equals the dispatcher start ts, which would let a barely started local // subscription win the race and stall the table while catching up from TiKV. remoteResolvedTs uint64 - // remoteProbeStartAt records when the current remote reuse probe was sent, - // so a silent remote (no ready and no not-reusable response) can be - // abandoned after remoteProbeTimeout instead of blocking the dispatcher. + // remoteProbeStartAt marks the start of the remote reuse probing effort: + // from when the reusable-event-service request is sent (t0) through the + // candidate register round-trips. While it is non-zero the local ready is + // held so the remote (which already serves the span) wins over a local + // subscription that has not served data yet. It is cleared when the probe + // concludes (remote accepted, no reusable candidate, or timeout). remoteProbeStartAt time.Time } @@ -108,6 +111,19 @@ func (d *dispatcherConnState) beginRegisterToLocal() bool { return true } +// beginRemoteProbeRequest records that the dispatcher has asked the log +// coordinator for reusable remote event services. The local ready is held from +// this point (not only after a candidate register is sent) so the remote gets a +// chance even when the coordinator response is slower than the local ready. +func (d *dispatcherConnState) beginRemoteProbeRequest() { + d.Lock() + defer d.Unlock() + if d.removed || !d.remoteProbeStartAt.IsZero() { + return + } + d.remoteProbeStartAt = time.Now() +} + // beginRegisterToRemote records that the dispatcher is attempting to register to // a specific remote EventService. func (d *dispatcherConnState) beginRegisterToRemote(serverID node.ID) bool { @@ -196,13 +212,15 @@ func (d *dispatcherConnState) acceptReady( if readyResolvedTs < baseline { return readyDecision{} } - } else if !d.pendingRemoteEventServiceID.IsEmpty() { - // A remote reuse probe is in flight. A local ready only means the - // local registration is complete, not that the local subscription - // has served any data yet: its resolved ts still starts at the move - // checkpoint. Prefer the remote, which already serves the span, over - // letting the local win by arriving first. The probe expires after - // remoteProbeTimeout so a silent remote cannot block the fallback. + } else if !d.remoteProbeStartAt.IsZero() { + // A remote reuse probe is in flight, either waiting for the log + // coordinator response or registering a candidate. A local ready + // only means the local registration is complete, not that the local + // subscription has served any data yet: its resolved ts still starts + // at the move checkpoint. Prefer the remote, which already serves + // the span, over letting the local win by arriving first. The probe + // expires after remoteProbeTimeout so a silent coordinator/remote + // cannot block the fallback. return readyDecision{} } @@ -309,6 +327,9 @@ func (d *dispatcherConnState) beginRemoteProbing(nodes []string) (node.ID, bool) return "", false } if len(nodes) == 0 { + // No remote can reuse the span, so the probing effort concludes and the + // local event service may take over. + d.remoteProbeStartAt = time.Time{} return "", false } candidate := node.ID(nodes[0]) @@ -328,7 +349,10 @@ func (d *dispatcherConnState) advanceRemoteProbeAfterNotReusable(from node.ID) ( return "", false } if len(d.remoteCandidates) == 0 { + // No more candidates: the probing effort concludes and the local event + // service may take over. d.pendingRemoteEventServiceID = "" + d.remoteProbeStartAt = time.Time{} return "", true } candidate := node.ID(d.remoteCandidates[0]) @@ -341,18 +365,27 @@ func (d *dispatcherConnState) advanceRemoteProbeAfterNotReusable(from node.ID) ( // expireRemoteProbeLocked abandons a remote reuse probe that has been in flight // longer than timeout without a ready or not-reusable response. It advances to // the next candidate when one remains, otherwise it clears the pending probe so -// the dispatcher can fall back to the local event service. +// the dispatcher can fall back to the local event service. The probe may still +// be waiting for the log coordinator response (no pending candidate yet); in +// that case the whole effort is abandoned the same way. func (d *dispatcherConnState) expireRemoteProbeLocked(now time.Time, timeout time.Duration) (node.ID, bool) { d.Lock() defer d.Unlock() - if d.removed || d.pendingRemoteEventServiceID.IsEmpty() { + if d.removed || d.remoteProbeStartAt.IsZero() { return "", false } - if d.remoteProbeStartAt.IsZero() || now.Sub(d.remoteProbeStartAt) < timeout { + if now.Sub(d.remoteProbeStartAt) < timeout { return "", false } + if d.pendingRemoteEventServiceID.IsEmpty() { + // Still waiting for the reusable-event-service response. Conclude the + // probing effort and let the local event service take over. + d.remoteProbeStartAt = time.Time{} + return "", true + } if len(d.remoteCandidates) == 0 { d.pendingRemoteEventServiceID = "" + d.remoteProbeStartAt = time.Time{} return "", true } candidate := node.ID(d.remoteCandidates[0]) @@ -720,6 +753,15 @@ func (s *dispatcherSession) startRemoteProbing(nodes []string) { s.sendRegisterRequest(candidate) } +// beginRemoteProbeRequest marks the start of the remote reuse probing effort. +// It is called when the reusable-event-service request is sent, so the local +// ready is held even before the log coordinator responds. +func (s *dispatcherSession) beginRemoteProbeRequest() { + s.requestMu.Lock() + defer s.requestMu.Unlock() + s.connState.beginRemoteProbeRequest() +} + // expireStaleRemoteProbe abandons a remote reuse probe that has been waiting // too long, advancing to the next candidate or clearing the pending probe so // the local event service can take over. It is called periodically so a silent diff --git a/downstreamadapter/eventcollector/dispatcher_session_test_helper_test.go b/downstreamadapter/eventcollector/dispatcher_session_test_helper_test.go index 7a04236f4b..6e3047a1ce 100644 --- a/downstreamadapter/eventcollector/dispatcher_session_test_helper_test.go +++ b/downstreamadapter/eventcollector/dispatcher_session_test_helper_test.go @@ -30,6 +30,9 @@ func setSessionState( session.connState.currentEventServiceID = currentEventServiceID session.connState.localReadyPending = localReadyPending session.connState.pendingRemoteEventServiceID = pendingRemoteTarget + if !pendingRemoteTarget.IsEmpty() { + session.connState.remoteProbeStartAt = time.Now() + } } func setSessionRemoteCandidates(session *dispatcherSession, nodes []string) { diff --git a/downstreamadapter/eventcollector/dispatcher_stat.go b/downstreamadapter/eventcollector/dispatcher_stat.go index 204a170cb3..fe9583dad2 100644 --- a/downstreamadapter/eventcollector/dispatcher_stat.go +++ b/downstreamadapter/eventcollector/dispatcher_stat.go @@ -133,6 +133,12 @@ func (d *dispatcherStat) startRemoteProbing(nodes []string) { d.session.startRemoteProbing(nodes) } +// beginRemoteProbeRequest marks the start of the remote reuse probing effort so +// the local ready is held even before the log coordinator responds. +func (d *dispatcherStat) beginRemoteProbeRequest() { + d.session.beginRemoteProbeRequest() +} + // expireStaleRemoteProbe is called periodically by the event collector to // abandon remote reuse probes that have been waiting too long, so a silent // remote cannot block the dispatcher from falling back to the local event diff --git a/downstreamadapter/eventcollector/dispatcher_stat_test.go b/downstreamadapter/eventcollector/dispatcher_stat_test.go index 2ab6ed9d00..cee1caaf71 100644 --- a/downstreamadapter/eventcollector/dispatcher_stat_test.go +++ b/downstreamadapter/eventcollector/dispatcher_stat_test.go @@ -1102,6 +1102,54 @@ func TestRemoteProbeExpiryFallsBackToLocal(t *testing.T) { }) } +func TestRemoteProbeRequestGatesLocalReady(t *testing.T) { + localServerID := node.ID("local-server") + remoteServerID := node.ID("remote-server") + dispatcherID := common.NewDispatcherID() + + newLocalReadyEvent := func(resolvedTs uint64) dispatcher.DispatcherEvent { + ready := commonEvent.NewReadyEventWithResolvedTs(dispatcherID, resolvedTs) + return dispatcher.DispatcherEvent{ + From: &localServerID, + Event: &ready, + } + } + + mockDisp := newMockDispatcher(dispatcherID, 100) + mockEventCollector := newTestEventCollector(localServerID) + stat := newDispatcherStat(mockDisp, mockEventCollector, nil) + setSessionState(stat.session, "", true, "") + + // The reusable-event-service request is sent before any candidate is known. + // The local ready must be held from this point so the remote gets a chance + // even when the log coordinator responds later than the local ready. + stat.beginRemoteProbeRequest() + stat.handleSignalEvent(newLocalReadyEvent(100)) + currentEventServiceID, localReadyPending, _ := sessionState(stat.session) + require.Empty(t, currentEventServiceID) + require.True(t, localReadyPending) + requireNoDispatcherRequest(t, mockEventCollector) + + // The log coordinator responds with a reusable candidate; the remote wins. + stat.startRemoteProbing([]string{remoteServerID.String()}) + requireDispatcherRequests( + t, + readDispatcherRequests(t, mockEventCollector, 1), + dispatcherRequestRecord{to: remoteServerID, action: eventpb.ActionType_ACTION_TYPE_REGISTER}, + ) + remoteReady := commonEvent.NewReadyEventWithResolvedTs(dispatcherID, 300) + stat.handleSignalEvent(dispatcher.DispatcherEvent{ + From: &remoteServerID, + Event: &remoteReady, + }) + requireDispatcherRequests( + t, + readDispatcherRequests(t, mockEventCollector, 1), + dispatcherRequestRecord{to: remoteServerID, action: eventpb.ActionType_ACTION_TYPE_RESET}, + ) + requireNoDispatcherRequest(t, mockEventCollector) +} + func TestInitialLocalReadyCallbackIsOneShot(t *testing.T) { localServerID := node.ID("local-server") dispatcherID := common.NewDispatcherID() diff --git a/downstreamadapter/eventcollector/event_collector.go b/downstreamadapter/eventcollector/event_collector.go index ccc5f9340e..17b5fbb715 100644 --- a/downstreamadapter/eventcollector/event_collector.go +++ b/downstreamadapter/eventcollector/event_collector.go @@ -240,6 +240,14 @@ func (c *EventCollector) Close() { func (c *EventCollector) AddDispatcher(target dispatcher.DispatcherService, memoryQuota uint64) { c.PrepareAddDispatcher(target, memoryQuota, nil) + // Mark the remote reuse probing effort as started before sending the + // reusable-event-service request, so the local ready is held from the very + // beginning instead of letting the local subscription win by arriving first. + if target.GetTableSpan().TableID != 0 { + if v, ok := c.dispatcherMap.Load(target.GetId()); ok { + v.(*dispatcherStat).beginRemoteProbeRequest() + } + } c.logCoordinatorClient.requestReusableEventService(target) } diff --git a/downstreamadapter/eventcollector/event_collector_test.go b/downstreamadapter/eventcollector/event_collector_test.go index 0fa5c75273..beccfd444c 100644 --- a/downstreamadapter/eventcollector/event_collector_test.go +++ b/downstreamadapter/eventcollector/event_collector_test.go @@ -159,6 +159,19 @@ func newMessage(id node.ID, msg messaging.IOTypeT) *messaging.TargetMessage { return targetMessage } +// concludeRemoteProbe simulates the log coordinator returning no reusable +// candidates so the remote probing effort ends and the local event service can +// take over. Tests that exercise the collector without the log coordinator must +// call this after AddDispatcher, otherwise the local ready is held while the +// probe is in flight. +func concludeRemoteProbe(c *EventCollector, id common.DispatcherID) { + v, ok := c.dispatcherMap.Load(id) + if !ok { + return + } + v.(*dispatcherStat).startRemoteProbing(nil) +} + func TestProcessMessage(t *testing.T) { ctx := context.Background() node := node.NewInfo("127.0.0.1:18300", "") @@ -214,6 +227,7 @@ func TestProcessMessage(t *testing.T) { } } c.AddDispatcher(d, util.GetOrZero(config.GetDefaultReplicaConfig().MemoryQuota)) + concludeRemoteProbe(c, did) ch <- newMessage(node.ID, &readyEvent) ch <- newMessage(node.ID, &handshakeEvent) @@ -440,6 +454,7 @@ func TestEventCollectorBatchByCount(t *testing.T) { return false } c.AddDispatcher(d, util.GetOrZero(config.GetDefaultReplicaConfig().MemoryQuota)) + concludeRemoteProbe(c, did) from := localServerID readyEvent := commonEvent.NewReadyEvent(did) @@ -529,6 +544,7 @@ func TestEventCollectorBatchByBytes(t *testing.T) { } } c.AddDispatcher(d, util.GetOrZero(config.GetDefaultReplicaConfig().MemoryQuota)) + concludeRemoteProbe(c, did) from := localServerID readyEvent := commonEvent.NewReadyEvent(did) From c0b40ef07ff24c19afeb844e517af9769afcc68c Mon Sep 17 00:00:00 2001 From: wk989898 Date: Wed, 19 Aug 2026 11:08:45 +0000 Subject: [PATCH 16/18] add localHasCaughtUp Signed-off-by: wk989898 --- .../eventcollector/dispatcher_session.go | 42 +++++++++++++++---- .../eventcollector/dispatcher_stat_test.go | 39 +++++++++++++++++ 2 files changed, 74 insertions(+), 7 deletions(-) diff --git a/downstreamadapter/eventcollector/dispatcher_session.go b/downstreamadapter/eventcollector/dispatcher_session.go index c6a3f537ea..f4e1de0ced 100644 --- a/downstreamadapter/eventcollector/dispatcher_session.go +++ b/downstreamadapter/eventcollector/dispatcher_session.go @@ -25,6 +25,7 @@ import ( commonEvent "github.com/pingcap/ticdc/pkg/common/event" "github.com/pingcap/ticdc/pkg/messaging" "github.com/pingcap/ticdc/pkg/node" + "github.com/tikv/client-go/v2/oracle" "go.uber.org/zap" ) @@ -33,7 +34,15 @@ import ( // up on that candidate and falls back to the local event service. Without it, a // silent remote would leave the dispatcher permanently waiting because the // local ready is held while the probe is in flight. -const remoteProbeTimeout = 10 * time.Second +const remoteProbeTimeout = 5 * time.Second + +// remoteProbeLocalCatchUpMargin is the minimum physical-ts lead the local +// subscription must have over the dispatcher start ts before its ready may win +// while a remote reuse probe is still in flight. A fresh subscription that has +// barely started should yield to the remote (which already serves the span); a +// reused subscription that is already serving real data may win immediately so +// the dispatcher never waits for a probe it does not need. +const remoteProbeLocalCatchUpMargin = time.Second // dispatcherConnState owns the EventService registration state for one dispatcher. // It does not send messages. Its job is to apply atomic state transitions and @@ -124,6 +133,19 @@ func (d *dispatcherConnState) beginRemoteProbeRequest() { d.remoteProbeStartAt = time.Now() } +// localHasCaughtUp reports whether the local subscription has already pulled at +// least remoteProbeLocalCatchUpMargin past the dispatcher start ts, i.e. it is +// serving real data and winning the local ready immediately does not risk a +// stall. A fresh subscription that has barely started returns false so the +// remote probe (which already serves the span) gets a chance. +func (d *dispatcherConnState) localHasCaughtUp(localResolvedTs uint64, dispatcherStartTs uint64) bool { + if localResolvedTs <= dispatcherStartTs { + return false + } + return oracle.ExtractPhysical(localResolvedTs)-oracle.ExtractPhysical(dispatcherStartTs) >= + remoteProbeLocalCatchUpMargin.Milliseconds() +} + // beginRegisterToRemote records that the dispatcher is attempting to register to // a specific remote EventService. func (d *dispatcherConnState) beginRegisterToRemote(serverID node.ID) bool { @@ -216,12 +238,18 @@ func (d *dispatcherConnState) acceptReady( // A remote reuse probe is in flight, either waiting for the log // coordinator response or registering a candidate. A local ready // only means the local registration is complete, not that the local - // subscription has served any data yet: its resolved ts still starts - // at the move checkpoint. Prefer the remote, which already serves - // the span, over letting the local win by arriving first. The probe - // expires after remoteProbeTimeout so a silent coordinator/remote - // cannot block the fallback. - return readyDecision{} + // subscription has served any data yet: a fresh subscription's + // resolved ts still starts at the move checkpoint. Prefer the + // remote, which already serves the span, over letting the local win + // by arriving first. However, if the local subscription has already + // pulled well past the move checkpoint (e.g. the node reused an + // existing subscription), it is already as fast as the remote and + // should win immediately instead of waiting. The probe expires after + // remoteProbeTimeout so a silent coordinator/remote cannot block the + // fallback. + if !d.localHasCaughtUp(readyResolvedTs, dispatcherStartTs) { + return readyDecision{} + } } decision := readyDecision{ diff --git a/downstreamadapter/eventcollector/dispatcher_stat_test.go b/downstreamadapter/eventcollector/dispatcher_stat_test.go index cee1caaf71..2a3542cbb5 100644 --- a/downstreamadapter/eventcollector/dispatcher_stat_test.go +++ b/downstreamadapter/eventcollector/dispatcher_stat_test.go @@ -1150,6 +1150,45 @@ func TestRemoteProbeRequestGatesLocalReady(t *testing.T) { requireNoDispatcherRequest(t, mockEventCollector) } +func TestLocalReadyAcceptedWhenCaughtUpDuringProbe(t *testing.T) { + localServerID := node.ID("local-server") + remoteServerID := node.ID("remote-server") + dispatcherID := common.NewDispatcherID() + baseTime := time.Now() + + mockDisp := newMockDispatcher(dispatcherID, oracle.GoTimeToTS(baseTime)) + mockEventCollector := newTestEventCollector(localServerID) + stat := newDispatcherStat(mockDisp, mockEventCollector, nil) + setSessionState(stat.session, "", true, remoteServerID) + + newLocalReadyEvent := func(resolvedTs uint64) dispatcher.DispatcherEvent { + ready := commonEvent.NewReadyEventWithResolvedTs(dispatcherID, resolvedTs) + return dispatcher.DispatcherEvent{ + From: &localServerID, + Event: &ready, + } + } + + // A fresh local subscription that has barely started is held for the probe. + stat.handleSignalEvent(newLocalReadyEvent(oracle.GoTimeToTS(baseTime))) + currentEventServiceID, localReadyPending, _ := sessionState(stat.session) + require.Empty(t, currentEventServiceID) + require.True(t, localReadyPending) + requireNoDispatcherRequest(t, mockEventCollector) + + // A local subscription that has already caught up (pulled well past the + // move checkpoint) wins immediately even while the probe is in flight, so + // the dispatcher never waits for a probe it does not need. + stat.handleSignalEvent(newLocalReadyEvent(oracle.GoTimeToTS(baseTime.Add(2 * time.Second)))) + requireDispatcherRequests( + t, + readDispatcherRequests(t, mockEventCollector, 2), + dispatcherRequestRecord{to: remoteServerID, action: eventpb.ActionType_ACTION_TYPE_REMOVE}, + dispatcherRequestRecord{to: localServerID, action: eventpb.ActionType_ACTION_TYPE_RESET}, + ) + requireNoDispatcherRequest(t, mockEventCollector) +} + func TestInitialLocalReadyCallbackIsOneShot(t *testing.T) { localServerID := node.ID("local-server") dispatcherID := common.NewDispatcherID() From 3db98b96605372e72d620135f3a2fcadbdf1f035 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Wed, 19 Aug 2026 14:42:47 +0000 Subject: [PATCH 17/18] make written resolved ts as ready event ts Signed-off-by: wk989898 --- .../eventcollector/dispatcher_session.go | 164 +------------- .../dispatcher_session_test_helper_test.go | 15 +- .../eventcollector/dispatcher_stat.go | 14 -- .../eventcollector/dispatcher_stat_test.go | 210 +++--------------- .../eventcollector/event_collector.go | 20 -- .../eventcollector/event_collector_test.go | 16 -- logservice/eventstore/event_store.go | 59 +++++ pkg/eventservice/event_broker.go | 10 +- pkg/eventservice/event_service_test.go | 4 + 9 files changed, 116 insertions(+), 396 deletions(-) diff --git a/downstreamadapter/eventcollector/dispatcher_session.go b/downstreamadapter/eventcollector/dispatcher_session.go index f4e1de0ced..754e8df15e 100644 --- a/downstreamadapter/eventcollector/dispatcher_session.go +++ b/downstreamadapter/eventcollector/dispatcher_session.go @@ -16,7 +16,6 @@ package eventcollector import ( "slices" "sync" - "time" "github.com/pingcap/log" "github.com/pingcap/ticdc/downstreamadapter/dispatcher" @@ -25,25 +24,9 @@ import ( commonEvent "github.com/pingcap/ticdc/pkg/common/event" "github.com/pingcap/ticdc/pkg/messaging" "github.com/pingcap/ticdc/pkg/node" - "github.com/tikv/client-go/v2/oracle" "go.uber.org/zap" ) -// remoteProbeTimeout bounds how long a single remote reuse probe may stay in -// flight without a ready or not-reusable response before the dispatcher gives -// up on that candidate and falls back to the local event service. Without it, a -// silent remote would leave the dispatcher permanently waiting because the -// local ready is held while the probe is in flight. -const remoteProbeTimeout = 5 * time.Second - -// remoteProbeLocalCatchUpMargin is the minimum physical-ts lead the local -// subscription must have over the dispatcher start ts before its ready may win -// while a remote reuse probe is still in flight. A fresh subscription that has -// barely started should yield to the remote (which already serves the span); a -// reused subscription that is already serving real data may win immediately so -// the dispatcher never waits for a probe it does not need. -const remoteProbeLocalCatchUpMargin = time.Second - // dispatcherConnState owns the EventService registration state for one dispatcher. // It does not send messages. Its job is to apply atomic state transitions and // return the side-effect inputs that dispatcherSession should execute. @@ -65,13 +48,11 @@ const remoteProbeLocalCatchUpMargin = time.Second // pendingRemoteEventServiceID="" // // This means local registration and remote probing can overlap. A remote service -// may serve data first, but after the local service catches up to the progress -// the remote reported (and the dispatcher checkpoint), a later local ready moves -// the dispatcher back to local and cleans up remote registrations. While a -// remote reuse probe is still in flight the local ready is held, so the remote -// (which already serves the span) wins over a local subscription that has not -// served data yet; the probe expires after remoteProbeTimeout to unblock the -// local fallback. +// may serve data first (its ready usually arrives before the local ready because +// the local subscription has to initialize first), but after the local service +// catches up to the progress the remote reported (and the dispatcher +// checkpoint), a later local ready moves the dispatcher back to local and +// cleans up remote registrations. type dispatcherConnState struct { sync.RWMutex // removed marks the session as terminal after removal starts. New @@ -100,13 +81,6 @@ type dispatcherConnState struct { // equals the dispatcher start ts, which would let a barely started local // subscription win the race and stall the table while catching up from TiKV. remoteResolvedTs uint64 - // remoteProbeStartAt marks the start of the remote reuse probing effort: - // from when the reusable-event-service request is sent (t0) through the - // candidate register round-trips. While it is non-zero the local ready is - // held so the remote (which already serves the span) wins over a local - // subscription that has not served data yet. It is cleared when the probe - // concludes (remote accepted, no reusable candidate, or timeout). - remoteProbeStartAt time.Time } // Registration state transitions. @@ -120,32 +94,6 @@ func (d *dispatcherConnState) beginRegisterToLocal() bool { return true } -// beginRemoteProbeRequest records that the dispatcher has asked the log -// coordinator for reusable remote event services. The local ready is held from -// this point (not only after a candidate register is sent) so the remote gets a -// chance even when the coordinator response is slower than the local ready. -func (d *dispatcherConnState) beginRemoteProbeRequest() { - d.Lock() - defer d.Unlock() - if d.removed || !d.remoteProbeStartAt.IsZero() { - return - } - d.remoteProbeStartAt = time.Now() -} - -// localHasCaughtUp reports whether the local subscription has already pulled at -// least remoteProbeLocalCatchUpMargin past the dispatcher start ts, i.e. it is -// serving real data and winning the local ready immediately does not risk a -// stall. A fresh subscription that has barely started returns false so the -// remote probe (which already serves the span) gets a chance. -func (d *dispatcherConnState) localHasCaughtUp(localResolvedTs uint64, dispatcherStartTs uint64) bool { - if localResolvedTs <= dispatcherStartTs { - return false - } - return oracle.ExtractPhysical(localResolvedTs)-oracle.ExtractPhysical(dispatcherStartTs) >= - remoteProbeLocalCatchUpMargin.Milliseconds() -} - // beginRegisterToRemote records that the dispatcher is attempting to register to // a specific remote EventService. func (d *dispatcherConnState) beginRegisterToRemote(serverID node.ID) bool { @@ -155,7 +103,6 @@ func (d *dispatcherConnState) beginRegisterToRemote(serverID node.ID) bool { return false } d.pendingRemoteEventServiceID = serverID - d.remoteProbeStartAt = time.Now() return true } @@ -189,12 +136,9 @@ type readyDecision struct { // acceptReady enforces the ready acceptance rules: // 1. once local is already serving, any later remote ready is stale and should // only trigger cleanup; -// 2. local ready can be accepted while local registration is still pending, -// but only after any in-flight remote reuse probe has concluded (the remote -// already serves the span and should win over a local subscription that has -// not served data yet) and, when a remote is already serving, after the -// local resolved ts covers both the current dispatcher checkpoint and the -// progress the remote reported; +// 2. local ready can be accepted while local registration is still pending; +// when a remote is already serving, the local resolved ts must cover both +// the current dispatcher checkpoint and the progress the remote reported; // 3. remote ready is accepted only from the single remote candidate currently // being probed. func (d *dispatcherConnState) acceptReady( @@ -234,22 +178,6 @@ func (d *dispatcherConnState) acceptReady( if readyResolvedTs < baseline { return readyDecision{} } - } else if !d.remoteProbeStartAt.IsZero() { - // A remote reuse probe is in flight, either waiting for the log - // coordinator response or registering a candidate. A local ready - // only means the local registration is complete, not that the local - // subscription has served any data yet: a fresh subscription's - // resolved ts still starts at the move checkpoint. Prefer the - // remote, which already serves the span, over letting the local win - // by arriving first. However, if the local subscription has already - // pulled well past the move checkpoint (e.g. the node reused an - // existing subscription), it is already as fast as the remote and - // should win immediately instead of waiting. The probe expires after - // remoteProbeTimeout so a silent coordinator/remote cannot block the - // fallback. - if !d.localHasCaughtUp(readyResolvedTs, dispatcherStartTs) { - return readyDecision{} - } } decision := readyDecision{ @@ -264,7 +192,6 @@ func (d *dispatcherConnState) acceptReady( d.pendingRemoteEventServiceID = "" d.remoteCandidates = nil d.remoteResolvedTs = 0 - d.remoteProbeStartAt = time.Time{} return decision } @@ -288,7 +215,6 @@ func (d *dispatcherConnState) acceptReady( // later local ready may move the dispatcher back to local. d.pendingRemoteEventServiceID = "" d.remoteCandidates = nil - d.remoteProbeStartAt = time.Time{} // Record the remote progress the local service must catch up to before it // may win back. This is more meaningful than the sink checkpoint, which // still equals the start ts before the remote delivers its first events. @@ -314,7 +240,6 @@ func (d *dispatcherConnState) beginRemove(localServerID node.ID) ([]node.ID, boo d.pendingRemoteEventServiceID = "" d.remoteCandidates = nil d.remoteResolvedTs = 0 - d.remoteProbeStartAt = time.Time{} return []node.ID(targets), false } @@ -355,14 +280,10 @@ func (d *dispatcherConnState) beginRemoteProbing(nodes []string) (node.ID, bool) return "", false } if len(nodes) == 0 { - // No remote can reuse the span, so the probing effort concludes and the - // local event service may take over. - d.remoteProbeStartAt = time.Time{} return "", false } candidate := node.ID(nodes[0]) d.pendingRemoteEventServiceID = candidate - d.remoteProbeStartAt = time.Now() d.remoteCandidates = nodes[1:] return candidate, true } @@ -376,50 +297,13 @@ func (d *dispatcherConnState) advanceRemoteProbeAfterNotReusable(from node.ID) ( if d.removed || d.pendingRemoteEventServiceID != from { return "", false } - if len(d.remoteCandidates) == 0 { - // No more candidates: the probing effort concludes and the local event - // service may take over. - d.pendingRemoteEventServiceID = "" - d.remoteProbeStartAt = time.Time{} - return "", true - } - candidate := node.ID(d.remoteCandidates[0]) - d.remoteCandidates = d.remoteCandidates[1:] - d.pendingRemoteEventServiceID = candidate - d.remoteProbeStartAt = time.Now() - return candidate, true -} - -// expireRemoteProbeLocked abandons a remote reuse probe that has been in flight -// longer than timeout without a ready or not-reusable response. It advances to -// the next candidate when one remains, otherwise it clears the pending probe so -// the dispatcher can fall back to the local event service. The probe may still -// be waiting for the log coordinator response (no pending candidate yet); in -// that case the whole effort is abandoned the same way. -func (d *dispatcherConnState) expireRemoteProbeLocked(now time.Time, timeout time.Duration) (node.ID, bool) { - d.Lock() - defer d.Unlock() - if d.removed || d.remoteProbeStartAt.IsZero() { - return "", false - } - if now.Sub(d.remoteProbeStartAt) < timeout { - return "", false - } - if d.pendingRemoteEventServiceID.IsEmpty() { - // Still waiting for the reusable-event-service response. Conclude the - // probing effort and let the local event service take over. - d.remoteProbeStartAt = time.Time{} - return "", true - } if len(d.remoteCandidates) == 0 { d.pendingRemoteEventServiceID = "" - d.remoteProbeStartAt = time.Time{} return "", true } candidate := node.ID(d.remoteCandidates[0]) d.remoteCandidates = d.remoteCandidates[1:] d.pendingRemoteEventServiceID = candidate - d.remoteProbeStartAt = now return candidate, true } @@ -781,38 +665,6 @@ func (s *dispatcherSession) startRemoteProbing(nodes []string) { s.sendRegisterRequest(candidate) } -// beginRemoteProbeRequest marks the start of the remote reuse probing effort. -// It is called when the reusable-event-service request is sent, so the local -// ready is held even before the log coordinator responds. -func (s *dispatcherSession) beginRemoteProbeRequest() { - s.requestMu.Lock() - defer s.requestMu.Unlock() - s.connState.beginRemoteProbeRequest() -} - -// expireStaleRemoteProbe abandons a remote reuse probe that has been waiting -// too long, advancing to the next candidate or clearing the pending probe so -// the local event service can take over. It is called periodically so a silent -// remote cannot block the dispatcher forever. -func (s *dispatcherSession) expireStaleRemoteProbe() { - s.requestMu.Lock() - defer s.requestMu.Unlock() - if s.connState.isRemoved() { - return - } - nextCandidate, advanced := s.connState.expireRemoteProbeLocked(time.Now(), remoteProbeTimeout) - if !advanced { - return - } - log.Info("remote reuse probe timed out", - zap.Stringer("changefeedID", s.target.GetChangefeedID()), - zap.Stringer("dispatcherID", s.target.GetId()), - zap.Stringer("nextCandidate", nextCandidate)) - if !nextCandidate.IsEmpty() { - s.sendRegisterRequest(nextCandidate) - } -} - // Read-only session queries. func (s *dispatcherSession) getEventServiceID() node.ID { return s.connState.getCurrentEventServiceID() diff --git a/downstreamadapter/eventcollector/dispatcher_session_test_helper_test.go b/downstreamadapter/eventcollector/dispatcher_session_test_helper_test.go index 6e3047a1ce..d5148e805b 100644 --- a/downstreamadapter/eventcollector/dispatcher_session_test_helper_test.go +++ b/downstreamadapter/eventcollector/dispatcher_session_test_helper_test.go @@ -13,11 +13,7 @@ package eventcollector -import ( - "time" - - "github.com/pingcap/ticdc/pkg/node" -) +import "github.com/pingcap/ticdc/pkg/node" func setSessionState( session *dispatcherSession, @@ -30,9 +26,6 @@ func setSessionState( session.connState.currentEventServiceID = currentEventServiceID session.connState.localReadyPending = localReadyPending session.connState.pendingRemoteEventServiceID = pendingRemoteTarget - if !pendingRemoteTarget.IsEmpty() { - session.connState.remoteProbeStartAt = time.Now() - } } func setSessionRemoteCandidates(session *dispatcherSession, nodes []string) { @@ -47,12 +40,6 @@ func setSessionRemoteResolvedTs(session *dispatcherSession, resolvedTs uint64) { session.connState.remoteResolvedTs = resolvedTs } -func setSessionRemoteProbeStartAt(session *dispatcherSession, startAt time.Time) { - session.connState.Lock() - defer session.connState.Unlock() - session.connState.remoteProbeStartAt = startAt -} - func setSessionReadyCallback(session *dispatcherSession, readyCallback func()) { session.readyCallback = readyCallback } diff --git a/downstreamadapter/eventcollector/dispatcher_stat.go b/downstreamadapter/eventcollector/dispatcher_stat.go index fe9583dad2..694c67b2af 100644 --- a/downstreamadapter/eventcollector/dispatcher_stat.go +++ b/downstreamadapter/eventcollector/dispatcher_stat.go @@ -133,20 +133,6 @@ func (d *dispatcherStat) startRemoteProbing(nodes []string) { d.session.startRemoteProbing(nodes) } -// beginRemoteProbeRequest marks the start of the remote reuse probing effort so -// the local ready is held even before the log coordinator responds. -func (d *dispatcherStat) beginRemoteProbeRequest() { - d.session.beginRemoteProbeRequest() -} - -// expireStaleRemoteProbe is called periodically by the event collector to -// abandon remote reuse probes that have been waiting too long, so a silent -// remote cannot block the dispatcher from falling back to the local event -// service. -func (d *dispatcherStat) expireStaleRemoteProbe() { - d.session.expireStaleRemoteProbe() -} - func (d *dispatcherStat) advanceEpochForReset(resetTs uint64) uint64 { for { currentState := d.loadCurrentEpochState() diff --git a/downstreamadapter/eventcollector/dispatcher_stat_test.go b/downstreamadapter/eventcollector/dispatcher_stat_test.go index 2a3542cbb5..5188baad1d 100644 --- a/downstreamadapter/eventcollector/dispatcher_stat_test.go +++ b/downstreamadapter/eventcollector/dispatcher_stat_test.go @@ -640,7 +640,7 @@ func TestHandleSignalEvent(t *testing.T) { expectedPendingRemoteTarget: "", }, { - name: "hold local ready while remote probe is in flight", + name: "handle ready event from local server while remote probe is in flight", event: dispatcher.DispatcherEvent{ From: &localServerID, Event: &mockEvent{ @@ -650,13 +650,12 @@ func TestHandleSignalEvent(t *testing.T) { initialState: func(stat *dispatcherStat) { setSessionState(stat.session, "", true, remoteServerID) }, - // The local ready only means the local registration is complete; the - // remote probe is still in flight and may serve the span, so the - // local must not win by arriving first. - expectedEventServiceID: "", - expectedReceivingData: false, - expectedAwaitingLocalReady: true, - expectedPendingRemoteTarget: remoteServerID, + // The local ready wins immediately because no remote is serving yet; + // the remote only wins if its ready arrives first. + expectedEventServiceID: localServerID, + expectedReceivingData: true, + expectedAwaitingLocalReady: false, + expectedPendingRemoteTarget: "", }, { name: "handle ready event from remote server", @@ -819,7 +818,7 @@ func TestHandleLocalReadyEventCleansUpRemoteRegistrations(t *testing.T) { } } - t.Run("local ready is held while remote probe is in flight", func(t *testing.T) { + t.Run("local ready wins immediately while remote probe is in flight", func(t *testing.T) { mockDisp := newMockDispatcher(dispatcherID, 0) mockEventCollector := newTestEventCollector(localServerID) stat := newDispatcherStat(mockDisp, mockEventCollector, nil) @@ -828,32 +827,40 @@ func TestHandleLocalReadyEventCleansUpRemoteRegistrations(t *testing.T) { stat.handleSignalEvent(newReadyEvent(localServerID)) currentEventServiceID, localReadyPending, pendingRemoteTarget := sessionState(stat.session) - require.Empty(t, currentEventServiceID) - require.True(t, localReadyPending) - require.Equal(t, remoteServerID, pendingRemoteTarget) + require.Equal(t, localServerID, currentEventServiceID) + require.False(t, localReadyPending) + require.Empty(t, pendingRemoteTarget) + requireDispatcherRequests( + t, + readDispatcherRequests(t, mockEventCollector, 2), + dispatcherRequestRecord{to: remoteServerID, action: eventpb.ActionType_ACTION_TYPE_REMOVE}, + dispatcherRequestRecord{to: localServerID, action: eventpb.ActionType_ACTION_TYPE_RESET}, + ) requireNoDispatcherRequest(t, mockEventCollector) }) - t.Run("local ready wins after remote probe reports not reusable", func(t *testing.T) { + t.Run("remote ready after local won is stale and only triggers cleanup", func(t *testing.T) { mockDisp := newMockDispatcher(dispatcherID, 0) mockEventCollector := newTestEventCollector(localServerID) stat := newDispatcherStat(mockDisp, mockEventCollector, nil) setSessionState(stat.session, "", true, remoteServerID) - // The only candidate reports not reusable, so the probe concludes with - // no remote to serve the span. The next local ready is accepted. - stat.handleSignalEvent(dispatcher.DispatcherEvent{ - From: &remoteServerID, - Event: &mockEvent{eventType: commonEvent.TypeNotReusableEvent}, - }) - requireNoDispatcherRequest(t, mockEventCollector) - stat.handleSignalEvent(newReadyEvent(localServerID)) requireDispatcherRequests( t, - readDispatcherRequests(t, mockEventCollector, 1), + readDispatcherRequests(t, mockEventCollector, 2), + dispatcherRequestRecord{to: remoteServerID, action: eventpb.ActionType_ACTION_TYPE_REMOVE}, dispatcherRequestRecord{to: localServerID, action: eventpb.ActionType_ACTION_TYPE_RESET}, ) + + // A late remote ready is stale: the dispatcher is already on local, so + // it only triggers a cleanup of the stale remote registration. + stat.handleSignalEvent(newReadyEvent(remoteServerID)) + requireDispatcherRequests( + t, + readDispatcherRequests(t, mockEventCollector, 1), + dispatcherRequestRecord{to: remoteServerID, action: eventpb.ActionType_ACTION_TYPE_REMOVE}, + ) requireNoDispatcherRequest(t, mockEventCollector) }) @@ -908,7 +915,7 @@ func TestHandleLocalReadyEventCleansUpRemoteRegistrations(t *testing.T) { requireNoDispatcherRequest(t, mockEventCollector) }) - t.Run("local ready with callback is held while remote probe is in flight", func(t *testing.T) { + t.Run("local ready with callback fires immediately while remote probe is in flight", func(t *testing.T) { mockDisp := newMockDispatcher(dispatcherID, 0) mockEventCollector := newTestEventCollector(localServerID) stat := newDispatcherStat(mockDisp, mockEventCollector, nil) @@ -918,17 +925,12 @@ func TestHandleLocalReadyEventCleansUpRemoteRegistrations(t *testing.T) { stat.handleSignalEvent(newReadyEvent(localServerID)) - require.False(t, callbackFired) - requireNoDispatcherRequest(t, mockEventCollector) - - // Once the probe concludes without a reusable remote, the local ready - // fires the callback and no speculative remote register remains. - stat.handleSignalEvent(dispatcher.DispatcherEvent{ - From: &remoteServerID, - Event: &mockEvent{eventType: commonEvent.TypeNotReusableEvent}, - }) - stat.handleSignalEvent(newReadyEvent(localServerID)) require.True(t, callbackFired) + requireDispatcherRequests( + t, + readDispatcherRequests(t, mockEventCollector, 1), + dispatcherRequestRecord{to: remoteServerID, action: eventpb.ActionType_ACTION_TYPE_REMOVE}, + ) requireNoDispatcherRequest(t, mockEventCollector) }) } @@ -1047,148 +1049,6 @@ func TestLocalReadyGatedByRemoteDeliveredResolvedTs(t *testing.T) { requireNoDispatcherRequest(t, mockEventCollector) } -func TestRemoteProbeExpiryFallsBackToLocal(t *testing.T) { - localServerID := node.ID("local-server") - remoteServerID := node.ID("remote-server") - dispatcherID := common.NewDispatcherID() - - newLocalReadyEvent := func(resolvedTs uint64) dispatcher.DispatcherEvent { - ready := commonEvent.NewReadyEventWithResolvedTs(dispatcherID, resolvedTs) - return dispatcher.DispatcherEvent{ - From: &localServerID, - Event: &ready, - } - } - - t.Run("silent remote probe expires and local ready is accepted", func(t *testing.T) { - mockDisp := newMockDispatcher(dispatcherID, 100) - mockEventCollector := newTestEventCollector(localServerID) - stat := newDispatcherStat(mockDisp, mockEventCollector, nil) - setSessionState(stat.session, "", true, remoteServerID) - - // While the probe is still young the local ready is held. - stat.handleSignalEvent(newLocalReadyEvent(100)) - requireNoDispatcherRequest(t, mockEventCollector) - - // Once the probe has been in flight past the timeout, the periodic - // expiry clears it so the next local ready wins. - setSessionRemoteProbeStartAt(stat.session, time.Now().Add(-remoteProbeTimeout-time.Second)) - stat.expireStaleRemoteProbe() - currentEventServiceID, localReadyPending, pendingRemoteTarget := sessionState(stat.session) - require.Empty(t, currentEventServiceID) - require.True(t, localReadyPending) - require.Empty(t, pendingRemoteTarget) - - stat.handleSignalEvent(newLocalReadyEvent(100)) - requireDispatcherRequests( - t, - readDispatcherRequests(t, mockEventCollector, 1), - dispatcherRequestRecord{to: localServerID, action: eventpb.ActionType_ACTION_TYPE_RESET}, - ) - requireNoDispatcherRequest(t, mockEventCollector) - }) - - t.Run("young remote probe is not expired", func(t *testing.T) { - mockDisp := newMockDispatcher(dispatcherID, 100) - mockEventCollector := newTestEventCollector(localServerID) - stat := newDispatcherStat(mockDisp, mockEventCollector, nil) - setSessionState(stat.session, "", true, remoteServerID) - - stat.expireStaleRemoteProbe() - currentEventServiceID, _, pendingRemoteTarget := sessionState(stat.session) - require.Empty(t, currentEventServiceID) - require.Equal(t, remoteServerID, pendingRemoteTarget) - requireNoDispatcherRequest(t, mockEventCollector) - }) -} - -func TestRemoteProbeRequestGatesLocalReady(t *testing.T) { - localServerID := node.ID("local-server") - remoteServerID := node.ID("remote-server") - dispatcherID := common.NewDispatcherID() - - newLocalReadyEvent := func(resolvedTs uint64) dispatcher.DispatcherEvent { - ready := commonEvent.NewReadyEventWithResolvedTs(dispatcherID, resolvedTs) - return dispatcher.DispatcherEvent{ - From: &localServerID, - Event: &ready, - } - } - - mockDisp := newMockDispatcher(dispatcherID, 100) - mockEventCollector := newTestEventCollector(localServerID) - stat := newDispatcherStat(mockDisp, mockEventCollector, nil) - setSessionState(stat.session, "", true, "") - - // The reusable-event-service request is sent before any candidate is known. - // The local ready must be held from this point so the remote gets a chance - // even when the log coordinator responds later than the local ready. - stat.beginRemoteProbeRequest() - stat.handleSignalEvent(newLocalReadyEvent(100)) - currentEventServiceID, localReadyPending, _ := sessionState(stat.session) - require.Empty(t, currentEventServiceID) - require.True(t, localReadyPending) - requireNoDispatcherRequest(t, mockEventCollector) - - // The log coordinator responds with a reusable candidate; the remote wins. - stat.startRemoteProbing([]string{remoteServerID.String()}) - requireDispatcherRequests( - t, - readDispatcherRequests(t, mockEventCollector, 1), - dispatcherRequestRecord{to: remoteServerID, action: eventpb.ActionType_ACTION_TYPE_REGISTER}, - ) - remoteReady := commonEvent.NewReadyEventWithResolvedTs(dispatcherID, 300) - stat.handleSignalEvent(dispatcher.DispatcherEvent{ - From: &remoteServerID, - Event: &remoteReady, - }) - requireDispatcherRequests( - t, - readDispatcherRequests(t, mockEventCollector, 1), - dispatcherRequestRecord{to: remoteServerID, action: eventpb.ActionType_ACTION_TYPE_RESET}, - ) - requireNoDispatcherRequest(t, mockEventCollector) -} - -func TestLocalReadyAcceptedWhenCaughtUpDuringProbe(t *testing.T) { - localServerID := node.ID("local-server") - remoteServerID := node.ID("remote-server") - dispatcherID := common.NewDispatcherID() - baseTime := time.Now() - - mockDisp := newMockDispatcher(dispatcherID, oracle.GoTimeToTS(baseTime)) - mockEventCollector := newTestEventCollector(localServerID) - stat := newDispatcherStat(mockDisp, mockEventCollector, nil) - setSessionState(stat.session, "", true, remoteServerID) - - newLocalReadyEvent := func(resolvedTs uint64) dispatcher.DispatcherEvent { - ready := commonEvent.NewReadyEventWithResolvedTs(dispatcherID, resolvedTs) - return dispatcher.DispatcherEvent{ - From: &localServerID, - Event: &ready, - } - } - - // A fresh local subscription that has barely started is held for the probe. - stat.handleSignalEvent(newLocalReadyEvent(oracle.GoTimeToTS(baseTime))) - currentEventServiceID, localReadyPending, _ := sessionState(stat.session) - require.Empty(t, currentEventServiceID) - require.True(t, localReadyPending) - requireNoDispatcherRequest(t, mockEventCollector) - - // A local subscription that has already caught up (pulled well past the - // move checkpoint) wins immediately even while the probe is in flight, so - // the dispatcher never waits for a probe it does not need. - stat.handleSignalEvent(newLocalReadyEvent(oracle.GoTimeToTS(baseTime.Add(2 * time.Second)))) - requireDispatcherRequests( - t, - readDispatcherRequests(t, mockEventCollector, 2), - dispatcherRequestRecord{to: remoteServerID, action: eventpb.ActionType_ACTION_TYPE_REMOVE}, - dispatcherRequestRecord{to: localServerID, action: eventpb.ActionType_ACTION_TYPE_RESET}, - ) - requireNoDispatcherRequest(t, mockEventCollector) -} - func TestInitialLocalReadyCallbackIsOneShot(t *testing.T) { localServerID := node.ID("local-server") dispatcherID := common.NewDispatcherID() diff --git a/downstreamadapter/eventcollector/event_collector.go b/downstreamadapter/eventcollector/event_collector.go index 17b5fbb715..6456f1ada2 100644 --- a/downstreamadapter/eventcollector/event_collector.go +++ b/downstreamadapter/eventcollector/event_collector.go @@ -240,14 +240,6 @@ func (c *EventCollector) Close() { func (c *EventCollector) AddDispatcher(target dispatcher.DispatcherService, memoryQuota uint64) { c.PrepareAddDispatcher(target, memoryQuota, nil) - // Mark the remote reuse probing effort as started before sending the - // reusable-event-service request, so the local ready is held from the very - // beginning instead of letting the local subscription win by arriving first. - if target.GetTableSpan().TableID != 0 { - if v, ok := c.dispatcherMap.Load(target.GetId()); ok { - v.(*dispatcherStat).beginRemoteProbeRequest() - } - } c.logCoordinatorClient.requestReusableEventService(target) } @@ -404,23 +396,11 @@ func (c *EventCollector) sendEventServiceHeartbeats(ctx context.Context) error { case <-ctx.Done(): return context.Cause(ctx) case <-ticker.C: - c.expireStaleRemoteProbes() c.sendDispatcherHeartbeat() } } } -// expireStaleRemoteProbes abandons remote reuse probes that have been waiting -// too long so the affected dispatchers can fall back to their local event -// service instead of blocking on a silent remote. -func (c *EventCollector) expireStaleRemoteProbes() { - c.dispatcherMap.Range(func(_, value any) bool { - stat := value.(*dispatcherStat) - stat.expireStaleRemoteProbe() - return true - }) -} - func (c *EventCollector) sendDispatcherHeartbeat() { groupedHeartbeats := c.groupHeartbeat() for serverID, heartbeat := range groupedHeartbeats { diff --git a/downstreamadapter/eventcollector/event_collector_test.go b/downstreamadapter/eventcollector/event_collector_test.go index beccfd444c..0fa5c75273 100644 --- a/downstreamadapter/eventcollector/event_collector_test.go +++ b/downstreamadapter/eventcollector/event_collector_test.go @@ -159,19 +159,6 @@ func newMessage(id node.ID, msg messaging.IOTypeT) *messaging.TargetMessage { return targetMessage } -// concludeRemoteProbe simulates the log coordinator returning no reusable -// candidates so the remote probing effort ends and the local event service can -// take over. Tests that exercise the collector without the log coordinator must -// call this after AddDispatcher, otherwise the local ready is held while the -// probe is in flight. -func concludeRemoteProbe(c *EventCollector, id common.DispatcherID) { - v, ok := c.dispatcherMap.Load(id) - if !ok { - return - } - v.(*dispatcherStat).startRemoteProbing(nil) -} - func TestProcessMessage(t *testing.T) { ctx := context.Background() node := node.NewInfo("127.0.0.1:18300", "") @@ -227,7 +214,6 @@ func TestProcessMessage(t *testing.T) { } } c.AddDispatcher(d, util.GetOrZero(config.GetDefaultReplicaConfig().MemoryQuota)) - concludeRemoteProbe(c, did) ch <- newMessage(node.ID, &readyEvent) ch <- newMessage(node.ID, &handshakeEvent) @@ -454,7 +440,6 @@ func TestEventCollectorBatchByCount(t *testing.T) { return false } c.AddDispatcher(d, util.GetOrZero(config.GetDefaultReplicaConfig().MemoryQuota)) - concludeRemoteProbe(c, did) from := localServerID readyEvent := commonEvent.NewReadyEvent(did) @@ -544,7 +529,6 @@ func TestEventCollectorBatchByBytes(t *testing.T) { } } c.AddDispatcher(d, util.GetOrZero(config.GetDefaultReplicaConfig().MemoryQuota)) - concludeRemoteProbe(c, did) from := localServerID readyEvent := commonEvent.NewReadyEvent(did) diff --git a/logservice/eventstore/event_store.go b/logservice/eventstore/event_store.go index b253e2a7f5..5ba2eb303a 100644 --- a/logservice/eventstore/event_store.go +++ b/logservice/eventstore/event_store.go @@ -97,6 +97,12 @@ type EventStore interface { // DispatcherCount returns the number of dispatchers currently reading from this event store. DispatcherCount() int + // GetSubscriptionWrittenResolvedTs returns the watermark of data that has + // actually been persisted for the dispatcher's subscription. It may be lower + // than the puller's resolved ts because of the write queue, and is what the + // ready event should report so it never promises data the scan cannot serve. + GetSubscriptionWrittenResolvedTs(dispatcherID common.DispatcherID) uint64 + // GetIterator returns an iterator for the requested range and resume cursor. GetIterator(dispatcherID common.DispatcherID, request ScanRequest) (EventIterator, error) @@ -197,6 +203,12 @@ type subscriptionStat struct { lastReceiveDMLTime atomic.Int64 // the resolveTs persisted in the store resolvedTs atomic.Uint64 + // writtenResolvedTs is the watermark of data that has actually been + // persisted to the store (written by the write path). It may lag resolvedTs + // (the puller's enqueue watermark) by the write queue, and it is the value + // the ready event should report so it never promises data the scan cannot + // serve yet. + writtenResolvedTs atomic.Uint64 // the max commit ts of dml event in the store maxEventCommitTs atomic.Uint64 } @@ -358,6 +370,40 @@ func newWriteTaskPool(store *eventStore, db *pebble.DB, index int, ch *chann.Unl } } +// advanceWrittenResolvedTs advances the written resolved ts of every +// subscription whose events were just persisted to pebble. The ready event and +// the scan range are driven by this watermark (via GetSubscriptionWrittenResolvedTs) +// so they never promise data that has not actually been written yet. +func (p *writeTaskPool) advanceWrittenResolvedTs(events []eventWithCallback) { + type subWriteKey struct { + tableKey tableStatsKey + subID logpuller.SubscriptionID + } + written := make(map[subWriteKey]uint64, 8) + for _, e := range events { + key := subWriteKey{ + tableKey: tableStatsKey{keyspaceID: e.keyspaceID, tableID: e.tableID}, + subID: e.subID, + } + if e.currentResolvedTs > written[key] { + written[key] = e.currentResolvedTs + } + } + if len(written) == 0 { + return + } + p.store.dispatcherMeta.RLock() + for key, ts := range written { + subStats := p.store.dispatcherMeta.tableStats[key.tableKey] + subStat, ok := subStats[key.subID] + if !ok { + continue + } + util.CompareAndMonotonicIncrease(&subStat.writtenResolvedTs, ts) + } + p.store.dispatcherMeta.RUnlock() +} + func (p *writeTaskPool) run(ctx context.Context) { p.store.wg.Add(p.workerNum) for i := 0; i < p.workerNum; i++ { @@ -401,6 +447,8 @@ func (p *writeTaskPool) run(ctx context.Context) { events[idx].callback() } + p.advanceWrittenResolvedTs(events) + buffer = buffer[:0] } } @@ -642,6 +690,7 @@ func (e *eventStore) RegisterDispatcher( }) subStat.checkpointTs.Store(startTs) subStat.resolvedTs.Store(startTs) + subStat.writtenResolvedTs.Store(startTs) subStat.maxEventCommitTs.Store(0) subStat.lastLogLagTime.Store(0) if stat.subStat == nil { @@ -753,6 +802,16 @@ func (e *eventStore) DispatcherCount() int { return len(e.dispatcherMeta.dispatcherStats) } +func (e *eventStore) GetSubscriptionWrittenResolvedTs(dispatcherID common.DispatcherID) uint64 { + e.dispatcherMeta.RLock() + defer e.dispatcherMeta.RUnlock() + stat, ok := e.dispatcherMeta.dispatcherStats[dispatcherID] + if !ok || stat.subStat == nil { + return 0 + } + return stat.subStat.writtenResolvedTs.Load() +} + func (e *eventStore) UpdateDispatcherCheckpointTs( dispatcherID common.DispatcherID, checkpointTs uint64, diff --git a/pkg/eventservice/event_broker.go b/pkg/eventservice/event_broker.go index e2e077b5cb..1e891be9c0 100644 --- a/pkg/eventservice/event_broker.go +++ b/pkg/eventservice/event_broker.go @@ -595,7 +595,15 @@ func (c *eventBroker) checkAndSendReady(task scanTask) bool { return false } remoteID := node.ID(task.info.GetServerID()) - event := event.NewReadyEventWithResolvedTs(task.info.GetID(), task.receivedResolvedTs.Load()) + // Use the written (persisted) resolved ts for the ready event, not the + // puller's enqueue watermark: the scan can only serve data that has been + // written to the event store, so reporting the enqueue watermark would + // promise data that is not servable yet. + writtenResolvedTs := c.eventStore.GetSubscriptionWrittenResolvedTs(task.id) + if writtenResolvedTs == 0 { + writtenResolvedTs = task.receivedResolvedTs.Load() + } + event := event.NewReadyEventWithResolvedTs(task.info.GetID(), writtenResolvedTs) wrapEvent := newWrapReadyEvent(remoteID, event) c.getMessageCh(task.messageWorkerIndex, common.IsRedoMode(task.info.GetMode())) <- wrapEvent log.Debug("send ready event to dispatcher", diff --git a/pkg/eventservice/event_service_test.go b/pkg/eventservice/event_service_test.go index 56b4ca1aed..80bb1101d1 100644 --- a/pkg/eventservice/event_service_test.go +++ b/pkg/eventservice/event_service_test.go @@ -311,6 +311,10 @@ func (m *mockEventStore) DispatcherCount() int { return count } +func (m *mockEventStore) GetSubscriptionWrittenResolvedTs(dispatcherID common.DispatcherID) uint64 { + return 0 +} + func (m *mockEventStore) UnregisterDispatcher(changefeedID common.ChangeFeedID, dispatcherID common.DispatcherID) { span, ok := m.dispatcherMap.Load(dispatcherID) if ok { From 65508f719084751427448496d288e727a8c4cbd3 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Fri, 21 Aug 2026 08:20:28 +0000 Subject: [PATCH 18/18] update Signed-off-by: wk989898 --- .../eventcollector/dispatcher_session.go | 5 +---- .../eventcollector/dispatcher_stat_test.go | 10 +++++----- pkg/common/event/ready_event.go | 6 +----- pkg/common/event/ready_event_test.go | 12 ++++++------ pkg/eventservice/event_broker.go | 2 +- 5 files changed, 14 insertions(+), 21 deletions(-) diff --git a/downstreamadapter/eventcollector/dispatcher_session.go b/downstreamadapter/eventcollector/dispatcher_session.go index 754e8df15e..170f99bb28 100644 --- a/downstreamadapter/eventcollector/dispatcher_session.go +++ b/downstreamadapter/eventcollector/dispatcher_session.go @@ -168,10 +168,7 @@ func (d *dispatcherConnState) acceptReady( // dispatcher start ts, which would let a barely started local // subscription win the race; hold on the remote's reported progress // until data actually flows. - baseline := requiredCheckpointTs - if remoteDeliveredTs > baseline { - baseline = remoteDeliveredTs - } + baseline := max(requiredCheckpointTs, remoteDeliveredTs) if baseline <= dispatcherStartTs && d.remoteResolvedTs > baseline { baseline = d.remoteResolvedTs } diff --git a/downstreamadapter/eventcollector/dispatcher_stat_test.go b/downstreamadapter/eventcollector/dispatcher_stat_test.go index 5188baad1d..1fd6546d00 100644 --- a/downstreamadapter/eventcollector/dispatcher_stat_test.go +++ b/downstreamadapter/eventcollector/dispatcher_stat_test.go @@ -889,7 +889,7 @@ func TestHandleLocalReadyEventCleansUpRemoteRegistrations(t *testing.T) { stat := newDispatcherStat(mockDisp, mockEventCollector, nil) setSessionState(stat.session, remoteServerID, true, "") newLocalReadyEvent := func(resolvedTs uint64) dispatcher.DispatcherEvent { - ready := commonEvent.NewReadyEventWithResolvedTs(dispatcherID, resolvedTs) + ready := commonEvent.NewReadyEvent(dispatcherID, resolvedTs) return dispatcher.DispatcherEvent{ From: &localServerID, Event: &ready, @@ -952,7 +952,7 @@ func TestLocalReadyGatedByRemoteServedResolvedTs(t *testing.T) { setSessionState(stat.session, "", true, remoteServerID) // Accept a remote ready that reports the remote is already serving at 300. - remoteReady := commonEvent.NewReadyEventWithResolvedTs(dispatcherID, 300) + remoteReady := commonEvent.NewReadyEvent(dispatcherID, 300) stat.handleSignalEvent(dispatcher.DispatcherEvent{ From: &remoteServerID, Event: &remoteReady, @@ -965,7 +965,7 @@ func TestLocalReadyGatedByRemoteServedResolvedTs(t *testing.T) { require.Equal(t, uint64(300), sessionRemoteResolvedTs(stat.session)) newLocalReadyEvent := func(resolvedTs uint64) dispatcher.DispatcherEvent { - ready := commonEvent.NewReadyEventWithResolvedTs(dispatcherID, resolvedTs) + ready := commonEvent.NewReadyEvent(dispatcherID, resolvedTs) return dispatcher.DispatcherEvent{ From: &localServerID, Event: &ready, @@ -1010,7 +1010,7 @@ func TestLocalReadyGatedByRemoteDeliveredResolvedTs(t *testing.T) { stat := newDispatcherStat(mockDisp, mockEventCollector, nil) setSessionState(stat.session, "", true, remoteServerID) - remoteReady := commonEvent.NewReadyEventWithResolvedTs(dispatcherID, 300) + remoteReady := commonEvent.NewReadyEvent(dispatcherID, 300) stat.handleSignalEvent(dispatcher.DispatcherEvent{ From: &remoteServerID, Event: &remoteReady, @@ -1023,7 +1023,7 @@ func TestLocalReadyGatedByRemoteDeliveredResolvedTs(t *testing.T) { require.Equal(t, uint64(300), sessionRemoteResolvedTs(stat.session)) newLocalReadyEvent := func(resolvedTs uint64) dispatcher.DispatcherEvent { - ready := commonEvent.NewReadyEventWithResolvedTs(dispatcherID, resolvedTs) + ready := commonEvent.NewReadyEvent(dispatcherID, resolvedTs) return dispatcher.DispatcherEvent{ From: &localServerID, Event: &ready, diff --git a/pkg/common/event/ready_event.go b/pkg/common/event/ready_event.go index efa019a9f5..a87f8a06ad 100644 --- a/pkg/common/event/ready_event.go +++ b/pkg/common/event/ready_event.go @@ -35,11 +35,7 @@ type ReadyEvent struct { ResolvedTs uint64 } -func NewReadyEvent(dispatcherID common.DispatcherID) ReadyEvent { - return NewReadyEventWithResolvedTs(dispatcherID, 0) -} - -func NewReadyEventWithResolvedTs(dispatcherID common.DispatcherID, resolvedTs uint64) ReadyEvent { +func NewReadyEvent(dispatcherID common.DispatcherID, resolvedTs uint64) ReadyEvent { return ReadyEvent{ Version: ReadyEventVersion1, DispatcherID: dispatcherID, diff --git a/pkg/common/event/ready_event_test.go b/pkg/common/event/ready_event_test.go index 0dd871e6b8..8ae7e3061a 100644 --- a/pkg/common/event/ready_event_test.go +++ b/pkg/common/event/ready_event_test.go @@ -23,7 +23,7 @@ import ( func TestReadyEvent(t *testing.T) { did := common.NewDispatcherID() - e := NewReadyEventWithResolvedTs(did, 123) + e := NewReadyEvent(did, 123) data, err := e.Marshal() require.NoError(t, err) require.Len(t, data, int(e.GetSize())+int(GetEventHeaderSize())) @@ -38,7 +38,7 @@ func TestReadyEvent(t *testing.T) { func TestReadyEventMethods(t *testing.T) { did := common.NewDispatcherID() - e := NewReadyEventWithResolvedTs(did, 123) + e := NewReadyEvent(did, 123) // Test GetType require.Equal(t, TypeReadyEvent, e.GetType()) @@ -66,7 +66,7 @@ func TestReadyEventMethods(t *testing.T) { } func TestReadyEventMarshalUnmarshal(t *testing.T) { - normalEvent := NewReadyEventWithResolvedTs(common.NewDispatcherID(), 123) + normalEvent := NewReadyEvent(common.NewDispatcherID(), 123) testCases := []struct { name string event *ReadyEvent @@ -127,7 +127,7 @@ func TestReadyEventDecodeLegacyPayload(t *testing.T) { func TestReadyEventLegacyDecoderIgnoresResolvedTs(t *testing.T) { did := common.NewDispatcherID() - event := NewReadyEventWithResolvedTs(did, 123) + event := NewReadyEvent(did, 123) data, err := event.Marshal() require.NoError(t, err) @@ -146,7 +146,7 @@ func TestReadyEventLegacyDecoderIgnoresResolvedTs(t *testing.T) { // TestReadyEventHeader verifies the unified header format func TestReadyEventHeader(t *testing.T) { did := common.NewDispatcherID() - e := NewReadyEvent(did) + e := NewReadyEvent(did, 123) data, err := e.Marshal() require.NoError(t, err) @@ -229,7 +229,7 @@ func TestReadyEventUnmarshalErrors(t *testing.T) { // TestReadyEventSize verifies GetSize calculation func TestReadyEventSize(t *testing.T) { did := common.NewDispatcherID() - e := NewReadyEvent(did) + e := NewReadyEvent(did, 123) // GetSize should only return business data size, not including header expectedSize := int64(did.GetSize() + 8) diff --git a/pkg/eventservice/event_broker.go b/pkg/eventservice/event_broker.go index 1e891be9c0..97e74ef72a 100644 --- a/pkg/eventservice/event_broker.go +++ b/pkg/eventservice/event_broker.go @@ -603,7 +603,7 @@ func (c *eventBroker) checkAndSendReady(task scanTask) bool { if writtenResolvedTs == 0 { writtenResolvedTs = task.receivedResolvedTs.Load() } - event := event.NewReadyEventWithResolvedTs(task.info.GetID(), writtenResolvedTs) + event := event.NewReadyEvent(task.info.GetID(), writtenResolvedTs) wrapEvent := newWrapReadyEvent(remoteID, event) c.getMessageCh(task.messageWorkerIndex, common.IsRedoMode(task.info.GetMode())) <- wrapEvent log.Debug("send ready event to dispatcher",