diff --git a/coordinator/controller_drain.go b/coordinator/controller_drain.go index 86f5889f17..1de4facd83 100644 --- a/coordinator/controller_drain.go +++ b/coordinator/controller_drain.go @@ -225,6 +225,7 @@ func (c *Controller) DrainNode(ctx context.Context, target node.ID) (int, error) zap.Int("dispatcherCountOnTarget", observation.dispatcherCountOnTarget), zap.Int("targetInflightDrainMoveCount", observation.targetInflightDrainMoveCount), zap.Int("pendingStatusCount", observation.pendingStatusCount), + zap.Int("logServiceDispatcherCount", observation.logServiceDispatcherCount), zap.Int("remaining", observation.remaining)) return ensureDrainRemainingNonZero(observation.remaining), nil } @@ -259,6 +260,7 @@ func (c *Controller) observeRemovedActiveDrainTarget(target node.ID, epoch uint6 zap.Int("dispatcherCountOnTarget", observation.dispatcherCountOnTarget), zap.Int("targetInflightDrainMoveCount", observation.targetInflightDrainMoveCount), zap.Int("pendingStatusCount", observation.pendingStatusCount), + zap.Int("logServiceDispatcherCount", observation.logServiceDispatcherCount), zap.Int("remaining", observation.remaining)) return ensureDrainRemainingNonZero(observation.remaining) } @@ -310,10 +312,11 @@ type drainNodeObservation struct { // pendingStatusCount is the number of running changefeeds not converged to the active target epoch. pendingStatusCount int // remaining is the max of all workload dimensions used by drain completion gating. - remaining int - nodeState drain.State - drainingObserved bool - stoppingObserved bool + remaining int + nodeState drain.State + drainingObserved bool + stoppingObserved bool + logServiceDispatcherCount int } func (c *Controller) observeDrainNode(target node.ID, epoch uint64) drainNodeObservation { @@ -323,12 +326,14 @@ func (c *Controller) observeDrainNode(target node.ID, epoch uint64) drainNodeObs } observation.dispatcherCountOnTarget, observation.targetInflightDrainMoveCount = c.aggregateDrainTargetProgress(target, epoch) observation.pendingStatusCount = c.collectDrainPendingStatus(target, epoch) + observation.logServiceDispatcherCount = c.drainController.GetLogServiceDispatcherCount(target) observation.remaining = drainRemainingEstimate( observation.maintainersOnTarget, observation.inflightOpsInvolvingTarget, observation.dispatcherCountOnTarget, observation.targetInflightDrainMoveCount, observation.pendingStatusCount, + observation.logServiceDispatcherCount, ) _, observation.drainingObserved, observation.stoppingObserved = c.drainController.GetStatus(target) @@ -1096,6 +1101,7 @@ func drainRemainingEstimate( dispatcherCountOnTarget int, targetInflightDrainMoveCount int, pendingStatusCount int, + logServiceDispatcherCount int, ) int { return max( maintainersOnTarget, @@ -1103,6 +1109,7 @@ func drainRemainingEstimate( dispatcherCountOnTarget, targetInflightDrainMoveCount, pendingStatusCount, + logServiceDispatcherCount, ) } diff --git a/coordinator/controller_drain_test.go b/coordinator/controller_drain_test.go index cd72ad7ca6..a7bc008a8e 100644 --- a/coordinator/controller_drain_test.go +++ b/coordinator/controller_drain_test.go @@ -154,6 +154,54 @@ func TestDrainNodeCompletesAfterCompletionObserved(t *testing.T) { require.Equal(t, epoch, c.drainSession.epoch) } +func TestDrainNodeUsesDefaultLogServiceDispatcherCount(t *testing.T) { + c, drainController, target := newDrainTestController(t) + setDrainProtocolVersion(c, target, heartbeatpb.CurrentDrainProtocolVersion) + cf := addRunningChangefeed(c, "cf1", node.ID("other"), 100) + + remaining, err := c.DrainNode(context.Background(), target) + require.NoError(t, err) + require.Equal(t, 1, remaining) + + _, epoch, ok := c.getDispatcherDrainTarget() + require.True(t, ok) + setChangefeedDrainStatus(cf, target, epoch, 0, 0) + + // Old nodes do not report the log service dispatcher count. Its protobuf + // default is zero, so the missing field must not block drain completion. + drainController.ObserveSetNodeLivenessResponse(target, &heartbeatpb.SetNodeLivenessResponse{ + Applied: heartbeatpb.NodeLiveness_STOPPING, + NodeEpoch: 1, + }) + remaining, err = c.DrainNode(context.Background(), target) + require.NoError(t, err) + require.Equal(t, 0, remaining) +} + +func TestDrainNodeWaitsForReportedLogServiceDispatchers(t *testing.T) { + c, drainController, target := newDrainTestController(t) + setDrainProtocolVersion(c, target, heartbeatpb.CurrentDrainProtocolVersion) + cf := addRunningChangefeed(c, "cf1", node.ID("other"), 100) + + remaining, err := c.DrainNode(context.Background(), target) + require.NoError(t, err) + require.Equal(t, 1, remaining) + + _, epoch, ok := c.getDispatcherDrainTarget() + require.True(t, ok) + setChangefeedDrainStatus(cf, target, epoch, 0, 0) + + setTargetStoppingHeartbeat(drainController, target, 2) + remaining, err = c.DrainNode(context.Background(), target) + require.NoError(t, err) + require.Equal(t, 2, remaining) + + setTargetStoppingHeartbeat(drainController, target, 0) + remaining, err = c.DrainNode(context.Background(), target) + require.NoError(t, err) + require.Equal(t, 0, remaining) +} + func TestDrainNodeDispatcherCountBlocksCompletion(t *testing.T) { c, drainController, target := newDrainTestController(t) setDrainProtocolVersion(c, target, heartbeatpb.CurrentDrainProtocolVersion) @@ -1171,11 +1219,19 @@ func setTargetStoppingObserved( drainController *drain.Controller, target node.ID, ) { - resp := &heartbeatpb.SetNodeLivenessResponse{ - Applied: heartbeatpb.NodeLiveness_STOPPING, - NodeEpoch: 1, - } - drainController.ObserveSetNodeLivenessResponse(target, resp) + setTargetStoppingHeartbeat(drainController, target, 0) +} + +func setTargetStoppingHeartbeat( + drainController *drain.Controller, + target node.ID, + logServiceDispatcherCount uint32, +) { + drainController.ObserveHeartbeat(target, &heartbeatpb.NodeHeartbeat{ + Liveness: heartbeatpb.NodeLiveness_STOPPING, + NodeEpoch: 1, + LogServiceDispatcherCount: logServiceDispatcherCount, + }) } func drainMessageChannel(ch chan *messaging.TargetMessage) { diff --git a/coordinator/drain/controller.go b/coordinator/drain/controller.go index 81b0b900ee..997aa24ce5 100644 --- a/coordinator/drain/controller.go +++ b/coordinator/drain/controller.go @@ -64,6 +64,10 @@ type nodeState struct { lastSeen time.Time nodeEpoch uint64 liveness heartbeatpb.NodeLiveness + + // logServiceDispatcherCount is valid only after a STOPPING heartbeat has + // been observed for the current node epoch. + logServiceDispatcherCount int } type drainTargetSchedulerGate struct { @@ -185,6 +189,10 @@ func (c *Controller) ObserveHeartbeat(nodeID node.ID, hb *heartbeatpb.NodeHeartb c.mu.Lock() defer c.mu.Unlock() c.observeLivenessLocked(nodeID, hb.NodeEpoch, hb.Liveness) + st := c.ensureNodeStateLocked(nodeID) + if hb.NodeEpoch == st.nodeEpoch && hb.Liveness == heartbeatpb.NodeLiveness_STOPPING { + st.logServiceDispatcherCount = int(hb.GetLogServiceDispatcherCount()) + } c.observeTargetSchedulerAckLocked(nodeID, hb) } @@ -415,6 +423,20 @@ func (c *Controller) GetStatus(nodeID node.ID) (drainRequested, drainingObserved return st.drainRequested, st.drainingObserved, st.stoppingObserved } +// GetLogServiceDispatcherCount returns the dispatcher count reported by a +// STOPPING heartbeat for the current node epoch. Nodes that do not report the +// field use the protobuf default value zero. +func (c *Controller) GetLogServiceDispatcherCount(nodeID node.ID) int { + c.mu.Lock() + defer c.mu.Unlock() + + st, ok := c.nodes[nodeID] + if !ok { + return 0 + } + return st.logServiceDispatcherCount +} + // GetDrainProtocolVersion returns the bootstrap-observed drain capability for a node. func (c *Controller) GetDrainProtocolVersion(nodeID node.ID) (uint32, bool) { c.mu.Lock() diff --git a/coordinator/drain/controller_test.go b/coordinator/drain/controller_test.go index d02e009693..47c5b9bb58 100644 --- a/coordinator/drain/controller_test.go +++ b/coordinator/drain/controller_test.go @@ -126,6 +126,38 @@ func TestDrainControllerResetObservedStateForNewEpoch(t *testing.T) { c.mu.Unlock() } +func TestDrainControllerTracksStoppingLogServiceDispatcherCountByEpoch(t *testing.T) { + c := NewController(messaging.NewMockMessageCenter()) + target := node.ID("n1") + + c.ObserveSetNodeLivenessResponse(target, &heartbeatpb.SetNodeLivenessResponse{ + Applied: heartbeatpb.NodeLiveness_STOPPING, + NodeEpoch: 42, + }) + require.Zero(t, c.GetLogServiceDispatcherCount(target)) + + c.ObserveHeartbeat(target, &heartbeatpb.NodeHeartbeat{ + Liveness: heartbeatpb.NodeLiveness_STOPPING, + NodeEpoch: 42, + LogServiceDispatcherCount: 2, + }) + require.Equal(t, 2, c.GetLogServiceDispatcherCount(target)) + + c.ObserveHeartbeat(target, &heartbeatpb.NodeHeartbeat{ + Liveness: heartbeatpb.NodeLiveness_ALIVE, + NodeEpoch: 43, + }) + require.Zero(t, c.GetLogServiceDispatcherCount(target)) + + // A delayed heartbeat from the old process must not satisfy the new epoch. + c.ObserveHeartbeat(target, &heartbeatpb.NodeHeartbeat{ + Liveness: heartbeatpb.NodeLiveness_STOPPING, + NodeEpoch: 42, + LogServiceDispatcherCount: 0, + }) + require.Zero(t, c.GetLogServiceDispatcherCount(target)) +} + func TestDrainControllerSkipStoppingForNewEpochWithoutDraining(t *testing.T) { mc := messaging.NewMockMessageCenter() c := NewController(mc) diff --git a/downstreamadapter/eventcollector/dispatcher_session.go b/downstreamadapter/eventcollector/dispatcher_session.go index ec1a600cfa..170f99bb28 100644 --- a/downstreamadapter/eventcollector/dispatcher_session.go +++ b/downstreamadapter/eventcollector/dispatcher_session.go @@ -48,8 +48,11 @@ import ( // pendingRemoteEventServiceID="" // // This means local registration and remote probing can overlap. A remote service -// may serve data first, but a later local ready still moves the dispatcher back -// to local and cleans up remote registrations. +// may serve data first (its ready usually arrives before the local ready because +// the local subscription has to initialize first), but after the local service +// catches up to the progress the remote reported (and the dispatcher +// checkpoint), a later local ready moves the dispatcher back to local and +// cleans up remote registrations. type dispatcherConnState struct { sync.RWMutex // removed marks the session as terminal after removal starts. New @@ -60,7 +63,8 @@ type dispatcherConnState struct { currentEventServiceID node.ID // localReadyPending means the local register request has been sent but local // ready has not been accepted. It may be true while a remote service is - // already current; local ready wins when it arrives later. + // already current; local ready wins after its resolved ts covers the current + // dispatcher checkpoint. localReadyPending bool // pendingRemoteEventServiceID is the remote EventService currently being // probed for reuse. It waits for either ready or not reusable, and only one @@ -69,6 +73,14 @@ type dispatcherConnState struct { // remoteCandidates are the remaining remote EventServices to probe after the // current pending remote reports not reusable. remoteCandidates []string + // remoteResolvedTs is the resolved ts reported by the remote EventService + // currently serving. It is only meaningful while currentEventServiceID is a + // remote service and is cleared when the local service wins back. The local + // ready must cover it before taking over: the sink checkpoint alone is not a + // safe baseline before the remote delivers its first events because it still + // equals the dispatcher start ts, which would let a barely started local + // subscription win the race and stall the table while catching up from TiKV. + remoteResolvedTs uint64 } // Registration state transitions. @@ -124,11 +136,19 @@ type readyDecision struct { // acceptReady enforces the ready acceptance rules: // 1. once local is already serving, any later remote ready is stale and should // only trigger cleanup; -// 2. local ready can be accepted while local registration is still pending, and -// local wins over any remote that started serving earlier; +// 2. local ready can be accepted while local registration is still pending; +// when a remote is already serving, the local resolved ts must cover both +// the current dispatcher checkpoint and the progress the remote reported; // 3. remote ready is accepted only from the single remote candidate currently // being probed. -func (d *dispatcherConnState) acceptReady(from node.ID, localServerID node.ID) readyDecision { +func (d *dispatcherConnState) acceptReady( + from node.ID, + localServerID node.ID, + readyResolvedTs uint64, + requiredCheckpointTs uint64, + remoteDeliveredTs uint64, + dispatcherStartTs uint64, +) readyDecision { d.Lock() defer d.Unlock() if d.removed { @@ -139,6 +159,23 @@ func (d *dispatcherConnState) acceptReady(from node.ID, localServerID node.ID) r if !d.localReadyPending { return readyDecision{} } + if !d.currentEventServiceID.IsEmpty() && + d.currentEventServiceID != localServerID { + // The local event service may only win back once its resolved ts + // covers the position the current remote has actually delivered to + // the sink, so the switch never stalls the table. Before the remote + // delivers its first events the sink position still equals the + // dispatcher start ts, which would let a barely started local + // subscription win the race; hold on the remote's reported progress + // until data actually flows. + baseline := max(requiredCheckpointTs, remoteDeliveredTs) + if baseline <= dispatcherStartTs && d.remoteResolvedTs > baseline { + baseline = d.remoteResolvedTs + } + if readyResolvedTs < baseline { + return readyDecision{} + } + } decision := readyDecision{ commitTarget: localServerID, @@ -151,6 +188,7 @@ func (d *dispatcherConnState) acceptReady(from node.ID, localServerID node.ID) r d.localReadyPending = false d.pendingRemoteEventServiceID = "" d.remoteCandidates = nil + d.remoteResolvedTs = 0 return decision } @@ -170,10 +208,14 @@ func (d *dispatcherConnState) acceptReady(from node.ID, localServerID node.ID) r } d.currentEventServiceID = from - // Keep localReadyPending unchanged: local ready may still arrive later and - // move the dispatcher back to local. + // Keep localReadyPending unchanged: after the local service catches up, a + // later local ready may move the dispatcher back to local. d.pendingRemoteEventServiceID = "" d.remoteCandidates = nil + // Record the remote progress the local service must catch up to before it + // may win back. This is more meaningful than the sink checkpoint, which + // still equals the start ts before the remote delivers its first events. + d.remoteResolvedTs = readyResolvedTs return readyDecision{ commitTarget: from, } @@ -194,6 +236,7 @@ func (d *dispatcherConnState) beginRemove(localServerID node.ID) ([]node.ID, boo d.localReadyPending = false d.pendingRemoteEventServiceID = "" d.remoteCandidates = nil + d.remoteResolvedTs = 0 return []node.ID(targets), false } @@ -440,7 +483,7 @@ func (s *dispatcherSession) handleSignalEvent(event dispatcher.DispatcherEvent) from := *event.From switch event.GetType() { case commonEvent.TypeReadyEvent: - s.handleReadyEvent(from) + s.handleReadyEvent(from, uint64(event.GetCommitTs())) case commonEvent.TypeNotReusableEvent: if from == s.localServerID { log.Panic("should not happen: local event service should not send not reusable event") @@ -459,12 +502,25 @@ func (s *dispatcherSession) handleSignalEvent(event dispatcher.DispatcherEvent) // handleReadyEvent applies the ready decision produced by connState: clean up // any stale registrations, then commit whichever target won the ready race. -func (s *dispatcherSession) handleReadyEvent(from node.ID) { +func (s *dispatcherSession) handleReadyEvent(from node.ID, readyResolvedTs uint64) { s.requestMu.Lock() defer s.requestMu.Unlock() // connState decides whether this ready should be accepted and which stale // registrations must be cleaned up. Session only applies the side effects. - accepted := s.connState.acceptReady(from, s.localServerID) + requiredCheckpointTs := s.target.GetCheckpointTs() + // remoteDeliveredTs is the highest resolved ts the sink has received from + // the current event service, i.e. the position the remote has actually + // delivered. The local service must catch up to it before winning back so + // the switch never stalls the table while the local catches up from TiKV. + remoteDeliveredTs := s.target.GetResolvedTs() + accepted := s.connState.acceptReady( + from, + s.localServerID, + readyResolvedTs, + requiredCheckpointTs, + remoteDeliveredTs, + s.target.GetStartTs(), + ) for _, target := range accepted.cleanupTargets { s.removeFromLocked(target) } @@ -472,13 +528,13 @@ func (s *dispatcherSession) handleReadyEvent(from node.ID) { return } if accepted.commitTarget == s.localServerID { - s.handleAcceptedLocalReadyLocked() + s.handleAcceptedLocalReadyLocked(requiredCheckpointTs) return } s.handleAcceptedRemoteReadyLocked(accepted.commitTarget) } -func (s *dispatcherSession) handleAcceptedLocalReadyLocked() { +func (s *dispatcherSession) handleAcceptedLocalReadyLocked(resetTs uint64) { if s.readyCallback != nil { // This path is used during the initial add flow before the dispatcher is // committed. Local is still authoritative, so any speculative remote @@ -491,7 +547,7 @@ func (s *dispatcherSession) handleAcceptedLocalReadyLocked() { log.Info("received ready signal from local event service, prepare to reset the dispatcher", zap.Stringer("changefeedID", s.target.GetChangefeedID()), zap.Stringer("dispatcher", s.target.GetId())) - s.doResetLocked(s.localServerID, s.target.GetCheckpointTs()) + s.doResetLocked(s.localServerID, resetTs) } func (s *dispatcherSession) handleAcceptedRemoteReadyLocked(serverID node.ID) { diff --git a/downstreamadapter/eventcollector/dispatcher_session_test_helper_test.go b/downstreamadapter/eventcollector/dispatcher_session_test_helper_test.go index 53bc40c314..d5148e805b 100644 --- a/downstreamadapter/eventcollector/dispatcher_session_test_helper_test.go +++ b/downstreamadapter/eventcollector/dispatcher_session_test_helper_test.go @@ -34,6 +34,12 @@ func setSessionRemoteCandidates(session *dispatcherSession, nodes []string) { session.connState.remoteCandidates = nodes } +func setSessionRemoteResolvedTs(session *dispatcherSession, resolvedTs uint64) { + session.connState.Lock() + defer session.connState.Unlock() + session.connState.remoteResolvedTs = resolvedTs +} + func setSessionReadyCallback(session *dispatcherSession, readyCallback func()) { session.readyCallback = readyCallback } @@ -57,3 +63,9 @@ func sessionState(session *dispatcherSession) (node.ID, bool, node.ID) { defer session.connState.RUnlock() return session.connState.currentEventServiceID, session.connState.localReadyPending, session.connState.pendingRemoteEventServiceID } + +func sessionRemoteResolvedTs(session *dispatcherSession) uint64 { + session.connState.RLock() + defer session.connState.RUnlock() + return session.connState.remoteResolvedTs +} diff --git a/downstreamadapter/eventcollector/dispatcher_stat_test.go b/downstreamadapter/eventcollector/dispatcher_stat_test.go index 76ec14c4f1..1fd6546d00 100644 --- a/downstreamadapter/eventcollector/dispatcher_stat_test.go +++ b/downstreamadapter/eventcollector/dispatcher_stat_test.go @@ -47,6 +47,7 @@ type mockDispatcher struct { handleError func(err error) events []dispatcher.DispatcherEvent checkPointTs uint64 + resolvedTs uint64 tableSpan *heartbeatpb.TableSpan lowLatencyMode bool @@ -128,7 +129,10 @@ func (m *mockDispatcher) GetSkipSyncpointAtStartTs() bool { } func (m *mockDispatcher) GetResolvedTs() uint64 { - return m.startTs + if m.resolvedTs == 0 { + return m.startTs + } + return m.resolvedTs } func (m *mockDispatcher) GetCheckpointTs() uint64 { @@ -636,7 +640,7 @@ func TestHandleSignalEvent(t *testing.T) { expectedPendingRemoteTarget: "", }, { - name: "handle ready event from local server without callback", + name: "handle ready event from local server while remote probe is in flight", event: dispatcher.DispatcherEvent{ From: &localServerID, Event: &mockEvent{ @@ -646,6 +650,8 @@ func TestHandleSignalEvent(t *testing.T) { initialState: func(stat *dispatcherStat) { setSessionState(stat.session, "", true, remoteServerID) }, + // The local ready wins immediately because no remote is serving yet; + // the remote only wins if its ready arrives first. expectedEventServiceID: localServerID, expectedReceivingData: true, expectedAwaitingLocalReady: false, @@ -812,7 +818,7 @@ func TestHandleLocalReadyEventCleansUpRemoteRegistrations(t *testing.T) { } } - t.Run("local ready removes pending remote register and resets local", func(t *testing.T) { + t.Run("local ready wins immediately while remote probe is in flight", func(t *testing.T) { mockDisp := newMockDispatcher(dispatcherID, 0) mockEventCollector := newTestEventCollector(localServerID) stat := newDispatcherStat(mockDisp, mockEventCollector, nil) @@ -820,6 +826,10 @@ func TestHandleLocalReadyEventCleansUpRemoteRegistrations(t *testing.T) { stat.handleSignalEvent(newReadyEvent(localServerID)) + currentEventServiceID, localReadyPending, pendingRemoteTarget := sessionState(stat.session) + require.Equal(t, localServerID, currentEventServiceID) + require.False(t, localReadyPending) + require.Empty(t, pendingRemoteTarget) requireDispatcherRequests( t, readDispatcherRequests(t, mockEventCollector, 2), @@ -829,6 +839,31 @@ func TestHandleLocalReadyEventCleansUpRemoteRegistrations(t *testing.T) { requireNoDispatcherRequest(t, mockEventCollector) }) + t.Run("remote ready after local won is stale and only triggers cleanup", func(t *testing.T) { + mockDisp := newMockDispatcher(dispatcherID, 0) + mockEventCollector := newTestEventCollector(localServerID) + stat := newDispatcherStat(mockDisp, mockEventCollector, nil) + setSessionState(stat.session, "", true, remoteServerID) + + stat.handleSignalEvent(newReadyEvent(localServerID)) + requireDispatcherRequests( + t, + readDispatcherRequests(t, mockEventCollector, 2), + dispatcherRequestRecord{to: remoteServerID, action: eventpb.ActionType_ACTION_TYPE_REMOVE}, + dispatcherRequestRecord{to: localServerID, action: eventpb.ActionType_ACTION_TYPE_RESET}, + ) + + // A late remote ready is stale: the dispatcher is already on local, so + // it only triggers a cleanup of the stale remote registration. + stat.handleSignalEvent(newReadyEvent(remoteServerID)) + requireDispatcherRequests( + t, + readDispatcherRequests(t, mockEventCollector, 1), + dispatcherRequestRecord{to: remoteServerID, action: eventpb.ActionType_ACTION_TYPE_REMOVE}, + ) + requireNoDispatcherRequest(t, mockEventCollector) + }) + t.Run("local ready removes current remote and pending remote without duplicates", func(t *testing.T) { mockDisp := newMockDispatcher(dispatcherID, 0) mockEventCollector := newTestEventCollector(localServerID) @@ -847,15 +882,50 @@ func TestHandleLocalReadyEventCleansUpRemoteRegistrations(t *testing.T) { requireNoDispatcherRequest(t, mockEventCollector) }) - t.Run("local ready with callback still removes speculative remote register", func(t *testing.T) { + t.Run("local ready waits until its resolved ts covers the current checkpoint", func(t *testing.T) { + mockDisp := newMockDispatcher(dispatcherID, 100) + mockDisp.checkPointTs = 200 + mockEventCollector := newTestEventCollector(localServerID) + stat := newDispatcherStat(mockDisp, mockEventCollector, nil) + setSessionState(stat.session, remoteServerID, true, "") + newLocalReadyEvent := func(resolvedTs uint64) dispatcher.DispatcherEvent { + ready := commonEvent.NewReadyEvent(dispatcherID, resolvedTs) + return dispatcher.DispatcherEvent{ + From: &localServerID, + Event: &ready, + } + } + + stat.handleSignalEvent(newLocalReadyEvent(199)) + + currentEventServiceID, localReadyPending, pendingRemoteTarget := sessionState(stat.session) + require.Equal(t, remoteServerID, currentEventServiceID) + require.True(t, localReadyPending) + require.Empty(t, pendingRemoteTarget) + requireNoDispatcherRequest(t, mockEventCollector) + + stat.handleSignalEvent(newLocalReadyEvent(200)) + + requireDispatcherRequests( + t, + readDispatcherRequests(t, mockEventCollector, 2), + dispatcherRequestRecord{to: remoteServerID, action: eventpb.ActionType_ACTION_TYPE_REMOVE}, + dispatcherRequestRecord{to: localServerID, action: eventpb.ActionType_ACTION_TYPE_RESET}, + ) + requireNoDispatcherRequest(t, mockEventCollector) + }) + + t.Run("local ready with callback fires immediately while remote probe is in flight", func(t *testing.T) { mockDisp := newMockDispatcher(dispatcherID, 0) mockEventCollector := newTestEventCollector(localServerID) stat := newDispatcherStat(mockDisp, mockEventCollector, nil) setSessionState(stat.session, "", true, remoteServerID) - setSessionReadyCallback(stat.session, func() {}) + callbackFired := false + setSessionReadyCallback(stat.session, func() { callbackFired = true }) stat.handleSignalEvent(newReadyEvent(localServerID)) + require.True(t, callbackFired) requireDispatcherRequests( t, readDispatcherRequests(t, mockEventCollector, 1), @@ -865,6 +935,120 @@ func TestHandleLocalReadyEventCleansUpRemoteRegistrations(t *testing.T) { }) } +func TestLocalReadyGatedByRemoteServedResolvedTs(t *testing.T) { + localServerID := node.ID("local-server") + remoteServerID := node.ID("remote-server") + dispatcherID := common.NewDispatcherID() + + // The remote has been accepted but has not delivered its first events yet, + // so the sink checkpoint and resolved ts both still equal the dispatcher + // start ts. The local ready must still cover the resolved ts the remote + // reported, otherwise the fresh local subscription would win the race and + // stall the table while catching up from TiKV. + mockDisp := newMockDispatcher(dispatcherID, 100) + mockDisp.checkPointTs = 100 + mockEventCollector := newTestEventCollector(localServerID) + stat := newDispatcherStat(mockDisp, mockEventCollector, nil) + setSessionState(stat.session, "", true, remoteServerID) + + // Accept a remote ready that reports the remote is already serving at 300. + remoteReady := commonEvent.NewReadyEvent(dispatcherID, 300) + stat.handleSignalEvent(dispatcher.DispatcherEvent{ + From: &remoteServerID, + Event: &remoteReady, + }) + requireDispatcherRequests( + t, + readDispatcherRequests(t, mockEventCollector, 1), + dispatcherRequestRecord{to: remoteServerID, action: eventpb.ActionType_ACTION_TYPE_RESET}, + ) + require.Equal(t, uint64(300), sessionRemoteResolvedTs(stat.session)) + + newLocalReadyEvent := func(resolvedTs uint64) dispatcher.DispatcherEvent { + ready := commonEvent.NewReadyEvent(dispatcherID, resolvedTs) + return dispatcher.DispatcherEvent{ + From: &localServerID, + Event: &ready, + } + } + + // Local ready above the sink checkpoint but below the remote progress is + // still rejected, otherwise the fresh local subscription would stall the + // table while catching up from TiKV. + stat.handleSignalEvent(newLocalReadyEvent(200)) + currentEventServiceID, localReadyPending, pendingRemoteTarget := sessionState(stat.session) + require.Equal(t, remoteServerID, currentEventServiceID) + require.True(t, localReadyPending) + require.Empty(t, pendingRemoteTarget) + requireNoDispatcherRequest(t, mockEventCollector) + + // Once local catches up to the remote progress it wins back and clears it. + stat.handleSignalEvent(newLocalReadyEvent(300)) + requireDispatcherRequests( + t, + readDispatcherRequests(t, mockEventCollector, 2), + dispatcherRequestRecord{to: remoteServerID, action: eventpb.ActionType_ACTION_TYPE_REMOVE}, + dispatcherRequestRecord{to: localServerID, action: eventpb.ActionType_ACTION_TYPE_RESET}, + ) + require.Zero(t, sessionRemoteResolvedTs(stat.session)) + requireNoDispatcherRequest(t, mockEventCollector) +} + +func TestLocalReadyGatedByRemoteDeliveredResolvedTs(t *testing.T) { + localServerID := node.ID("local-server") + remoteServerID := node.ID("remote-server") + dispatcherID := common.NewDispatcherID() + + // Once the remote delivers data the sink resolved ts reflects the position + // it actually delivered (250), which is the catch-up target. The remote's + // reported frontier (300) must not inflate the baseline, otherwise the + // local would wait on the remote's pull progress it does not need. + mockDisp := newMockDispatcher(dispatcherID, 100) + mockDisp.checkPointTs = 100 + mockDisp.resolvedTs = 250 + mockEventCollector := newTestEventCollector(localServerID) + stat := newDispatcherStat(mockDisp, mockEventCollector, nil) + setSessionState(stat.session, "", true, remoteServerID) + + remoteReady := commonEvent.NewReadyEvent(dispatcherID, 300) + stat.handleSignalEvent(dispatcher.DispatcherEvent{ + From: &remoteServerID, + Event: &remoteReady, + }) + requireDispatcherRequests( + t, + readDispatcherRequests(t, mockEventCollector, 1), + dispatcherRequestRecord{to: remoteServerID, action: eventpb.ActionType_ACTION_TYPE_RESET}, + ) + require.Equal(t, uint64(300), sessionRemoteResolvedTs(stat.session)) + + newLocalReadyEvent := func(resolvedTs uint64) dispatcher.DispatcherEvent { + ready := commonEvent.NewReadyEvent(dispatcherID, resolvedTs) + return dispatcher.DispatcherEvent{ + From: &localServerID, + Event: &ready, + } + } + + // The local must cover what the remote delivered (250), not its frontier. + stat.handleSignalEvent(newLocalReadyEvent(249)) + currentEventServiceID, localReadyPending, pendingRemoteTarget := sessionState(stat.session) + require.Equal(t, remoteServerID, currentEventServiceID) + require.True(t, localReadyPending) + require.Empty(t, pendingRemoteTarget) + requireNoDispatcherRequest(t, mockEventCollector) + + stat.handleSignalEvent(newLocalReadyEvent(250)) + requireDispatcherRequests( + t, + readDispatcherRequests(t, mockEventCollector, 2), + dispatcherRequestRecord{to: remoteServerID, action: eventpb.ActionType_ACTION_TYPE_REMOVE}, + dispatcherRequestRecord{to: localServerID, action: eventpb.ActionType_ACTION_TYPE_RESET}, + ) + require.Zero(t, sessionRemoteResolvedTs(stat.session)) + requireNoDispatcherRequest(t, mockEventCollector) +} + func TestInitialLocalReadyCallbackIsOneShot(t *testing.T) { localServerID := node.ID("local-server") dispatcherID := common.NewDispatcherID() diff --git a/downstreamadapter/eventcollector/event_collector_test.go b/downstreamadapter/eventcollector/event_collector_test.go index 1d29b8568e..0fa5c75273 100644 --- a/downstreamadapter/eventcollector/event_collector_test.go +++ b/downstreamadapter/eventcollector/event_collector_test.go @@ -272,11 +272,8 @@ func TestRemoveLastDispatcher(t *testing.T) { } func TestGroupHeartbeatUsesEpochAndClamp(t *testing.T) { - ctx := context.Background() serverInfo := node.NewInfo("127.0.0.1:18300", "") - mc := messaging.NewMessageCenter(ctx, serverInfo.ID, config.NewDefaultMessageCenterConfig(serverInfo.AdvertiseAddr), nil) - mc.Run(ctx) - defer mc.Close() + mc := messaging.NewMockMessageCenter() appcontext.SetService(appcontext.MessageCenter, mc) c := New(serverInfo.ID) diff --git a/heartbeatpb/heartbeat.pb.go b/heartbeatpb/heartbeat.pb.go index 94ff2f969d..02015a4434 100644 --- a/heartbeatpb/heartbeat.pb.go +++ b/heartbeatpb/heartbeat.pb.go @@ -1625,6 +1625,9 @@ type NodeHeartbeat struct { // currently applied on this node. Empty target means the drain target is clear. DispatcherDrainTargetNodeId string `protobuf:"bytes,3,opt,name=dispatcher_drain_target_node_id,json=dispatcherDrainTargetNodeId,proto3" json:"dispatcher_drain_target_node_id,omitempty"` DispatcherDrainTargetEpoch uint64 `protobuf:"varint,4,opt,name=dispatcher_drain_target_epoch,json=dispatcherDrainTargetEpoch,proto3" json:"dispatcher_drain_target_epoch,omitempty"` + // log_service_dispatcher_count is the number of dispatchers currently + // reading from this node's event store. + LogServiceDispatcherCount uint32 `protobuf:"varint,5,opt,name=log_service_dispatcher_count,json=logServiceDispatcherCount,proto3" json:"log_service_dispatcher_count,omitempty"` } func (m *NodeHeartbeat) Reset() { *m = NodeHeartbeat{} } @@ -1688,6 +1691,13 @@ func (m *NodeHeartbeat) GetDispatcherDrainTargetEpoch() uint64 { return 0 } +func (m *NodeHeartbeat) GetLogServiceDispatcherCount() uint32 { + if m != nil { + return m.LogServiceDispatcherCount + } + return 0 +} + // SetNodeLivenessRequest asks a node to transition its local liveness. type SetNodeLivenessRequest struct { Target NodeLiveness `protobuf:"varint,1,opt,name=target,proto3,enum=heartbeatpb.NodeLiveness" json:"target,omitempty"` @@ -4006,196 +4016,198 @@ func init() { func init() { proto.RegisterFile("heartbeatpb/heartbeat.proto", fileDescriptor_6d584080fdadb670) } var fileDescriptor_6d584080fdadb670 = []byte{ - // 3020 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x1a, 0x4d, 0x8f, 0x1c, 0x47, - 0xd5, 0xdd, 0xf3, 0xfd, 0x76, 0x67, 0xb7, 0x5d, 0xb6, 0xd7, 0x6b, 0x7b, 0xbd, 0xde, 0x74, 0x02, - 0xda, 0x4c, 0x82, 0x8d, 0x9d, 0x98, 0x8f, 0x10, 0x12, 0xc6, 0x33, 0x9b, 0x78, 0xe4, 0x9d, 0xdd, - 0x55, 0xcd, 0x26, 0x46, 0xe1, 0x30, 0xf4, 0x76, 0x97, 0x67, 0x3b, 0x3b, 0xd3, 0x35, 0xe9, 0xee, - 0xf1, 0xda, 0x96, 0x00, 0x45, 0x88, 0x1b, 0x07, 0xb8, 0x71, 0x20, 0x17, 0x4e, 0x9c, 0x10, 0x7f, - 0x00, 0xc1, 0x81, 0x03, 0x27, 0x14, 0x71, 0x40, 0x39, 0x41, 0x94, 0xdc, 0x11, 0x12, 0x12, 0x5c, - 0x51, 0x7d, 0x74, 0x77, 0xf5, 0x4c, 0xcf, 0x7e, 0xb0, 0xa3, 0x88, 0xd3, 0xf4, 0x7b, 0xf5, 0xde, - 0xab, 0x57, 0xaf, 0x5e, 0xbd, 0x7a, 0xef, 0xd5, 0xc0, 0xb5, 0x7d, 0x62, 0xf9, 0xe1, 0x1e, 0xb1, - 0xc2, 0xe1, 0xde, 0xad, 0xf8, 0xfb, 0xe6, 0xd0, 0xa7, 0x21, 0x45, 0x73, 0xca, 0xa0, 0xf9, 0x14, - 0x2a, 0xbb, 0xd6, 0x5e, 0x9f, 0x74, 0x86, 0x96, 0x87, 0x96, 0xa1, 0xc4, 0x81, 0x56, 0x73, 0x59, - 0x5b, 0xd3, 0xd6, 0x73, 0x38, 0x02, 0xd1, 0x55, 0x28, 0x77, 0x42, 0xcb, 0x0f, 0x1f, 0x90, 0xa7, - 0xcb, 0xfa, 0x9a, 0xb6, 0x3e, 0x8f, 0x63, 0x18, 0x2d, 0x41, 0x71, 0xc3, 0x73, 0xd8, 0x48, 0x8e, - 0x8f, 0x48, 0x08, 0xad, 0x02, 0x3c, 0x20, 0x4f, 0x83, 0xa1, 0x65, 0x33, 0x81, 0xf9, 0x35, 0x6d, - 0xbd, 0x8a, 0x15, 0x8c, 0xf9, 0x57, 0x1d, 0x8c, 0xfb, 0x4c, 0x95, 0x7b, 0xc4, 0x0a, 0x31, 0xf9, - 0x60, 0x44, 0x82, 0x10, 0x7d, 0x1b, 0xe6, 0xed, 0x7d, 0xcb, 0xeb, 0x91, 0x47, 0x84, 0x38, 0x52, - 0x8f, 0xb9, 0x3b, 0x57, 0x6e, 0x2a, 0x3a, 0xdf, 0x6c, 0x28, 0x04, 0x38, 0x45, 0x8e, 0x5e, 0x85, - 0xca, 0xa1, 0x15, 0x12, 0x7f, 0x60, 0xf9, 0x07, 0x5c, 0xd1, 0xb9, 0x3b, 0x4b, 0x29, 0xde, 0x87, - 0xd1, 0x28, 0x4e, 0x08, 0xd1, 0xeb, 0x50, 0xf5, 0x89, 0x43, 0xe3, 0x31, 0xbe, 0x90, 0xe9, 0x9c, - 0x69, 0x62, 0xf4, 0x0d, 0x28, 0x07, 0xa1, 0x15, 0x8e, 0x02, 0x12, 0x2c, 0xe7, 0xd7, 0x72, 0xeb, - 0x73, 0x77, 0x56, 0x52, 0x8c, 0xb1, 0x7d, 0x3b, 0x9c, 0x0a, 0xc7, 0xd4, 0x68, 0x1d, 0x16, 0x6d, - 0x3a, 0x18, 0x92, 0x3e, 0x09, 0x89, 0x18, 0x5c, 0x2e, 0xac, 0x69, 0xeb, 0x65, 0x3c, 0x8e, 0x46, - 0x2f, 0x41, 0x8e, 0xf8, 0xfe, 0x72, 0x31, 0xc3, 0x1a, 0x78, 0xe4, 0x79, 0xae, 0xd7, 0xdb, 0xf0, - 0x7d, 0xea, 0x63, 0x46, 0x65, 0xfe, 0x44, 0x83, 0x4a, 0xa2, 0x9e, 0xc9, 0x2c, 0x4a, 0xec, 0x83, - 0x21, 0x75, 0xbd, 0x70, 0x37, 0xe0, 0x16, 0xcd, 0xe3, 0x14, 0x8e, 0x6d, 0x95, 0x4f, 0x02, 0xda, - 0x7f, 0x4c, 0x9c, 0xdd, 0x80, 0xdb, 0x2d, 0x8f, 0x15, 0x0c, 0x32, 0x20, 0x17, 0x90, 0x0f, 0xb8, - 0x59, 0xf2, 0x98, 0x7d, 0x32, 0xa9, 0x7d, 0x2b, 0x08, 0x3b, 0x4f, 0x3d, 0x9b, 0xf3, 0xe4, 0x85, - 0x54, 0x15, 0x67, 0xfe, 0x00, 0x8c, 0xa6, 0x1b, 0x0c, 0xad, 0xd0, 0xde, 0x27, 0x7e, 0xdd, 0x0e, - 0x5d, 0xea, 0xa1, 0x97, 0xa0, 0x68, 0xf1, 0x2f, 0xae, 0xc7, 0xc2, 0x9d, 0x0b, 0xa9, 0xb5, 0x08, - 0x22, 0x2c, 0x49, 0x98, 0xd7, 0x35, 0xe8, 0x60, 0xe0, 0x86, 0xb1, 0x52, 0x31, 0x8c, 0xd6, 0x60, - 0xae, 0x15, 0xb0, 0xa9, 0x76, 0xd8, 0x1a, 0xb8, 0x6a, 0x65, 0xac, 0xa2, 0xcc, 0x06, 0xe4, 0xea, - 0x8d, 0x07, 0x29, 0x21, 0xda, 0xd1, 0x42, 0xf4, 0x49, 0x21, 0x18, 0x50, 0xab, 0xe7, 0x51, 0x9f, - 0x38, 0xf7, 0xfa, 0xd4, 0x3e, 0x90, 0xdb, 0x71, 0x36, 0x99, 0x3f, 0xd6, 0xe1, 0x52, 0xcb, 0x7b, - 0xd4, 0x1f, 0x11, 0x66, 0xa8, 0xc4, 0x44, 0x01, 0xfa, 0x0e, 0x54, 0xe3, 0x81, 0xdd, 0xa7, 0x43, - 0x22, 0x8d, 0x74, 0x35, 0x65, 0xa4, 0x14, 0x05, 0x4e, 0x33, 0xa0, 0x37, 0xa1, 0x9a, 0x08, 0x6c, - 0x35, 0x99, 0xdd, 0x72, 0x13, 0x2e, 0xa3, 0x52, 0xe0, 0x34, 0x3d, 0x3f, 0xe9, 0xf6, 0x3e, 0x19, - 0x58, 0xad, 0x26, 0x37, 0x6a, 0x0e, 0xc7, 0x30, 0x7a, 0x00, 0x17, 0xc8, 0x13, 0xbb, 0x3f, 0x72, - 0x88, 0xc2, 0xe3, 0xf0, 0xbd, 0x3f, 0x72, 0x8a, 0x2c, 0x2e, 0xf3, 0x17, 0xba, 0xea, 0x1e, 0xd2, - 0xb0, 0xdf, 0x85, 0x4b, 0x6e, 0x96, 0x65, 0x64, 0x1c, 0x30, 0xb3, 0x0d, 0xa1, 0x52, 0xe2, 0x6c, - 0x01, 0xe8, 0x6e, 0xec, 0x78, 0x22, 0x2c, 0x5c, 0x9f, 0xa2, 0xee, 0x98, 0x0b, 0x9a, 0x90, 0xb3, - 0xec, 0x28, 0x20, 0x18, 0x69, 0x67, 0x6d, 0x3c, 0xc0, 0x6c, 0x10, 0x6d, 0x03, 0x72, 0x27, 0x7c, - 0x44, 0x5a, 0xe5, 0x46, 0x5a, 0xe3, 0x09, 0x32, 0x9c, 0xc1, 0x6a, 0x7e, 0xaa, 0xc1, 0x79, 0x25, - 0x32, 0x06, 0x43, 0xea, 0x05, 0xe4, 0xac, 0xa1, 0xb1, 0x0d, 0xc8, 0x19, 0x33, 0x37, 0x89, 0xdc, - 0x63, 0x9a, 0x31, 0x22, 0x1d, 0x27, 0x19, 0x11, 0x82, 0xfc, 0x80, 0x3a, 0x44, 0xfa, 0x08, 0xff, - 0x46, 0x2f, 0x82, 0x31, 0xb0, 0x5c, 0x2f, 0xb4, 0x5c, 0x8f, 0xf8, 0x5d, 0x32, 0xa4, 0xf6, 0xbe, - 0x0c, 0x0c, 0x8b, 0x09, 0x7e, 0x83, 0xa1, 0xcd, 0x27, 0x70, 0xa1, 0xa1, 0x44, 0xa0, 0x36, 0x09, - 0x02, 0xab, 0x77, 0xe6, 0x35, 0x8e, 0xc7, 0x3a, 0x7d, 0x32, 0xd6, 0x99, 0xbf, 0xd7, 0x60, 0x11, - 0x13, 0x87, 0xb6, 0x49, 0x68, 0xcd, 0x68, 0xda, 0xe3, 0xc2, 0xe7, 0xb8, 0x5a, 0xb9, 0x8c, 0x10, - 0x7c, 0x0a, 0xdb, 0xfd, 0x10, 0xae, 0xb3, 0x05, 0xe0, 0x78, 0x82, 0x1d, 0x9f, 0xf6, 0x7c, 0x12, - 0x04, 0x5f, 0xcc, 0x72, 0xcc, 0x5f, 0x6b, 0xb0, 0x92, 0x56, 0xe0, 0x2d, 0xea, 0x1f, 0x5a, 0xbe, - 0xf3, 0x05, 0x99, 0x33, 0xcb, 0x54, 0xb9, 0x6c, 0x53, 0xfd, 0x4b, 0x53, 0x83, 0x4c, 0x83, 0x7a, - 0x8f, 0xdc, 0x1e, 0xaa, 0x41, 0x3e, 0x18, 0x5a, 0x9e, 0x54, 0x6b, 0x29, 0xfb, 0xb2, 0xc6, 0x9c, - 0x86, 0xa5, 0x44, 0x01, 0x4b, 0x74, 0x62, 0x45, 0x22, 0x90, 0x2d, 0xd2, 0x51, 0x82, 0x9c, 0x0c, - 0x11, 0x47, 0x44, 0xc1, 0x14, 0x39, 0x8b, 0xb3, 0x41, 0x14, 0x67, 0xf3, 0x22, 0xce, 0x46, 0x70, - 0x7c, 0xb6, 0x0a, 0xca, 0xd9, 0xaa, 0x81, 0x11, 0x1c, 0xb8, 0xc3, 0x66, 0x7b, 0xb3, 0x1e, 0x74, - 0xa4, 0x46, 0x45, 0x7e, 0xb7, 0x4c, 0xe0, 0xcd, 0x3f, 0xe8, 0x70, 0x85, 0x05, 0x6d, 0x67, 0xd4, - 0x57, 0x62, 0xee, 0x8c, 0x52, 0xac, 0xbb, 0x50, 0xb4, 0xb9, 0x1d, 0x8f, 0x09, 0xa4, 0xc2, 0xd8, - 0x58, 0x12, 0xa3, 0x06, 0x2c, 0x04, 0x52, 0x25, 0x11, 0x62, 0xb9, 0xc1, 0x16, 0xee, 0x5c, 0x4b, - 0xb1, 0x77, 0x52, 0x24, 0x78, 0x8c, 0x85, 0xa9, 0x4e, 0x87, 0xc4, 0xb7, 0x42, 0xea, 0xf3, 0xeb, - 0x31, 0xcf, 0x45, 0xa4, 0x55, 0xdf, 0x56, 0x08, 0x70, 0x8a, 0x3c, 0xd3, 0x71, 0x0a, 0xd9, 0x8e, - 0xf3, 0x2b, 0x1d, 0x96, 0xda, 0xc4, 0xef, 0xcd, 0xde, 0x7e, 0x6f, 0x42, 0xd5, 0x39, 0xe5, 0x0d, - 0x9d, 0xa2, 0x47, 0x2d, 0x40, 0x03, 0xa6, 0x99, 0xd3, 0x3c, 0x95, 0xfb, 0x65, 0x30, 0xc5, 0x8e, - 0x96, 0x3f, 0x26, 0x88, 0x4f, 0x31, 0xd2, 0x0e, 0x5c, 0x68, 0xc7, 0xa8, 0xfb, 0xd1, 0xc4, 0xe8, - 0x9b, 0x4a, 0x42, 0xac, 0x65, 0xdc, 0x2f, 0x09, 0xcf, 0x78, 0x46, 0x6c, 0x7e, 0xa2, 0x41, 0xb5, - 0xe9, 0x5b, 0xae, 0x17, 0x85, 0x34, 0xf4, 0x02, 0x2c, 0x84, 0x96, 0xdf, 0x23, 0x61, 0xd7, 0xa3, - 0x0e, 0xe9, 0xba, 0x0e, 0xb7, 0x77, 0x05, 0xcf, 0x0b, 0xec, 0x16, 0x75, 0x48, 0xcb, 0x41, 0xcf, - 0x81, 0x84, 0xa5, 0xc2, 0xe2, 0xac, 0xce, 0x09, 0x1c, 0x57, 0x16, 0x7d, 0x0d, 0x2e, 0x4b, 0x92, - 0xc4, 0x9c, 0x5d, 0x9b, 0x8e, 0x64, 0xf2, 0x58, 0xc5, 0x97, 0xc4, 0xb0, 0xea, 0xc1, 0x23, 0x2f, - 0x44, 0x6f, 0xc1, 0x9a, 0xe4, 0x63, 0x89, 0x85, 0xdb, 0xdb, 0x0f, 0xbb, 0x0e, 0xd3, 0xb0, 0x3b, - 0xa0, 0x8f, 0x89, 0x14, 0x20, 0x8a, 0x9b, 0x15, 0x41, 0xd7, 0x92, 0x64, 0x7c, 0x1d, 0x6d, 0xfa, - 0x98, 0x70, 0x39, 0xe6, 0x6f, 0x72, 0x60, 0x8c, 0xaf, 0xfc, 0xac, 0xbe, 0x74, 0x1d, 0x80, 0x7d, - 0x75, 0x99, 0xfd, 0x08, 0x5f, 0x74, 0x05, 0x57, 0x18, 0x86, 0x89, 0x27, 0xe8, 0x36, 0x14, 0xc4, - 0x48, 0xd6, 0x51, 0x6b, 0xd0, 0xc1, 0x90, 0x7a, 0xc4, 0x0b, 0x39, 0x2d, 0x16, 0x94, 0xe8, 0x79, - 0xa8, 0x26, 0xd7, 0x52, 0x37, 0x8c, 0x13, 0xfb, 0xd4, 0x5d, 0x25, 0xab, 0x91, 0x42, 0x86, 0xe3, - 0x4e, 0x54, 0x23, 0xe8, 0x4b, 0xb0, 0xb0, 0x47, 0x69, 0x18, 0x84, 0xbe, 0x35, 0xec, 0x3a, 0xd4, - 0x23, 0x32, 0x6c, 0x55, 0x63, 0x6c, 0x93, 0x7a, 0x64, 0xa2, 0xa0, 0x28, 0x4d, 0x16, 0x14, 0xa8, - 0x0e, 0x0b, 0xc2, 0xf4, 0x43, 0xe9, 0x1d, 0xcb, 0x65, 0x6e, 0xaf, 0x74, 0x7e, 0x9c, 0xf2, 0x1f, - 0x5c, 0x75, 0x52, 0xee, 0x94, 0xe5, 0xdd, 0x95, 0x6c, 0xef, 0xfe, 0x87, 0x06, 0x55, 0xe6, 0x5e, - 0x89, 0x63, 0xdf, 0x85, 0x72, 0xdf, 0x7d, 0x4c, 0x3c, 0x36, 0xb3, 0x96, 0x11, 0x7a, 0x18, 0xf5, - 0xa6, 0x24, 0xc0, 0x31, 0x29, 0xdb, 0x25, 0xee, 0xbb, 0xaa, 0x6b, 0x56, 0x18, 0x46, 0x38, 0x66, - 0x13, 0x6e, 0x28, 0x1e, 0x29, 0x16, 0x38, 0xe6, 0xf2, 0x39, 0xbe, 0xb3, 0xd7, 0x12, 0x32, 0xbe, - 0xc6, 0x5d, 0xf5, 0x04, 0xd4, 0xe1, 0xfa, 0x34, 0x29, 0x6a, 0x32, 0x71, 0x35, 0x53, 0x86, 0x58, - 0xf0, 0xfb, 0xb0, 0xd4, 0x11, 0xf2, 0xe2, 0x45, 0xc8, 0x90, 0x77, 0x1b, 0x8a, 0x42, 0xd6, 0xf1, - 0xcb, 0x96, 0x84, 0xc7, 0x2c, 0xda, 0x1c, 0xc0, 0xe5, 0x89, 0xb9, 0x64, 0x9e, 0xfb, 0x0a, 0x94, - 0xac, 0xe1, 0xb0, 0xef, 0x12, 0xe7, 0xf8, 0xd9, 0x22, 0xca, 0xe3, 0xa6, 0x7b, 0x1f, 0x6e, 0x74, - 0xd4, 0xa3, 0xad, 0xac, 0x3d, 0x5a, 0xe3, 0xac, 0x02, 0x8d, 0xf9, 0x75, 0xb8, 0xd6, 0xa0, 0xd4, - 0x77, 0x5c, 0x8f, 0x5d, 0x3c, 0xf7, 0x22, 0x2f, 0x8f, 0xe6, 0x59, 0x86, 0xd2, 0x63, 0xe2, 0x07, - 0x51, 0x09, 0x9c, 0xc3, 0x11, 0xc8, 0x2a, 0xa2, 0x95, 0x6c, 0x4e, 0x69, 0x99, 0xff, 0x3d, 0xb0, - 0xa2, 0x57, 0x61, 0x29, 0x3e, 0x3a, 0x21, 0xb5, 0x69, 0xbf, 0x1b, 0x29, 0xa1, 0xf3, 0xd8, 0x75, - 0x31, 0x3a, 0x26, 0x7c, 0xf0, 0x5d, 0x31, 0xf6, 0xff, 0xe3, 0x9a, 0xff, 0xd6, 0xe0, 0x62, 0xdd, - 0x71, 0x92, 0x05, 0x46, 0xd6, 0x7c, 0x11, 0x74, 0xb9, 0x53, 0x47, 0x86, 0x4d, 0xdd, 0x75, 0xd0, - 0x52, 0x2a, 0x71, 0x99, 0x8f, 0x33, 0x93, 0x89, 0x90, 0x97, 0x95, 0x9e, 0xd7, 0xe0, 0xbc, 0x1b, - 0x74, 0x3d, 0x72, 0xd8, 0x4d, 0x02, 0x30, 0xd7, 0xbb, 0x8c, 0x17, 0xdd, 0x60, 0x8b, 0x1c, 0x26, - 0xd3, 0xa1, 0x1b, 0x30, 0x77, 0x20, 0xdb, 0x5c, 0xcc, 0x42, 0x05, 0xd1, 0xf9, 0x8a, 0x50, 0x2d, - 0x27, 0x33, 0x08, 0x15, 0xb3, 0x83, 0xd0, 0x1f, 0x35, 0xb8, 0x8c, 0x09, 0xbb, 0x6a, 0xce, 0xb4, - 0xf6, 0x65, 0x28, 0xd9, 0x56, 0x60, 0x5b, 0x0e, 0x91, 0x0d, 0x89, 0x08, 0x64, 0x23, 0x3e, 0x97, - 0xef, 0xc8, 0x1e, 0x4a, 0x04, 0x8e, 0x2f, 0x23, 0x7f, 0xa2, 0x65, 0x4c, 0xc9, 0x14, 0xfe, 0x9c, - 0x83, 0xab, 0xc9, 0x02, 0x26, 0xce, 0xc4, 0x19, 0xaf, 0xc1, 0x69, 0x3b, 0x7b, 0x85, 0x9f, 0x17, - 0x5f, 0xd9, 0xd4, 0x38, 0x7b, 0xb7, 0xe1, 0xb9, 0x90, 0xa5, 0xfa, 0xdd, 0xd0, 0x77, 0x7b, 0x3d, - 0xa6, 0xfe, 0x63, 0xe2, 0xa5, 0x52, 0x03, 0xf7, 0x04, 0x8d, 0x8d, 0xeb, 0x5c, 0xc6, 0xae, 0x10, - 0xb1, 0xc1, 0x24, 0xa8, 0x2d, 0x8e, 0x6c, 0xa7, 0x29, 0x64, 0x3b, 0x8d, 0xc5, 0xd2, 0x0c, 0x55, - 0x21, 0x9f, 0x38, 0x74, 0x4c, 0x9f, 0xe2, 0x71, 0xfa, 0xac, 0xa8, 0xfa, 0xb0, 0x12, 0x2d, 0xa5, - 0xce, 0xd8, 0x86, 0x96, 0x4e, 0xb4, 0xa1, 0xe5, 0xec, 0x0d, 0xfd, 0x4b, 0x0e, 0xae, 0x65, 0x6e, - 0xe8, 0x6c, 0x9a, 0x15, 0x77, 0xa1, 0xc0, 0xca, 0xaf, 0x28, 0x39, 0x4e, 0x77, 0x51, 0xe2, 0xd9, - 0x92, 0x62, 0x4d, 0x50, 0x47, 0x89, 0x49, 0xee, 0x24, 0x6d, 0xd2, 0x93, 0xa5, 0x3a, 0x2f, 0x03, - 0xe2, 0x1b, 0x91, 0xa6, 0x14, 0x5e, 0x6e, 0xb0, 0x11, 0xb5, 0x8b, 0x81, 0x9a, 0x50, 0x89, 0x0a, - 0x0e, 0x56, 0x9d, 0x31, 0xd5, 0xbf, 0x9c, 0x59, 0xdf, 0x4c, 0x54, 0x15, 0x38, 0x61, 0xcc, 0xdc, - 0x86, 0x52, 0xe6, 0x36, 0xa0, 0x4d, 0x58, 0xe4, 0x69, 0x7d, 0x37, 0x99, 0xb6, 0xcc, 0xa7, 0x7d, - 0x3e, 0x7d, 0x31, 0x64, 0x56, 0x32, 0x78, 0x81, 0xf3, 0x46, 0x05, 0x53, 0x60, 0xfe, 0x4d, 0x87, - 0xd5, 0x64, 0x53, 0x77, 0x68, 0x10, 0xce, 0xfa, 0xa4, 0x9e, 0xe8, 0xd8, 0xe9, 0x67, 0x3c, 0x76, - 0xb7, 0xa1, 0x24, 0x4a, 0x69, 0x76, 0xea, 0x99, 0x31, 0x2e, 0x4f, 0xec, 0xc1, 0xc0, 0x6a, 0x79, - 0x8f, 0x28, 0x8e, 0xe8, 0xd0, 0x6b, 0x30, 0xcf, 0xb7, 0x39, 0xe2, 0xcb, 0x1f, 0xcd, 0x37, 0xc7, - 0x88, 0x3b, 0x92, 0xf7, 0x14, 0x61, 0xf0, 0x23, 0x1d, 0x6e, 0x4c, 0x35, 0xf0, 0x6c, 0x4e, 0xce, - 0x17, 0x62, 0xe1, 0x53, 0x9d, 0xb3, 0x53, 0x75, 0x05, 0x21, 0xb1, 0x72, 0xaa, 0x15, 0xad, 0x8d, - 0xb5, 0xa2, 0x57, 0x23, 0xca, 0x2d, 0x6b, 0x10, 0x55, 0x3e, 0x0a, 0x06, 0xdd, 0x84, 0x22, 0x8f, - 0x0e, 0x91, 0x0b, 0x64, 0x74, 0x79, 0xf8, 0x4e, 0x4a, 0x2a, 0xb3, 0x21, 0xdf, 0xc1, 0xf8, 0xc4, - 0xd3, 0xdf, 0xc1, 0x56, 0x24, 0x99, 0x32, 0x6b, 0x82, 0x30, 0x7f, 0xa7, 0x03, 0x9a, 0x0c, 0x4e, - 0xec, 0x9e, 0x9e, 0xb2, 0x8f, 0x29, 0x9b, 0xeb, 0xf2, 0x9d, 0x2d, 0x5a, 0xb2, 0x3e, 0xb6, 0xe4, - 0xa8, 0x6d, 0x95, 0x3b, 0x41, 0xdb, 0xea, 0x2d, 0x30, 0xec, 0xa8, 0xbe, 0xeb, 0x06, 0x49, 0x43, - 0xfa, 0x98, 0x22, 0x70, 0xd1, 0x56, 0xe1, 0x51, 0x30, 0x19, 0x23, 0x0b, 0x19, 0x31, 0xf2, 0x15, - 0x98, 0xdb, 0xeb, 0x53, 0xfb, 0x40, 0x96, 0xa1, 0xe2, 0x96, 0x42, 0xe9, 0xb3, 0xc3, 0xc5, 0xc3, - 0x5e, 0xd4, 0xe4, 0x26, 0x71, 0xeb, 0xa1, 0x94, 0xb4, 0x1e, 0xcc, 0x5f, 0x6a, 0xb0, 0x94, 0x1c, - 0x8f, 0x46, 0x9f, 0x06, 0x64, 0x46, 0x71, 0x47, 0xc9, 0x72, 0xf4, 0x74, 0x96, 0x73, 0x8a, 0x66, - 0xe2, 0x47, 0x1a, 0x5c, 0x9e, 0x50, 0x6f, 0x36, 0xa7, 0x76, 0x19, 0x4a, 0xc1, 0xc8, 0xb6, 0x59, - 0x61, 0x29, 0xf5, 0x93, 0xe0, 0x69, 0xf4, 0xfb, 0xa9, 0x06, 0x46, 0xf2, 0x26, 0x22, 0x1c, 0x7b, - 0x06, 0x4f, 0x4a, 0x57, 0xa1, 0x2c, 0xdd, 0x5f, 0x5c, 0xc7, 0x39, 0x1c, 0xc3, 0x47, 0xbd, 0x16, - 0x99, 0xdf, 0x83, 0x02, 0xa7, 0x3b, 0xe6, 0x59, 0x79, 0x9a, 0xbb, 0xaf, 0x40, 0xa5, 0x33, 0xec, - 0xbb, 0x3c, 0x10, 0xc9, 0xd4, 0x34, 0x41, 0x98, 0x1f, 0xea, 0x70, 0x01, 0xd3, 0x51, 0x48, 0xb8, - 0xa8, 0xba, 0x33, 0x70, 0x03, 0x5e, 0xb1, 0xd4, 0xc0, 0xe8, 0xd0, 0x91, 0x6f, 0x13, 0x25, 0x3a, - 0x88, 0x3a, 0x6e, 0x02, 0x8f, 0xd6, 0x61, 0x51, 0xe0, 0xc6, 0x8f, 0xf4, 0x38, 0x9a, 0x49, 0x15, - 0xd5, 0x88, 0x22, 0x55, 0x14, 0x3e, 0x13, 0x78, 0x26, 0x55, 0xe0, 0x12, 0xa9, 0x79, 0x21, 0x75, - 0x0c, 0x8d, 0xde, 0x80, 0xa2, 0x6c, 0x85, 0x16, 0xf8, 0x9e, 0xa4, 0x53, 0x85, 0x8c, 0xd5, 0x45, - 0x6f, 0x53, 0xe2, 0xd7, 0xf4, 0x60, 0x21, 0xb2, 0x96, 0x70, 0xad, 0x23, 0x2c, 0xbd, 0x06, 0x73, - 0xdb, 0x7d, 0x67, 0xcc, 0xd8, 0x2a, 0x8a, 0x51, 0x6c, 0x91, 0xc3, 0xb1, 0xdd, 0x54, 0x51, 0xe6, - 0x7f, 0x72, 0x50, 0x10, 0x87, 0x77, 0x05, 0x2a, 0xad, 0x80, 0xbf, 0x58, 0xc9, 0x22, 0xbd, 0x8c, - 0x13, 0x04, 0xd3, 0x82, 0x7f, 0x26, 0x3d, 0x73, 0x09, 0xa2, 0x37, 0x61, 0x4e, 0x7c, 0x46, 0xa1, - 0x79, 0xb2, 0x81, 0x3c, 0xee, 0xc0, 0x58, 0xe5, 0x40, 0x0f, 0xe0, 0xfc, 0x16, 0x21, 0x4e, 0xd3, - 0xa7, 0xc3, 0x61, 0x44, 0x21, 0xd3, 0xf4, 0x63, 0xc4, 0x4c, 0xf2, 0xa1, 0xd7, 0x61, 0x91, 0x21, - 0xeb, 0x8e, 0x13, 0x8b, 0x12, 0x2d, 0x2d, 0x34, 0x19, 0x5b, 0xf1, 0x38, 0x29, 0x6a, 0xc0, 0xc2, - 0x3b, 0x43, 0xc7, 0x0a, 0x89, 0x34, 0x61, 0x94, 0xf0, 0x5d, 0xcb, 0x4a, 0x1a, 0xe4, 0x06, 0xe1, - 0x31, 0x96, 0xf1, 0xc7, 0xe2, 0xd2, 0xc4, 0x63, 0x31, 0xfa, 0x0a, 0xef, 0xe1, 0xf5, 0x08, 0x4f, - 0xc4, 0x17, 0xc6, 0x52, 0x92, 0xe8, 0xd1, 0xb0, 0x27, 0xfa, 0x77, 0x3d, 0x82, 0x76, 0xe1, 0x62, - 0x86, 0xe3, 0x04, 0xcb, 0x15, 0xae, 0xdb, 0xda, 0x71, 0x1e, 0x86, 0x33, 0xb9, 0xcd, 0x1f, 0xc1, - 0xc5, 0xf8, 0x86, 0x51, 0xdf, 0xc1, 0x4f, 0x71, 0xb3, 0xad, 0x47, 0xbd, 0x48, 0x7d, 0xea, 0xf5, - 0x20, 0x5b, 0x90, 0x19, 0x2f, 0x8b, 0xe6, 0x3f, 0x35, 0x76, 0xaa, 0x52, 0xff, 0xa3, 0x38, 0xcd, - 0xe4, 0x59, 0xd7, 0xa1, 0x3e, 0x8b, 0xeb, 0x30, 0xab, 0x55, 0x70, 0x1b, 0x2e, 0x89, 0x9c, 0x2b, - 0x70, 0x9f, 0x91, 0xee, 0x90, 0xf8, 0xdd, 0x80, 0xd8, 0xd4, 0x13, 0xe5, 0xa4, 0x8e, 0x11, 0x1f, - 0xec, 0xb8, 0xcf, 0xc8, 0x0e, 0xf1, 0x3b, 0x7c, 0x24, 0xeb, 0xc1, 0xc7, 0xfc, 0xad, 0x06, 0x48, - 0x7d, 0x28, 0x9e, 0xcd, 0x45, 0xf8, 0x36, 0x54, 0xf7, 0x12, 0xa1, 0xf1, 0x03, 0xf0, 0x73, 0xd9, - 0xd9, 0x84, 0x3a, 0x7f, 0x9a, 0x2f, 0x73, 0x97, 0x1c, 0x98, 0x57, 0xd3, 0x3f, 0x46, 0x13, 0xba, - 0x71, 0x00, 0xe6, 0xdf, 0x0c, 0xe7, 0x51, 0x27, 0x8a, 0xb4, 0xfc, 0x9b, 0xe1, 0xec, 0x48, 0x56, - 0x05, 0xf3, 0x6f, 0x16, 0x44, 0x06, 0xe2, 0x39, 0x51, 0x86, 0xcf, 0x08, 0x34, 0x5f, 0x85, 0xf9, - 0xf1, 0x47, 0x8c, 0x7d, 0xb7, 0xb7, 0x2f, 0xff, 0x88, 0xc1, 0xbf, 0x91, 0x01, 0xb9, 0x3e, 0x3d, - 0x94, 0xe1, 0x87, 0x7d, 0x32, 0xdd, 0x54, 0xb3, 0x9c, 0x8c, 0x8b, 0x6b, 0x9b, 0x04, 0x7b, 0xfe, - 0xcd, 0x2e, 0xad, 0xa8, 0x66, 0x96, 0xaa, 0xc5, 0xb0, 0xf9, 0x7d, 0xb8, 0xb1, 0x49, 0x7b, 0x4a, - 0x13, 0x2f, 0x79, 0x23, 0x9d, 0xcd, 0x06, 0x9a, 0x1f, 0x6a, 0xb0, 0x36, 0x7d, 0x8a, 0xd9, 0x64, - 0x23, 0xc7, 0x3d, 0x00, 0xf7, 0x99, 0x2d, 0x89, 0x7d, 0x10, 0x8c, 0x06, 0x6d, 0x12, 0x5a, 0xe8, - 0xab, 0xd1, 0xd9, 0xce, 0xca, 0x2d, 0x22, 0xca, 0xd4, 0x19, 0xaf, 0x81, 0x61, 0xab, 0xf8, 0x0e, - 0xf9, 0x40, 0xce, 0x33, 0x81, 0x37, 0x7f, 0xae, 0xc1, 0x25, 0xe5, 0x2f, 0x09, 0x24, 0x8c, 0x24, - 0xa2, 0x8b, 0x50, 0x10, 0xef, 0x2f, 0x62, 0x13, 0x05, 0xc0, 0x3c, 0xe7, 0x09, 0xf5, 0xef, 0xb3, - 0xcd, 0x95, 0xd7, 0x8f, 0x04, 0xd1, 0x12, 0x14, 0x9f, 0x50, 0x7f, 0x93, 0x1e, 0xca, 0x73, 0x2b, - 0x21, 0x91, 0x7d, 0x0d, 0x38, 0x47, 0x5e, 0xb6, 0x89, 0x04, 0xc8, 0x38, 0x82, 0xd1, 0x80, 0x71, - 0x88, 0xc4, 0x57, 0x42, 0x2c, 0x15, 0x5c, 0xcb, 0xd4, 0xa9, 0x6e, 0x1f, 0xcc, 0x6a, 0x17, 0x2e, - 0x42, 0x41, 0xed, 0x31, 0x0b, 0x20, 0xf3, 0x7f, 0x17, 0xf2, 0xef, 0x59, 0xf9, 0xf8, 0xef, 0x59, - 0xe6, 0xdf, 0x35, 0x30, 0x33, 0xf5, 0x13, 0xf7, 0xcf, 0x8c, 0x82, 0xc9, 0x19, 0x34, 0x44, 0x6f, - 0x40, 0x39, 0xda, 0x69, 0x6e, 0xdb, 0xf1, 0x3f, 0xf7, 0x64, 0x6a, 0x8f, 0x63, 0x9e, 0xda, 0xf5, - 0x28, 0x79, 0x42, 0x15, 0x28, 0x3c, 0xf4, 0xdd, 0x90, 0x18, 0xe7, 0x50, 0x19, 0xf2, 0x3b, 0x56, - 0x10, 0x18, 0x5a, 0x6d, 0x5d, 0xe4, 0x46, 0xca, 0xdb, 0x31, 0x40, 0xb1, 0xe1, 0x13, 0x8b, 0xd3, - 0x01, 0x14, 0x45, 0x53, 0xd5, 0xd0, 0x6a, 0x6d, 0x98, 0x57, 0x9f, 0x8c, 0x99, 0xb8, 0xed, 0x6e, - 0xdd, 0x71, 0x8c, 0x73, 0x68, 0x1e, 0xca, 0xdb, 0xdd, 0x88, 0x90, 0x31, 0x6d, 0x77, 0xdb, 0xec, - 0x5b, 0x47, 0x73, 0x50, 0xda, 0xee, 0xf2, 0x6c, 0xd4, 0xc8, 0x09, 0x80, 0xb7, 0x58, 0x8c, 0x7c, - 0xed, 0x2e, 0xcc, 0xab, 0x2f, 0x14, 0x4c, 0x5c, 0x7d, 0xb3, 0xf5, 0xee, 0x86, 0x10, 0xd7, 0xc4, - 0xf5, 0xd6, 0x56, 0x6b, 0xeb, 0x6d, 0x43, 0x63, 0x50, 0x67, 0x77, 0x7b, 0x67, 0x87, 0x41, 0x7a, - 0xed, 0x35, 0x80, 0xe4, 0x32, 0x67, 0xeb, 0xd8, 0xda, 0xde, 0x62, 0x3c, 0x73, 0x50, 0x7a, 0x58, - 0x6f, 0xed, 0x0a, 0x16, 0x06, 0x60, 0x01, 0xe8, 0x8c, 0xa6, 0xc9, 0x68, 0x72, 0xb5, 0x97, 0xc7, - 0x52, 0x7c, 0x54, 0x82, 0x5c, 0xbd, 0xdf, 0x37, 0xce, 0xa1, 0x22, 0xe8, 0xcd, 0x7b, 0x42, 0xf5, - 0x2d, 0xea, 0x0f, 0xac, 0xbe, 0xa1, 0xd7, 0xde, 0x86, 0x2b, 0x53, 0x53, 0x4b, 0xae, 0x6d, 0xb3, - 0xdd, 0xda, 0x15, 0x33, 0xe3, 0x8d, 0xcd, 0x8d, 0x7a, 0x67, 0xc3, 0xd0, 0x10, 0x82, 0x05, 0x09, - 0x74, 0x3b, 0x8d, 0xfb, 0x1b, 0xed, 0xba, 0xa1, 0xd7, 0x9e, 0xc1, 0x42, 0xfa, 0xbe, 0xe4, 0xfa, - 0x51, 0xff, 0xc0, 0xf5, 0x7a, 0x82, 0xbf, 0x13, 0xf2, 0x74, 0x4b, 0x68, 0x2e, 0xec, 0xe8, 0x18, - 0x3a, 0x32, 0x60, 0xbe, 0xe5, 0xb9, 0xa1, 0x6b, 0xf5, 0xdd, 0x67, 0x8c, 0x36, 0x87, 0xaa, 0x50, - 0xd9, 0xf1, 0xc9, 0xd0, 0xf2, 0x19, 0x98, 0x47, 0x0b, 0x00, 0xdc, 0x9c, 0x98, 0x58, 0xce, 0x53, - 0xa3, 0xc0, 0x18, 0x1e, 0x5a, 0x6e, 0xe8, 0x7a, 0x3d, 0x61, 0xe5, 0x62, 0xed, 0x5b, 0x50, 0x4d, - 0xc5, 0x15, 0x74, 0x1e, 0xaa, 0xef, 0x6c, 0xb5, 0xb6, 0x5a, 0xbb, 0xad, 0xfa, 0x66, 0xeb, 0xbd, - 0x8d, 0xa6, 0x30, 0x77, 0xbb, 0xd5, 0x69, 0xd7, 0x77, 0x1b, 0xf7, 0x0d, 0x8d, 0xad, 0x4c, 0x7c, - 0xea, 0xf7, 0xde, 0xf8, 0xd3, 0x67, 0xab, 0xda, 0xc7, 0x9f, 0xad, 0x6a, 0x9f, 0x7e, 0xb6, 0xaa, - 0xfd, 0xec, 0xf3, 0xd5, 0x73, 0x1f, 0x7f, 0xbe, 0x7a, 0xee, 0x93, 0xcf, 0x57, 0xcf, 0xbd, 0xf7, - 0x42, 0xcf, 0x0d, 0xf7, 0x47, 0x7b, 0x37, 0x6d, 0x3a, 0xb8, 0x35, 0x74, 0xbd, 0x9e, 0x6d, 0x0d, - 0x6f, 0x85, 0xae, 0xed, 0xd8, 0xb7, 0x14, 0xd7, 0xdc, 0x2b, 0xf2, 0x37, 0x94, 0x57, 0xfe, 0x1b, - 0x00, 0x00, 0xff, 0xff, 0x50, 0xf3, 0xe4, 0x7c, 0x66, 0x2b, 0x00, 0x00, + // 3045 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x1a, 0x4d, 0x73, 0x1c, 0x57, + 0xd1, 0x33, 0xfb, 0xdd, 0xd2, 0x4a, 0xe3, 0x67, 0x5b, 0x96, 0x6d, 0x59, 0x56, 0x26, 0x81, 0x52, + 0x36, 0xc1, 0xc6, 0x4e, 0xcc, 0x47, 0x08, 0x31, 0xeb, 0x5d, 0x25, 0xde, 0xb2, 0x56, 0x52, 0xbd, + 0x55, 0x62, 0x2a, 0x1c, 0x96, 0xd1, 0xcc, 0xf3, 0x6a, 0xa2, 0xdd, 0x79, 0x9b, 0x99, 0x59, 0xc9, + 0x76, 0x15, 0x50, 0x29, 0x8a, 0x1b, 0x07, 0xb8, 0x71, 0x20, 0x17, 0x4e, 0x9c, 0x80, 0x3f, 0x40, + 0xc1, 0x81, 0x03, 0x27, 0x2a, 0xc5, 0x81, 0xca, 0x09, 0x52, 0xc9, 0x1f, 0xa0, 0x8a, 0x2a, 0xb8, + 0x52, 0xef, 0x63, 0x3e, 0x77, 0x56, 0x2b, 0xa1, 0x2d, 0x17, 0xa7, 0x9d, 0xee, 0xd7, 0xdd, 0xaf, + 0x5f, 0xbf, 0x7e, 0xfd, 0xba, 0xfb, 0x2d, 0x5c, 0xdb, 0x27, 0x86, 0xeb, 0xef, 0x11, 0xc3, 0x1f, + 0xee, 0xdd, 0x0a, 0xbf, 0x6f, 0x0e, 0x5d, 0xea, 0x53, 0x34, 0x17, 0x1b, 0xd4, 0x9f, 0x42, 0x65, + 0xd7, 0xd8, 0xeb, 0x93, 0xce, 0xd0, 0x70, 0xd0, 0x32, 0x94, 0x38, 0xd0, 0x6a, 0x2e, 0x2b, 0x6b, + 0xca, 0x7a, 0x0e, 0x07, 0x20, 0xba, 0x0a, 0xe5, 0x8e, 0x6f, 0xb8, 0xfe, 0x43, 0xf2, 0x74, 0x59, + 0x5d, 0x53, 0xd6, 0xe7, 0x71, 0x08, 0xa3, 0x25, 0x28, 0x6e, 0x38, 0x16, 0x1b, 0xc9, 0xf1, 0x11, + 0x09, 0xa1, 0x55, 0x80, 0x87, 0xe4, 0xa9, 0x37, 0x34, 0x4c, 0x26, 0x30, 0xbf, 0xa6, 0xac, 0x57, + 0x71, 0x0c, 0xa3, 0xff, 0x4d, 0x05, 0xed, 0x01, 0x53, 0xe5, 0x3e, 0x31, 0x7c, 0x4c, 0x3e, 0x1c, + 0x11, 0xcf, 0x47, 0xdf, 0x86, 0x79, 0x73, 0xdf, 0x70, 0x7a, 0xe4, 0x31, 0x21, 0x96, 0xd4, 0x63, + 0xee, 0xce, 0x95, 0x9b, 0x31, 0x9d, 0x6f, 0x36, 0x62, 0x04, 0x38, 0x41, 0x8e, 0x5e, 0x87, 0xca, + 0x91, 0xe1, 0x13, 0x77, 0x60, 0xb8, 0x07, 0x5c, 0xd1, 0xb9, 0x3b, 0x4b, 0x09, 0xde, 0x47, 0xc1, + 0x28, 0x8e, 0x08, 0xd1, 0x9b, 0x50, 0x75, 0x89, 0x45, 0xc3, 0x31, 0xbe, 0x90, 0xc9, 0x9c, 0x49, + 0x62, 0xf4, 0x0d, 0x28, 0x7b, 0xbe, 0xe1, 0x8f, 0x3c, 0xe2, 0x2d, 0xe7, 0xd7, 0x72, 0xeb, 0x73, + 0x77, 0x56, 0x12, 0x8c, 0xa1, 0x7d, 0x3b, 0x9c, 0x0a, 0x87, 0xd4, 0x68, 0x1d, 0x16, 0x4d, 0x3a, + 0x18, 0x92, 0x3e, 0xf1, 0x89, 0x18, 0x5c, 0x2e, 0xac, 0x29, 0xeb, 0x65, 0x9c, 0x46, 0xa3, 0x57, + 0x20, 0x47, 0x5c, 0x77, 0xb9, 0x98, 0x61, 0x0d, 0x3c, 0x72, 0x1c, 0xdb, 0xe9, 0x6d, 0xb8, 0x2e, + 0x75, 0x31, 0xa3, 0xd2, 0x7f, 0xa2, 0x40, 0x25, 0x52, 0x4f, 0x67, 0x16, 0x25, 0xe6, 0xc1, 0x90, + 0xda, 0x8e, 0xbf, 0xeb, 0x71, 0x8b, 0xe6, 0x71, 0x02, 0xc7, 0xb6, 0xca, 0x25, 0x1e, 0xed, 0x1f, + 0x12, 0x6b, 0xd7, 0xe3, 0x76, 0xcb, 0xe3, 0x18, 0x06, 0x69, 0x90, 0xf3, 0xc8, 0x87, 0xdc, 0x2c, + 0x79, 0xcc, 0x3e, 0x99, 0xd4, 0xbe, 0xe1, 0xf9, 0x9d, 0xa7, 0x8e, 0xc9, 0x79, 0xf2, 0x42, 0x6a, + 0x1c, 0xa7, 0xff, 0x00, 0xb4, 0xa6, 0xed, 0x0d, 0x0d, 0xdf, 0xdc, 0x27, 0x6e, 0xdd, 0xf4, 0x6d, + 0xea, 0xa0, 0x57, 0xa0, 0x68, 0xf0, 0x2f, 0xae, 0xc7, 0xc2, 0x9d, 0x0b, 0x89, 0xb5, 0x08, 0x22, + 0x2c, 0x49, 0x98, 0xd7, 0x35, 0xe8, 0x60, 0x60, 0xfb, 0xa1, 0x52, 0x21, 0x8c, 0xd6, 0x60, 0xae, + 0xe5, 0xb1, 0xa9, 0x76, 0xd8, 0x1a, 0xb8, 0x6a, 0x65, 0x1c, 0x47, 0xe9, 0x0d, 0xc8, 0xd5, 0x1b, + 0x0f, 0x13, 0x42, 0x94, 0xe3, 0x85, 0xa8, 0xe3, 0x42, 0x30, 0xa0, 0x56, 0xcf, 0xa1, 0x2e, 0xb1, + 0xee, 0xf7, 0xa9, 0x79, 0x20, 0xb7, 0xe3, 0x6c, 0x32, 0x7f, 0xac, 0xc2, 0xa5, 0x96, 0xf3, 0xb8, + 0x3f, 0x22, 0xcc, 0x50, 0x91, 0x89, 0x3c, 0xf4, 0x1d, 0xa8, 0x86, 0x03, 0xbb, 0x4f, 0x87, 0x44, + 0x1a, 0xe9, 0x6a, 0xc2, 0x48, 0x09, 0x0a, 0x9c, 0x64, 0x40, 0xf7, 0xa0, 0x1a, 0x09, 0x6c, 0x35, + 0x99, 0xdd, 0x72, 0x63, 0x2e, 0x13, 0xa7, 0xc0, 0x49, 0x7a, 0x7e, 0xd2, 0xcd, 0x7d, 0x32, 0x30, + 0x5a, 0x4d, 0x6e, 0xd4, 0x1c, 0x0e, 0x61, 0xf4, 0x10, 0x2e, 0x90, 0x27, 0x66, 0x7f, 0x64, 0x91, + 0x18, 0x8f, 0xc5, 0xf7, 0xfe, 0xd8, 0x29, 0xb2, 0xb8, 0xf4, 0x5f, 0xa8, 0x71, 0xf7, 0x90, 0x86, + 0xfd, 0x2e, 0x5c, 0xb2, 0xb3, 0x2c, 0x23, 0xe3, 0x80, 0x9e, 0x6d, 0x88, 0x38, 0x25, 0xce, 0x16, + 0x80, 0xee, 0x86, 0x8e, 0x27, 0xc2, 0xc2, 0xf5, 0x09, 0xea, 0xa6, 0x5c, 0x50, 0x87, 0x9c, 0x61, + 0x06, 0x01, 0x41, 0x4b, 0x3a, 0x6b, 0xe3, 0x21, 0x66, 0x83, 0x68, 0x1b, 0x90, 0x3d, 0xe6, 0x23, + 0xd2, 0x2a, 0x37, 0x92, 0x1a, 0x8f, 0x91, 0xe1, 0x0c, 0x56, 0xfd, 0x33, 0x05, 0xce, 0xc7, 0x22, + 0xa3, 0x37, 0xa4, 0x8e, 0x47, 0xce, 0x1a, 0x1a, 0xdb, 0x80, 0xac, 0x94, 0xb9, 0x49, 0xe0, 0x1e, + 0x93, 0x8c, 0x11, 0xe8, 0x38, 0xce, 0x88, 0x10, 0xe4, 0x07, 0xd4, 0x22, 0xd2, 0x47, 0xf8, 0x37, + 0x7a, 0x19, 0xb4, 0x81, 0x61, 0x3b, 0xbe, 0x61, 0x3b, 0xc4, 0xed, 0x92, 0x21, 0x35, 0xf7, 0x65, + 0x60, 0x58, 0x8c, 0xf0, 0x1b, 0x0c, 0xad, 0x3f, 0x81, 0x0b, 0x8d, 0x58, 0x04, 0x6a, 0x13, 0xcf, + 0x33, 0x7a, 0x67, 0x5e, 0x63, 0x3a, 0xd6, 0xa9, 0xe3, 0xb1, 0x4e, 0xff, 0x83, 0x02, 0x8b, 0x98, + 0x58, 0xb4, 0x4d, 0x7c, 0x63, 0x46, 0xd3, 0x4e, 0x0b, 0x9f, 0x69, 0xb5, 0x72, 0x19, 0x21, 0xf8, + 0x14, 0xb6, 0xfb, 0x21, 0x5c, 0x67, 0x0b, 0xc0, 0xe1, 0x04, 0x3b, 0x2e, 0xed, 0xb9, 0xc4, 0xf3, + 0x9e, 0xcf, 0x72, 0xf4, 0x5f, 0x2b, 0xb0, 0x92, 0x54, 0xe0, 0x6d, 0xea, 0x1e, 0x19, 0xae, 0xf5, + 0x9c, 0xcc, 0x99, 0x65, 0xaa, 0x5c, 0xb6, 0xa9, 0xfe, 0xa5, 0xc4, 0x83, 0x4c, 0x83, 0x3a, 0x8f, + 0xed, 0x1e, 0xaa, 0x41, 0xde, 0x1b, 0x1a, 0x8e, 0x54, 0x6b, 0x29, 0xfb, 0xb2, 0xc6, 0x9c, 0x86, + 0xa5, 0x44, 0x1e, 0x4b, 0x74, 0x42, 0x45, 0x02, 0x90, 0x2d, 0xd2, 0x8a, 0x05, 0x39, 0x19, 0x22, + 0x8e, 0x89, 0x82, 0x09, 0x72, 0x16, 0x67, 0xbd, 0x20, 0xce, 0xe6, 0x45, 0x9c, 0x0d, 0xe0, 0xf0, + 0x6c, 0x15, 0x62, 0x67, 0xab, 0x06, 0x9a, 0x77, 0x60, 0x0f, 0x9b, 0xed, 0xcd, 0xba, 0xd7, 0x91, + 0x1a, 0x15, 0xf9, 0xdd, 0x32, 0x86, 0xd7, 0xff, 0xa8, 0xc2, 0x15, 0x16, 0xb4, 0xad, 0x51, 0x3f, + 0x16, 0x73, 0x67, 0x94, 0x62, 0xdd, 0x85, 0xa2, 0xc9, 0xed, 0x38, 0x25, 0x90, 0x0a, 0x63, 0x63, + 0x49, 0x8c, 0x1a, 0xb0, 0xe0, 0x49, 0x95, 0x44, 0x88, 0xe5, 0x06, 0x5b, 0xb8, 0x73, 0x2d, 0xc1, + 0xde, 0x49, 0x90, 0xe0, 0x14, 0x0b, 0x53, 0x9d, 0x0e, 0x89, 0x6b, 0xf8, 0xd4, 0xe5, 0xd7, 0x63, + 0x9e, 0x8b, 0x48, 0xaa, 0xbe, 0x1d, 0x23, 0xc0, 0x09, 0xf2, 0x4c, 0xc7, 0x29, 0x64, 0x3b, 0xce, + 0xaf, 0x54, 0x58, 0x6a, 0x13, 0xb7, 0x37, 0x7b, 0xfb, 0xdd, 0x83, 0xaa, 0x75, 0xca, 0x1b, 0x3a, + 0x41, 0x8f, 0x5a, 0x80, 0x06, 0x4c, 0x33, 0xab, 0x79, 0x2a, 0xf7, 0xcb, 0x60, 0x0a, 0x1d, 0x2d, + 0x3f, 0x25, 0x88, 0x4f, 0x30, 0xd2, 0x0e, 0x5c, 0x68, 0x87, 0xa8, 0x07, 0xc1, 0xc4, 0xe8, 0x9b, + 0xb1, 0x84, 0x58, 0xc9, 0xb8, 0x5f, 0x22, 0x9e, 0x74, 0x46, 0xac, 0x7f, 0xaa, 0x40, 0xb5, 0xe9, + 0x1a, 0xb6, 0x13, 0x84, 0x34, 0xf4, 0x12, 0x2c, 0xf8, 0x86, 0xdb, 0x23, 0x7e, 0xd7, 0xa1, 0x16, + 0xe9, 0xda, 0x16, 0xb7, 0x77, 0x05, 0xcf, 0x0b, 0xec, 0x16, 0xb5, 0x48, 0xcb, 0x42, 0x2f, 0x80, + 0x84, 0xa5, 0xc2, 0xe2, 0xac, 0xce, 0x09, 0x1c, 0x57, 0x16, 0x7d, 0x0d, 0x2e, 0x4b, 0x92, 0xc8, + 0x9c, 0x5d, 0x93, 0x8e, 0x64, 0xf2, 0x58, 0xc5, 0x97, 0xc4, 0x70, 0xdc, 0x83, 0x47, 0x8e, 0x8f, + 0xde, 0x86, 0x35, 0xc9, 0xc7, 0x12, 0x0b, 0xbb, 0xb7, 0xef, 0x77, 0x2d, 0xa6, 0x61, 0x77, 0x40, + 0x0f, 0x89, 0x14, 0x20, 0x8a, 0x9b, 0x15, 0x41, 0xd7, 0x92, 0x64, 0x7c, 0x1d, 0x6d, 0x7a, 0x48, + 0xb8, 0x1c, 0xfd, 0x37, 0x39, 0xd0, 0xd2, 0x2b, 0x3f, 0xab, 0x2f, 0x5d, 0x07, 0x60, 0x5f, 0x5d, + 0x66, 0x3f, 0xc2, 0x17, 0x5d, 0xc1, 0x15, 0x86, 0x61, 0xe2, 0x09, 0xba, 0x0d, 0x05, 0x31, 0x92, + 0x75, 0xd4, 0x1a, 0x74, 0x30, 0xa4, 0x0e, 0x71, 0x7c, 0x4e, 0x8b, 0x05, 0x25, 0x7a, 0x11, 0xaa, + 0xd1, 0xb5, 0xd4, 0xf5, 0xc3, 0xc4, 0x3e, 0x71, 0x57, 0xc9, 0x6a, 0xa4, 0x90, 0xe1, 0xb8, 0x63, + 0xd5, 0x08, 0xfa, 0x12, 0x2c, 0xec, 0x51, 0xea, 0x7b, 0xbe, 0x6b, 0x0c, 0xbb, 0x16, 0x75, 0x88, + 0x0c, 0x5b, 0xd5, 0x10, 0xdb, 0xa4, 0x0e, 0x19, 0x2b, 0x28, 0x4a, 0xe3, 0x05, 0x05, 0xaa, 0xc3, + 0x82, 0x30, 0xfd, 0x50, 0x7a, 0xc7, 0x72, 0x99, 0xdb, 0x2b, 0x99, 0x1f, 0x27, 0xfc, 0x07, 0x57, + 0xad, 0x84, 0x3b, 0x65, 0x79, 0x77, 0x25, 0xdb, 0xbb, 0x7f, 0xab, 0x42, 0x95, 0xb9, 0x57, 0xe4, + 0xd8, 0x77, 0xa1, 0xdc, 0xb7, 0x0f, 0x89, 0xc3, 0x66, 0x56, 0x32, 0x42, 0x0f, 0xa3, 0xde, 0x94, + 0x04, 0x38, 0x24, 0x65, 0xbb, 0xc4, 0x7d, 0x37, 0xee, 0x9a, 0x15, 0x86, 0x11, 0x8e, 0xd9, 0x84, + 0x1b, 0x31, 0x8f, 0x14, 0x0b, 0x4c, 0xb9, 0x7c, 0x8e, 0xef, 0xec, 0xb5, 0x88, 0x8c, 0xaf, 0x71, + 0x37, 0x7e, 0x02, 0xea, 0x70, 0x7d, 0x92, 0x94, 0x78, 0x32, 0x71, 0x35, 0x53, 0x86, 0x50, 0xe4, + 0x1e, 0xac, 0xf4, 0x69, 0xaf, 0xeb, 0x11, 0xf7, 0xd0, 0x36, 0xc9, 0xf8, 0x31, 0x29, 0x70, 0x2f, + 0xbf, 0xd2, 0xa7, 0xbd, 0x8e, 0x20, 0x49, 0x1d, 0x15, 0xfd, 0x03, 0x58, 0xea, 0x08, 0x85, 0x42, + 0x2b, 0xc8, 0x98, 0x79, 0x1b, 0x8a, 0x42, 0x99, 0xe9, 0x76, 0x93, 0x84, 0x53, 0xac, 0xa6, 0x0f, + 0xe0, 0xf2, 0xd8, 0x5c, 0x32, 0x51, 0x7e, 0x0d, 0x4a, 0xc6, 0x70, 0xd8, 0xb7, 0x89, 0x35, 0x7d, + 0xb6, 0x80, 0x72, 0xda, 0x74, 0x1f, 0xc0, 0x8d, 0x4e, 0x3c, 0x36, 0xc4, 0x8c, 0x17, 0xac, 0x71, + 0x56, 0x91, 0x4a, 0xff, 0x3a, 0x5c, 0x6b, 0x50, 0xea, 0x5a, 0xb6, 0xc3, 0x6e, 0xae, 0xfb, 0xc1, + 0x31, 0x09, 0xe6, 0x59, 0x86, 0xd2, 0x21, 0x71, 0xbd, 0xa0, 0x86, 0xce, 0xe1, 0x00, 0x64, 0x25, + 0xd5, 0x4a, 0x36, 0xa7, 0xb4, 0xcc, 0xff, 0x1e, 0x99, 0xd1, 0xeb, 0xb0, 0x14, 0x9e, 0x3d, 0x9f, + 0x9a, 0xb4, 0xdf, 0x0d, 0x94, 0x50, 0xb9, 0x5b, 0x5c, 0x0c, 0xce, 0x19, 0x1f, 0x7c, 0x4f, 0x8c, + 0xfd, 0xdf, 0xf8, 0xb6, 0xfe, 0x6f, 0x05, 0x2e, 0xd6, 0x2d, 0x2b, 0x5a, 0x60, 0x60, 0xcd, 0x97, + 0x41, 0x95, 0x3b, 0x75, 0x6c, 0xdc, 0x55, 0x6d, 0x0b, 0x2d, 0x25, 0x32, 0x9f, 0xf9, 0x30, 0xb5, + 0x19, 0x8b, 0x99, 0x59, 0xf9, 0x7d, 0x0d, 0xce, 0xdb, 0x5e, 0xd7, 0x21, 0x47, 0xdd, 0x28, 0x82, + 0x73, 0xbd, 0xcb, 0x78, 0xd1, 0xf6, 0xb6, 0xc8, 0x51, 0x34, 0x1d, 0xba, 0x01, 0x73, 0x07, 0xb2, + 0x4f, 0xc6, 0x2c, 0x24, 0xce, 0x1d, 0x04, 0xa8, 0x96, 0x95, 0x19, 0xc5, 0x8a, 0xd9, 0x51, 0xec, + 0x4f, 0x0a, 0x5c, 0xc6, 0x84, 0xdd, 0x55, 0x67, 0x5a, 0xfb, 0x32, 0x94, 0x4c, 0xc3, 0x33, 0x0d, + 0x8b, 0xc8, 0x8e, 0x46, 0x00, 0xb2, 0x11, 0x97, 0xcb, 0xb7, 0x64, 0x13, 0x26, 0x00, 0xd3, 0xcb, + 0xc8, 0x9f, 0x68, 0x19, 0x13, 0x52, 0x8d, 0xbf, 0xe4, 0xe0, 0x6a, 0xb4, 0x80, 0xb1, 0x33, 0x71, + 0xc6, 0x7b, 0x74, 0xd2, 0xce, 0x5e, 0xe1, 0xe7, 0xc5, 0x8d, 0x6d, 0x6a, 0x98, 0xfe, 0x9b, 0xf0, + 0x82, 0xcf, 0x6a, 0x85, 0xae, 0xef, 0xda, 0xbd, 0x1e, 0x53, 0xff, 0x90, 0x38, 0x89, 0xdc, 0xc2, + 0x3e, 0x41, 0x67, 0xe4, 0x3a, 0x97, 0xb1, 0x2b, 0x44, 0x6c, 0x30, 0x09, 0xf1, 0x1e, 0x49, 0xb6, + 0xd3, 0x14, 0xb2, 0x9d, 0xc6, 0x60, 0x79, 0x4a, 0x5c, 0x21, 0x97, 0x58, 0x34, 0xa5, 0x4f, 0x71, + 0x9a, 0x3e, 0x2b, 0x71, 0x7d, 0x58, 0x8d, 0x97, 0x50, 0x27, 0xb5, 0xa1, 0xa5, 0x13, 0x6d, 0x68, + 0x39, 0x7b, 0x43, 0xff, 0x9a, 0x83, 0x6b, 0x99, 0x1b, 0x3a, 0x9b, 0x6e, 0xc7, 0x5d, 0x28, 0xb0, + 0xfa, 0x2d, 0xc8, 0xae, 0x93, 0x6d, 0x98, 0x70, 0xb6, 0xa8, 0xda, 0x13, 0xd4, 0x41, 0x66, 0x93, + 0x3b, 0x49, 0x9f, 0xf5, 0x64, 0xb9, 0xd2, 0xab, 0x80, 0xf8, 0x46, 0x24, 0x29, 0x85, 0x97, 0x6b, + 0x6c, 0x24, 0xde, 0x06, 0x41, 0x4d, 0xa8, 0x04, 0x15, 0x0b, 0x2b, 0xef, 0x98, 0xea, 0x5f, 0xce, + 0x2c, 0x90, 0xc6, 0xca, 0x12, 0x1c, 0x31, 0x66, 0x6e, 0x43, 0x29, 0x73, 0x1b, 0xd0, 0x26, 0x2c, + 0xf2, 0xba, 0xa0, 0x1b, 0x4d, 0x5b, 0xe6, 0xd3, 0xbe, 0x98, 0xbc, 0x18, 0x32, 0x4b, 0x21, 0xbc, + 0xc0, 0x79, 0x83, 0x8a, 0xcb, 0xd3, 0xff, 0xae, 0xc2, 0x6a, 0xb4, 0xa9, 0x3b, 0xd4, 0xf3, 0x67, + 0x7d, 0x52, 0x4f, 0x74, 0xec, 0xd4, 0x33, 0x1e, 0xbb, 0xdb, 0x50, 0x12, 0xb5, 0x38, 0x3b, 0xf5, + 0xcc, 0x18, 0x97, 0xc7, 0xf6, 0x60, 0x60, 0xb4, 0x9c, 0xc7, 0x14, 0x07, 0x74, 0xe8, 0x0d, 0x98, + 0xe7, 0xdb, 0x1c, 0xf0, 0xe5, 0x8f, 0xe7, 0x9b, 0x63, 0xc4, 0x1d, 0xc9, 0x7b, 0x8a, 0x30, 0xf8, + 0xb1, 0x0a, 0x37, 0x26, 0x1a, 0x78, 0x36, 0x27, 0xe7, 0xb9, 0x58, 0xf8, 0x54, 0xe7, 0xec, 0x54, + 0x6d, 0x45, 0x88, 0xac, 0x9c, 0xe8, 0x65, 0x2b, 0xa9, 0x5e, 0xf6, 0x6a, 0x40, 0xb9, 0x65, 0x0c, + 0x82, 0xd2, 0x29, 0x86, 0x41, 0x37, 0xa1, 0xc8, 0xa3, 0x43, 0xe0, 0x02, 0x19, 0x6d, 0x22, 0xbe, + 0x93, 0x92, 0x4a, 0x6f, 0xc8, 0x87, 0x34, 0x3e, 0xf1, 0xe4, 0x87, 0xb4, 0x15, 0x49, 0x16, 0x9b, + 0x35, 0x42, 0xe8, 0xbf, 0x57, 0x01, 0x8d, 0x07, 0x27, 0x76, 0x4f, 0x4f, 0xd8, 0xc7, 0x84, 0xcd, + 0x55, 0xf9, 0x50, 0x17, 0x2c, 0x59, 0x4d, 0x2d, 0x39, 0xe8, 0x7b, 0xe5, 0x4e, 0xd0, 0xf7, 0x7a, + 0x1b, 0x34, 0x33, 0x28, 0x10, 0xbb, 0x5e, 0xd4, 0xd1, 0x9e, 0x52, 0x45, 0x2e, 0x9a, 0x71, 0x78, + 0xe4, 0x8d, 0xc7, 0xc8, 0x42, 0x46, 0x8c, 0x7c, 0x0d, 0xe6, 0xf6, 0xfa, 0xd4, 0x3c, 0x90, 0x75, + 0xac, 0xb8, 0xa5, 0x50, 0xf2, 0xec, 0x70, 0xf1, 0xb0, 0x17, 0x74, 0xc9, 0x49, 0xd8, 0xbb, 0x28, + 0x45, 0xbd, 0x0b, 0xfd, 0x97, 0x0a, 0x2c, 0x45, 0xc7, 0xa3, 0xd1, 0xa7, 0x1e, 0x99, 0x51, 0xdc, + 0x89, 0x65, 0x39, 0x6a, 0x32, 0xcb, 0x39, 0x45, 0x37, 0xf2, 0x63, 0x05, 0x2e, 0x8f, 0xa9, 0x37, + 0x9b, 0x53, 0xbb, 0x0c, 0x25, 0x6f, 0x64, 0x9a, 0xac, 0x32, 0x95, 0xfa, 0x49, 0xf0, 0x34, 0xfa, + 0xfd, 0x54, 0x01, 0x2d, 0x7a, 0x54, 0x11, 0x8e, 0x3d, 0x83, 0x37, 0xa9, 0xab, 0x50, 0x96, 0xee, + 0x2f, 0xae, 0xe3, 0x1c, 0x0e, 0xe1, 0xe3, 0x9e, 0x9b, 0xf4, 0xef, 0x41, 0x81, 0xd3, 0x4d, 0x79, + 0x97, 0x9e, 0xe4, 0xee, 0x2b, 0x50, 0xe9, 0x0c, 0xfb, 0x36, 0x0f, 0x44, 0x32, 0x35, 0x8d, 0x10, + 0xfa, 0x47, 0x2a, 0x5c, 0xc0, 0x74, 0xe4, 0x13, 0x2e, 0xaa, 0x6e, 0x0d, 0x6c, 0x8f, 0x57, 0x2c, + 0x35, 0xd0, 0x3a, 0x74, 0xe4, 0x9a, 0x24, 0x16, 0x1d, 0x44, 0x1d, 0x37, 0x86, 0x47, 0xeb, 0xb0, + 0x28, 0x70, 0xe9, 0x23, 0x9d, 0x46, 0x33, 0xa9, 0xa2, 0x1a, 0x89, 0x49, 0x15, 0x85, 0xcf, 0x18, + 0x9e, 0x49, 0x15, 0xb8, 0x48, 0x6a, 0x5e, 0x48, 0x4d, 0xa1, 0xd1, 0x5b, 0x50, 0x94, 0xbd, 0xd4, + 0x02, 0xdf, 0x93, 0x64, 0xaa, 0x90, 0xb1, 0xba, 0xe0, 0x71, 0x4b, 0xfc, 0xea, 0x0e, 0x2c, 0x04, + 0xd6, 0x12, 0xae, 0x75, 0x8c, 0xa5, 0xd7, 0x60, 0x6e, 0xbb, 0x6f, 0xa5, 0x8c, 0x1d, 0x47, 0x31, + 0x8a, 0x2d, 0x72, 0x94, 0xda, 0xcd, 0x38, 0x4a, 0xff, 0x4f, 0x0e, 0x0a, 0xe2, 0xf0, 0xae, 0x40, + 0xa5, 0xe5, 0xf1, 0x27, 0x2f, 0x59, 0xa4, 0x97, 0x71, 0x84, 0x60, 0x5a, 0xf0, 0xcf, 0xa8, 0xe9, + 0x2e, 0x41, 0x74, 0x0f, 0xe6, 0xc4, 0x67, 0x10, 0x9a, 0xc7, 0x3b, 0xd0, 0x69, 0x07, 0xc6, 0x71, + 0x0e, 0xf4, 0x10, 0xce, 0x6f, 0x11, 0x62, 0x35, 0x5d, 0x3a, 0x1c, 0x06, 0x14, 0x32, 0x4d, 0x9f, + 0x22, 0x66, 0x9c, 0x0f, 0xbd, 0x09, 0x8b, 0x0c, 0x59, 0xb7, 0xac, 0x50, 0x94, 0xe8, 0x89, 0xa1, + 0xf1, 0xd8, 0x8a, 0xd3, 0xa4, 0xa8, 0x01, 0x0b, 0xef, 0x0e, 0x2d, 0xc3, 0x27, 0xd2, 0x84, 0x41, + 0xc2, 0x77, 0x2d, 0x2b, 0x69, 0x90, 0x1b, 0x84, 0x53, 0x2c, 0xe9, 0xd7, 0xe6, 0xd2, 0xd8, 0x6b, + 0x33, 0xfa, 0x0a, 0x6f, 0x02, 0xf6, 0x08, 0x4f, 0xc4, 0x17, 0x52, 0x29, 0x49, 0xf0, 0xea, 0xd8, + 0x13, 0x0d, 0xc0, 0x1e, 0x41, 0xbb, 0x70, 0x31, 0xc3, 0x71, 0xbc, 0xe5, 0x0a, 0xd7, 0x6d, 0x6d, + 0x9a, 0x87, 0xe1, 0x4c, 0x6e, 0xfd, 0x47, 0x70, 0x31, 0xbc, 0x61, 0xe2, 0x0f, 0xe9, 0xa7, 0xb8, + 0xd9, 0xd6, 0x83, 0x66, 0xa6, 0x3a, 0xf1, 0x7a, 0x90, 0x3d, 0xcc, 0x8c, 0xa7, 0x49, 0xfd, 0x9f, + 0x0a, 0x3b, 0x55, 0x89, 0x3f, 0x62, 0x9c, 0x66, 0xf2, 0xac, 0xeb, 0x50, 0x9d, 0xc5, 0x75, 0x98, + 0xd5, 0x2a, 0xb8, 0x0d, 0x97, 0x44, 0xce, 0xe5, 0xd9, 0xcf, 0x48, 0x77, 0x48, 0xdc, 0xae, 0x47, + 0x4c, 0xea, 0x88, 0x72, 0x52, 0xc5, 0x88, 0x0f, 0x76, 0xec, 0x67, 0x64, 0x87, 0xb8, 0x1d, 0x3e, + 0x92, 0xf5, 0x62, 0xa4, 0xff, 0x4e, 0x01, 0x14, 0x7f, 0x69, 0x9e, 0xcd, 0x45, 0xf8, 0x0e, 0x54, + 0xf7, 0x22, 0xa1, 0xe1, 0x0b, 0xf2, 0x0b, 0xd9, 0xd9, 0x44, 0x7c, 0xfe, 0x24, 0x5f, 0xe6, 0x2e, + 0x59, 0x30, 0x1f, 0x4f, 0xff, 0x18, 0x8d, 0x6f, 0x87, 0x01, 0x98, 0x7f, 0x33, 0x9c, 0x43, 0xad, + 0x20, 0xd2, 0xf2, 0x6f, 0x86, 0x33, 0x03, 0x59, 0x15, 0xcc, 0xbf, 0x59, 0x10, 0x19, 0x88, 0xf7, + 0x48, 0x19, 0x3e, 0x03, 0x50, 0x7f, 0x1d, 0xe6, 0xd3, 0xaf, 0x20, 0xfb, 0x76, 0x6f, 0x5f, 0xfe, + 0x93, 0x83, 0x7f, 0x23, 0x0d, 0x72, 0x7d, 0x7a, 0x24, 0xc3, 0x0f, 0xfb, 0x64, 0xba, 0xc5, 0xcd, + 0x72, 0x32, 0x2e, 0xae, 0x6d, 0x14, 0xec, 0xf9, 0x37, 0xbb, 0xb4, 0x82, 0x9a, 0x59, 0xaa, 0x16, + 0xc2, 0xfa, 0xf7, 0xe1, 0xc6, 0x26, 0xed, 0xc5, 0x9a, 0x78, 0xd1, 0x23, 0xeb, 0x6c, 0x36, 0x50, + 0xff, 0x48, 0x81, 0xb5, 0xc9, 0x53, 0xcc, 0x26, 0x1b, 0x99, 0xf6, 0x82, 0xdc, 0x67, 0xb6, 0x24, + 0xe6, 0x81, 0x37, 0x1a, 0xb4, 0x89, 0x6f, 0xa0, 0xaf, 0x06, 0x67, 0x3b, 0x2b, 0xb7, 0x08, 0x28, + 0x13, 0x67, 0xbc, 0x06, 0x9a, 0x19, 0xc7, 0x77, 0xc8, 0x87, 0x72, 0x9e, 0x31, 0xbc, 0xfe, 0x73, + 0x05, 0x2e, 0xc5, 0xfe, 0xd3, 0x40, 0xfc, 0x40, 0x22, 0xba, 0x08, 0x05, 0xd1, 0xda, 0x16, 0x9b, + 0x28, 0x00, 0xe6, 0x39, 0x4f, 0xa8, 0xfb, 0x80, 0x6d, 0xae, 0xbc, 0x7e, 0x24, 0x88, 0x96, 0xa0, + 0xf8, 0x84, 0xba, 0x9b, 0xf4, 0x48, 0x9e, 0x5b, 0x09, 0x89, 0xec, 0x6b, 0xc0, 0x39, 0xf2, 0xb2, + 0x4d, 0x24, 0x40, 0xc6, 0xe1, 0x8d, 0x06, 0x8c, 0x43, 0x24, 0xbe, 0x12, 0x62, 0xa9, 0xe0, 0x5a, + 0xa6, 0x4e, 0x75, 0xf3, 0x60, 0x56, 0xbb, 0x70, 0x11, 0x0a, 0xf1, 0x1e, 0xb3, 0x00, 0x32, 0xff, + 0xb8, 0x21, 0xff, 0xdf, 0x95, 0x0f, 0xff, 0xdf, 0xa5, 0xff, 0x43, 0x01, 0x3d, 0x53, 0x3f, 0x71, + 0xff, 0xcc, 0x28, 0x98, 0x9c, 0x41, 0x43, 0xf4, 0x16, 0x94, 0x83, 0x9d, 0xe6, 0xb6, 0x4d, 0xff, + 0x3b, 0x28, 0x53, 0x7b, 0x1c, 0xf2, 0xd4, 0xae, 0x07, 0xc9, 0x13, 0xaa, 0x40, 0xe1, 0x91, 0x6b, + 0xfb, 0x44, 0x3b, 0x87, 0xca, 0x90, 0xdf, 0x31, 0x3c, 0x4f, 0x53, 0x6a, 0xeb, 0x22, 0x37, 0x8a, + 0x3d, 0x3e, 0x03, 0x14, 0x1b, 0x2e, 0x31, 0x38, 0x1d, 0x40, 0x51, 0x34, 0x55, 0x35, 0xa5, 0xd6, + 0x86, 0xf9, 0xf8, 0x9b, 0x33, 0x13, 0xb7, 0xdd, 0xad, 0x5b, 0x96, 0x76, 0x0e, 0xcd, 0x43, 0x79, + 0xbb, 0x1b, 0x10, 0x32, 0xa6, 0xed, 0x6e, 0x9b, 0x7d, 0xab, 0x68, 0x0e, 0x4a, 0xdb, 0x5d, 0x9e, + 0x8d, 0x6a, 0x39, 0x01, 0xf0, 0x16, 0x8b, 0x96, 0xaf, 0xdd, 0x85, 0xf9, 0xf8, 0x0b, 0x05, 0x13, + 0x57, 0xdf, 0x6c, 0xbd, 0xb7, 0x21, 0xc4, 0x35, 0x71, 0xbd, 0xb5, 0xd5, 0xda, 0x7a, 0x47, 0x53, + 0x18, 0xd4, 0xd9, 0xdd, 0xde, 0xd9, 0x61, 0x90, 0x5a, 0x7b, 0x03, 0x20, 0xba, 0xcc, 0xd9, 0x3a, + 0xb6, 0xb6, 0xb7, 0x18, 0xcf, 0x1c, 0x94, 0x1e, 0xd5, 0x5b, 0xbb, 0x82, 0x85, 0x01, 0x58, 0x00, + 0x2a, 0xa3, 0x69, 0x32, 0x9a, 0x5c, 0xed, 0xd5, 0x54, 0x8a, 0x8f, 0x4a, 0x90, 0xab, 0xf7, 0xfb, + 0xda, 0x39, 0x54, 0x04, 0xb5, 0x79, 0x5f, 0xa8, 0xbe, 0x45, 0xdd, 0x81, 0xd1, 0xd7, 0xd4, 0xda, + 0x3b, 0x70, 0x65, 0x62, 0x6a, 0xc9, 0xb5, 0x6d, 0xb6, 0x5b, 0xbb, 0x62, 0x66, 0xbc, 0xb1, 0xb9, + 0x51, 0xef, 0x6c, 0x68, 0x0a, 0x42, 0xb0, 0x20, 0x81, 0x6e, 0xa7, 0xf1, 0x60, 0xa3, 0x5d, 0xd7, + 0xd4, 0xda, 0x33, 0x58, 0x48, 0xde, 0x97, 0x5c, 0x3f, 0xea, 0x1e, 0xd8, 0x4e, 0x4f, 0xf0, 0x77, + 0x7c, 0x9e, 0x6e, 0x09, 0xcd, 0x85, 0x1d, 0x2d, 0x4d, 0x45, 0x1a, 0xcc, 0xb7, 0x1c, 0xdb, 0xb7, + 0x8d, 0xbe, 0xfd, 0x8c, 0xd1, 0xe6, 0x50, 0x15, 0x2a, 0x3b, 0x2e, 0x19, 0x1a, 0x2e, 0x03, 0xf3, + 0x68, 0x01, 0x80, 0x9b, 0x13, 0x13, 0xc3, 0x7a, 0xaa, 0x15, 0x18, 0xc3, 0x23, 0xc3, 0xf6, 0x6d, + 0xa7, 0x27, 0xac, 0x5c, 0xac, 0x7d, 0x0b, 0xaa, 0x89, 0xb8, 0x82, 0xce, 0x43, 0xf5, 0xdd, 0xad, + 0xd6, 0x56, 0x6b, 0xb7, 0x55, 0xdf, 0x6c, 0xbd, 0xbf, 0xd1, 0x14, 0xe6, 0x6e, 0xb7, 0x3a, 0xed, + 0xfa, 0x6e, 0xe3, 0x81, 0xa6, 0xb0, 0x95, 0x89, 0x4f, 0xf5, 0xfe, 0x5b, 0x7f, 0xfe, 0x7c, 0x55, + 0xf9, 0xe4, 0xf3, 0x55, 0xe5, 0xb3, 0xcf, 0x57, 0x95, 0x9f, 0x7d, 0xb1, 0x7a, 0xee, 0x93, 0x2f, + 0x56, 0xcf, 0x7d, 0xfa, 0xc5, 0xea, 0xb9, 0xf7, 0x5f, 0xea, 0xd9, 0xfe, 0xfe, 0x68, 0xef, 0xa6, + 0x49, 0x07, 0xb7, 0x86, 0xb6, 0xd3, 0x33, 0x8d, 0xe1, 0x2d, 0xdf, 0x36, 0x2d, 0xf3, 0x56, 0xcc, + 0x35, 0xf7, 0x8a, 0xfc, 0x0d, 0xe5, 0xb5, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0x6e, 0x77, 0x18, + 0x03, 0xa7, 0x2b, 0x00, 0x00, } func (m *TableSpan) Marshal() (dAtA []byte, err error) { @@ -5271,6 +5283,11 @@ func (m *NodeHeartbeat) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.LogServiceDispatcherCount != 0 { + i = encodeVarintHeartbeat(dAtA, i, uint64(m.LogServiceDispatcherCount)) + i-- + dAtA[i] = 0x28 + } if m.DispatcherDrainTargetEpoch != 0 { i = encodeVarintHeartbeat(dAtA, i, uint64(m.DispatcherDrainTargetEpoch)) i-- @@ -7549,6 +7566,9 @@ func (m *NodeHeartbeat) Size() (n int) { if m.DispatcherDrainTargetEpoch != 0 { n += 1 + sovHeartbeat(uint64(m.DispatcherDrainTargetEpoch)) } + if m.LogServiceDispatcherCount != 0 { + n += 1 + sovHeartbeat(uint64(m.LogServiceDispatcherCount)) + } return n } @@ -11320,6 +11340,25 @@ func (m *NodeHeartbeat) Unmarshal(dAtA []byte) error { break } } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LogServiceDispatcherCount", wireType) + } + m.LogServiceDispatcherCount = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LogServiceDispatcherCount |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipHeartbeat(dAtA[iNdEx:]) diff --git a/heartbeatpb/heartbeat.proto b/heartbeatpb/heartbeat.proto index b55a9117af..e9e322b204 100644 --- a/heartbeatpb/heartbeat.proto +++ b/heartbeatpb/heartbeat.proto @@ -206,6 +206,9 @@ message NodeHeartbeat { // currently applied on this node. Empty target means the drain target is clear. string dispatcher_drain_target_node_id = 3; uint64 dispatcher_drain_target_epoch = 4; + // log_service_dispatcher_count is the number of dispatchers currently + // reading from this node's event store. + uint32 log_service_dispatcher_count = 5; } // SetNodeLivenessRequest asks a node to transition its local liveness. diff --git a/logservice/eventstore/event_store.go b/logservice/eventstore/event_store.go index d85319b97c..5ba2eb303a 100644 --- a/logservice/eventstore/event_store.go +++ b/logservice/eventstore/event_store.go @@ -94,6 +94,15 @@ type EventStore interface { UpdateDispatcherCheckpointTs(dispatcherID common.DispatcherID, checkpointTs uint64) + // DispatcherCount returns the number of dispatchers currently reading from this event store. + DispatcherCount() int + + // GetSubscriptionWrittenResolvedTs returns the watermark of data that has + // actually been persisted for the dispatcher's subscription. It may be lower + // than the puller's resolved ts because of the write queue, and is what the + // ready event should report so it never promises data the scan cannot serve. + GetSubscriptionWrittenResolvedTs(dispatcherID common.DispatcherID) uint64 + // GetIterator returns an iterator for the requested range and resume cursor. GetIterator(dispatcherID common.DispatcherID, request ScanRequest) (EventIterator, error) @@ -194,6 +203,12 @@ type subscriptionStat struct { lastReceiveDMLTime atomic.Int64 // the resolveTs persisted in the store resolvedTs atomic.Uint64 + // writtenResolvedTs is the watermark of data that has actually been + // persisted to the store (written by the write path). It may lag resolvedTs + // (the puller's enqueue watermark) by the write queue, and it is the value + // the ready event should report so it never promises data the scan cannot + // serve yet. + writtenResolvedTs atomic.Uint64 // the max commit ts of dml event in the store maxEventCommitTs atomic.Uint64 } @@ -355,6 +370,40 @@ func newWriteTaskPool(store *eventStore, db *pebble.DB, index int, ch *chann.Unl } } +// advanceWrittenResolvedTs advances the written resolved ts of every +// subscription whose events were just persisted to pebble. The ready event and +// the scan range are driven by this watermark (via GetSubscriptionWrittenResolvedTs) +// so they never promise data that has not actually been written yet. +func (p *writeTaskPool) advanceWrittenResolvedTs(events []eventWithCallback) { + type subWriteKey struct { + tableKey tableStatsKey + subID logpuller.SubscriptionID + } + written := make(map[subWriteKey]uint64, 8) + for _, e := range events { + key := subWriteKey{ + tableKey: tableStatsKey{keyspaceID: e.keyspaceID, tableID: e.tableID}, + subID: e.subID, + } + if e.currentResolvedTs > written[key] { + written[key] = e.currentResolvedTs + } + } + if len(written) == 0 { + return + } + p.store.dispatcherMeta.RLock() + for key, ts := range written { + subStats := p.store.dispatcherMeta.tableStats[key.tableKey] + subStat, ok := subStats[key.subID] + if !ok { + continue + } + util.CompareAndMonotonicIncrease(&subStat.writtenResolvedTs, ts) + } + p.store.dispatcherMeta.RUnlock() +} + func (p *writeTaskPool) run(ctx context.Context) { p.store.wg.Add(p.workerNum) for i := 0; i < p.workerNum; i++ { @@ -398,6 +447,8 @@ func (p *writeTaskPool) run(ctx context.Context) { events[idx].callback() } + p.advanceWrittenResolvedTs(events) + buffer = buffer[:0] } } @@ -639,6 +690,7 @@ func (e *eventStore) RegisterDispatcher( }) subStat.checkpointTs.Store(startTs) subStat.resolvedTs.Store(startTs) + subStat.writtenResolvedTs.Store(startTs) subStat.maxEventCommitTs.Store(0) subStat.lastLogLagTime.Store(0) if stat.subStat == nil { @@ -744,6 +796,22 @@ func (e *eventStore) UnregisterDispatcher(changefeedID common.ChangeFeedID, disp zap.Stringer("dispatcherID", dispatcherID)) } +func (e *eventStore) DispatcherCount() int { + e.dispatcherMeta.RLock() + defer e.dispatcherMeta.RUnlock() + return len(e.dispatcherMeta.dispatcherStats) +} + +func (e *eventStore) GetSubscriptionWrittenResolvedTs(dispatcherID common.DispatcherID) uint64 { + e.dispatcherMeta.RLock() + defer e.dispatcherMeta.RUnlock() + stat, ok := e.dispatcherMeta.dispatcherStats[dispatcherID] + if !ok || stat.subStat == nil { + return 0 + } + return stat.subStat.writtenResolvedTs.Load() +} + func (e *eventStore) UpdateDispatcherCheckpointTs( dispatcherID common.DispatcherID, checkpointTs uint64, diff --git a/logservice/eventstore/event_store_test.go b/logservice/eventstore/event_store_test.go index 8e5380b3d2..a721c385f2 100644 --- a/logservice/eventstore/event_store_test.go +++ b/logservice/eventstore/event_store_test.go @@ -866,6 +866,7 @@ func TestEventStoreUnregisterDispatcherWithoutDataSharingRemovesSubscription(t * EndKey: []byte("h"), } require.True(t, store.RegisterDispatcher(cfID, dispatcherID, span, 100, func(uint64, uint64) {}, false, false, false)) + require.Equal(t, 1, store.DispatcherCount()) mockSubClient := subClient.(*mockSubscriptionClient) mockSubClient.mu.Lock() @@ -873,6 +874,7 @@ func TestEventStoreUnregisterDispatcherWithoutDataSharingRemovesSubscription(t * mockSubClient.mu.Unlock() store.UnregisterDispatcher(cfID, dispatcherID) + require.Zero(t, store.DispatcherCount()) mockSubClient.mu.Lock() require.Equal(t, 1, len(mockSubClient.subscriptions)) @@ -908,8 +910,10 @@ func TestEventStoreUnregisterDispatcherWithDataSharingKeepsSubscriptionForTTL(t EndKey: []byte("h"), } require.True(t, store.RegisterDispatcher(cfID, dispatcherID, span, 100, func(uint64, uint64) {}, false, false, false)) + require.Equal(t, 1, store.DispatcherCount()) store.UnregisterDispatcher(cfID, dispatcherID) + require.Zero(t, store.DispatcherCount()) mockSubClient := subClient.(*mockSubscriptionClient) mockSubClient.mu.Lock() diff --git a/logservice/schemastore/schema_store.go b/logservice/schemastore/schema_store.go index 5273c93ca8..aaeccf2d58 100644 --- a/logservice/schemastore/schema_store.go +++ b/logservice/schemastore/schema_store.go @@ -74,6 +74,7 @@ type DDLEventState struct { type keyspaceSchemaStore struct { cancel context.CancelFunc + wg sync.WaitGroup ddlJobFetcher *ddlJobFetcher gcKeeper *schemaStoreGCKeeper pdClock pdutil.Clock @@ -349,6 +350,7 @@ func (s *schemaStore) Close(ctx context.Context) error { if store.cancel != nil { store.cancel() } + store.wg.Wait() if store.gcKeeper != nil { err := closeSchemaStoreGCKeeper(keyspaceID, store.gcKeeper) if err != nil { @@ -603,7 +605,9 @@ func (s *schemaStore) RegisterKeyspace( return store.resolvedTs.Load() }) + store.wg.Add(1) go func(ctx context.Context, schemaStore *keyspaceSchemaStore) { + defer schemaStore.wg.Done() ticker := time.NewTicker(50 * time.Millisecond) defer ticker.Stop() for { diff --git a/logservice/schemastore/schema_store_test.go b/logservice/schemastore/schema_store_test.go index 1c30d52f95..1d7cabfca5 100644 --- a/logservice/schemastore/schema_store_test.go +++ b/logservice/schemastore/schema_store_test.go @@ -152,6 +152,36 @@ func TestIgnoreDDLByCommitTs(t *testing.T) { require.NotContains(t, tableNames, "t2") } +func TestSchemaStoreCloseWaitsForWorkersBeforeClosingPebble(t *testing.T) { + pstorage := newPersistentStorageForTest(t.TempDir(), nil) + workerStarted := make(chan struct{}) + releaseWorker := make(chan struct{}) + store := &keyspaceSchemaStore{dataStorage: pstorage} + store.wg.Go(func() { + close(workerStarted) + <-releaseWorker + }) + <-workerStarted + + schemaStore := &schemaStore{ + keyspaceSchemaStoreMap: map[uint32]*keyspaceSchemaStore{0: store}, + } + closeDone := make(chan error, 1) + go func() { + closeDone <- schemaStore.Close(context.Background()) + }() + + select { + case err := <-closeDone: + require.FailNow(t, "schema store closed before its worker exited", "error: %v", err) + case <-time.After(50 * time.Millisecond): + } + require.NoError(t, pstorage.db.Flush()) + + close(releaseWorker) + require.NoError(t, <-closeDone) +} + func TestTryUpdateResolvedTsRetryAfterDDLHandleFailure(t *testing.T) { mockPDClock := pdutil.NewClock4Test() appcontext.SetService(appcontext.DefaultPDClock, mockPDClock) diff --git a/maintainer/maintainer_manager_node.go b/maintainer/maintainer_manager_node.go index 3b67b3d5f1..0cd4361fa5 100644 --- a/maintainer/maintainer_manager_node.go +++ b/maintainer/maintainer_manager_node.go @@ -19,6 +19,8 @@ import ( "github.com/pingcap/log" "github.com/pingcap/ticdc/heartbeatpb" + "github.com/pingcap/ticdc/logservice/eventstore" + appcontext "github.com/pingcap/ticdc/pkg/common/context" "github.com/pingcap/ticdc/pkg/liveness" "github.com/pingcap/ticdc/pkg/messaging" "github.com/pingcap/ticdc/pkg/node" @@ -74,9 +76,10 @@ func newNodeEpoch() uint64 { return nodeEpoch } -// sendNodeHeartbeat reports node-scoped liveness and dispatcher drain target to -// coordinator. It is the authoritative acknowledgement channel for node-level -// drain state, including cases where no changefeed maintainer exists locally. +// sendNodeHeartbeat reports node-scoped liveness, dispatcher drain target, and +// the local log service dispatcher count to coordinator. It is the authoritative +// acknowledgement channel for node-level drain state, including cases where no +// changefeed maintainer exists locally. func (m *Manager) sendNodeHeartbeat(force bool) { if !m.isBootstrap() { return @@ -95,9 +98,14 @@ func (m *Manager) sendNodeHeartbeat(force bool) { currentLiveness = m.node.liveness.Load() } drainTarget, drainEpoch := m.getDispatcherDrainTarget() + logServiceDispatcherCount := 0 + if store, ok := appcontext.TryGetService[eventstore.EventStore](appcontext.EventStore); ok { + logServiceDispatcherCount = store.DispatcherCount() + } hb := &heartbeatpb.NodeHeartbeat{ - Liveness: m.toNodeLivenessPB(currentLiveness), - NodeEpoch: m.node.nodeEpoch, + Liveness: m.toNodeLivenessPB(currentLiveness), + NodeEpoch: m.node.nodeEpoch, + LogServiceDispatcherCount: uint32(logServiceDispatcherCount), // Report the manager-level dispatcher drain target so coordinator can // confirm both activation and clearing even when no maintainers exist. DispatcherDrainTargetNodeId: drainTarget.String(), diff --git a/maintainer/node_liveness_test.go b/maintainer/node_liveness_test.go index b35e056dec..13178ec631 100644 --- a/maintainer/node_liveness_test.go +++ b/maintainer/node_liveness_test.go @@ -17,6 +17,7 @@ import ( "testing" "github.com/pingcap/ticdc/heartbeatpb" + "github.com/pingcap/ticdc/logservice/eventstore" "github.com/pingcap/ticdc/pkg/common" appcontext "github.com/pingcap/ticdc/pkg/common/context" "github.com/pingcap/ticdc/pkg/config" @@ -26,6 +27,17 @@ import ( "github.com/stretchr/testify/require" ) +type mockEventStore struct { + eventstore.EventStore + count int +} + +var _ eventstore.EventStore = (*mockEventStore)(nil) + +func (m *mockEventStore) DispatcherCount() int { + return m.count +} + func TestSetNodeLivenessRejectEpochMismatch(t *testing.T) { mc := messaging.NewMockMessageCenter() appcontext.SetService(appcontext.MessageCenter, mc) @@ -159,6 +171,16 @@ func TestSetDispatcherDrainTargetRejectStaleUpdate(t *testing.T) { func TestSetDispatcherDrainTargetSendsNodeHeartbeatAck(t *testing.T) { mc := messaging.NewMockMessageCenter() appcontext.SetService(appcontext.MessageCenter, mc) + previousEventStore, hadPreviousEventStore := appcontext.TryGetService[any](appcontext.EventStore) + logService := &mockEventStore{count: 2} + appcontext.SetService(appcontext.EventStore, logService) + t.Cleanup(func() { + if hadPreviousEventStore { + appcontext.SetService(appcontext.EventStore, previousEventStore) + } else { + appcontext.DeleteService(appcontext.EventStore) + } + }) var nodeLiveness liveness.Liveness m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness) @@ -185,6 +207,7 @@ func TestSetDispatcherDrainTargetSendsNodeHeartbeatAck(t *testing.T) { hb := apply("n2", 1) require.Equal(t, "n2", hb.DispatcherDrainTargetNodeId) require.Equal(t, uint64(1), hb.DispatcherDrainTargetEpoch) + require.Equal(t, uint32(2), hb.LogServiceDispatcherCount) hb = apply("", 1) require.Equal(t, "", hb.DispatcherDrainTargetNodeId) diff --git a/pkg/common/context/app_context.go b/pkg/common/context/app_context.go index 84bca4d842..c9f5d4816c 100644 --- a/pkg/common/context/app_context.go +++ b/pkg/common/context/app_context.go @@ -60,6 +60,7 @@ func SetID(id string) { GetGlobalContext().id = id } func GetID() string { return GetGlobalContext().id } func SetService[T any](name string, t T) { GetGlobalContext().serviceMap.Store(name, t) } +func DeleteService(name string) { GetGlobalContext().serviceMap.Delete(name) } func GetService[T any](name string) T { v, _ := GetGlobalContext().serviceMap.Load(name) return v.(T) diff --git a/pkg/common/event/ready_event.go b/pkg/common/event/ready_event.go index 5908c953d3..a87f8a06ad 100644 --- a/pkg/common/event/ready_event.go +++ b/pkg/common/event/ready_event.go @@ -14,6 +14,7 @@ package event import ( + "encoding/binary" "fmt" "github.com/pingcap/ticdc/pkg/common" @@ -28,17 +29,22 @@ var _ Event = &ReadyEvent{} type ReadyEvent struct { Version int DispatcherID common.DispatcherID + // ResolvedTs is the highest timestamp the EventService can currently serve. + // It is appended to the version 1 payload so old decoders can ignore it, while + // new decoders still accept legacy payloads that only contain DispatcherID. + ResolvedTs uint64 } -func NewReadyEvent(dispatcherID common.DispatcherID) ReadyEvent { +func NewReadyEvent(dispatcherID common.DispatcherID, resolvedTs uint64) ReadyEvent { return ReadyEvent{ Version: ReadyEventVersion1, DispatcherID: dispatcherID, + ResolvedTs: resolvedTs, } } func (e *ReadyEvent) String() string { - return fmt.Sprintf("ReadyEvent{Version: %d, DispatcherID: %s}", e.Version, e.DispatcherID) + return fmt.Sprintf("ReadyEvent{Version: %d, DispatcherID: %s, ResolvedTs: %d}", e.Version, e.DispatcherID, e.ResolvedTs) } // GetType returns the event type @@ -64,8 +70,7 @@ func (e *ReadyEvent) GetDispatcherID() common.DispatcherID { // GetCommitTs returns the commit timestamp func (e *ReadyEvent) GetCommitTs() common.Ts { - // not used - return 0 + return common.Ts(e.ResolvedTs) } // GetStartTs returns the start timestamp @@ -77,8 +82,8 @@ func (e *ReadyEvent) GetStartTs() common.Ts { // GetSize returns the approximate size of the event in bytes func (e *ReadyEvent) GetSize() int64 { // Size does not include header or version (those are only for serialization) - // Only business data: dispatcherID - return int64(e.DispatcherID.GetSize()) + // Only business data: dispatcherID + resolvedTs + return int64(e.DispatcherID.GetSize() + 8) } func (e *ReadyEvent) IsPaused() bool { @@ -129,13 +134,16 @@ func (e *ReadyEvent) Unmarshal(data []byte) error { func (e ReadyEvent) encodeV1() ([]byte, error) { // Note: version is now handled in the header by Marshal(), not here - // payload: dispatcherID - payloadSize := e.DispatcherID.GetSize() + // payload: dispatcherID + optional resolvedTs + payloadSize := e.DispatcherID.GetSize() + 8 data := make([]byte, payloadSize) offset := 0 // DispatcherID copy(data[offset:], e.DispatcherID.Marshal()) + offset += e.DispatcherID.GetSize() + + binary.BigEndian.PutUint64(data[offset:], e.ResolvedTs) return data, nil } @@ -149,6 +157,13 @@ func (e *ReadyEvent) decodeV1(data []byte) error { if err != nil { return err } + offset += e.DispatcherID.GetSize() + + // ResolvedTs was appended to the version 1 payload. Treat its absence as + // zero so messages produced by older EventServices remain decodable. + if len(data) >= offset+8 { + e.ResolvedTs = binary.BigEndian.Uint64(data[offset:]) + } return nil } diff --git a/pkg/common/event/ready_event_test.go b/pkg/common/event/ready_event_test.go index e8fde74e46..8ae7e3061a 100644 --- a/pkg/common/event/ready_event_test.go +++ b/pkg/common/event/ready_event_test.go @@ -23,7 +23,7 @@ import ( func TestReadyEvent(t *testing.T) { did := common.NewDispatcherID() - e := NewReadyEvent(did) + e := NewReadyEvent(did, 123) data, err := e.Marshal() require.NoError(t, err) require.Len(t, data, int(e.GetSize())+int(GetEventHeaderSize())) @@ -33,11 +33,12 @@ func TestReadyEvent(t *testing.T) { require.NoError(t, err) require.Equal(t, e.Version, e2.Version) require.Equal(t, e.DispatcherID, e2.DispatcherID) + require.Equal(t, e.ResolvedTs, e2.ResolvedTs) } func TestReadyEventMethods(t *testing.T) { did := common.NewDispatcherID() - e := NewReadyEvent(did) + e := NewReadyEvent(did, 123) // Test GetType require.Equal(t, TypeReadyEvent, e.GetType()) @@ -52,7 +53,7 @@ func TestReadyEventMethods(t *testing.T) { require.Equal(t, did, e.GetDispatcherID()) // Test GetCommitTs - require.Equal(t, common.Ts(0), e.GetCommitTs()) + require.Equal(t, common.Ts(123), e.GetCommitTs()) // Test GetStartTs require.Equal(t, common.Ts(0), e.GetStartTs()) @@ -65,7 +66,7 @@ func TestReadyEventMethods(t *testing.T) { } func TestReadyEventMarshalUnmarshal(t *testing.T) { - normalEvent := NewReadyEvent(common.NewDispatcherID()) + normalEvent := NewReadyEvent(common.NewDispatcherID(), 123) testCases := []struct { name string event *ReadyEvent @@ -108,14 +109,44 @@ func TestReadyEventMarshalUnmarshal(t *testing.T) { require.NoError(t, err) require.Equal(t, tc.event.Version, e2.Version) require.Equal(t, tc.event.DispatcherID, e2.DispatcherID) + require.Equal(t, tc.event.ResolvedTs, e2.ResolvedTs) }) } } +func TestReadyEventDecodeLegacyPayload(t *testing.T) { + did := common.NewDispatcherID() + data, err := MarshalEventWithHeader(TypeReadyEvent, ReadyEventVersion1, did.Marshal()) + require.NoError(t, err) + + var event ReadyEvent + require.NoError(t, event.Unmarshal(data)) + require.Equal(t, did, event.DispatcherID) + require.Zero(t, event.ResolvedTs) +} + +func TestReadyEventLegacyDecoderIgnoresResolvedTs(t *testing.T) { + did := common.NewDispatcherID() + event := NewReadyEvent(did, 123) + data, err := event.Marshal() + require.NoError(t, err) + + payload, version, err := ValidateAndExtractPayload(data, TypeReadyEvent) + require.NoError(t, err) + require.Equal(t, ReadyEventVersion1, version) + + // ReadyEvent's legacy version 1 decoder only unmarshals DispatcherID. Keep + // this assertion to guarantee that appending ResolvedTs remains compatible + // with older nodes during a rolling upgrade. + var legacyDispatcherID common.DispatcherID + require.NoError(t, legacyDispatcherID.Unmarshal(payload)) + require.Equal(t, did, legacyDispatcherID) +} + // TestReadyEventHeader verifies the unified header format func TestReadyEventHeader(t *testing.T) { did := common.NewDispatcherID() - e := NewReadyEvent(did) + e := NewReadyEvent(did, 123) data, err := e.Marshal() require.NoError(t, err) @@ -198,10 +229,10 @@ func TestReadyEventUnmarshalErrors(t *testing.T) { // TestReadyEventSize verifies GetSize calculation func TestReadyEventSize(t *testing.T) { did := common.NewDispatcherID() - e := NewReadyEvent(did) + e := NewReadyEvent(did, 123) // GetSize should only return business data size, not including header - expectedSize := int64(did.GetSize()) + expectedSize := int64(did.GetSize() + 8) require.Equal(t, expectedSize, e.GetSize()) // Marshaled data should include header diff --git a/pkg/config/debug.go b/pkg/config/debug.go index 54a5ba868c..9eda141614 100644 --- a/pkg/config/debug.go +++ b/pkg/config/debug.go @@ -133,7 +133,7 @@ func NewDefaultEventStoreConfig() *EventStoreConfig { return &EventStoreConfig{ CompressionThreshold: 16384, // 16KB EnableZstdCompression: false, - EnableDataSharing: false, + EnableDataSharing: true, } } diff --git a/pkg/eventservice/event_broker.go b/pkg/eventservice/event_broker.go index 542556b88f..97e74ef72a 100644 --- a/pkg/eventservice/event_broker.go +++ b/pkg/eventservice/event_broker.go @@ -595,7 +595,15 @@ func (c *eventBroker) checkAndSendReady(task scanTask) bool { return false } remoteID := node.ID(task.info.GetServerID()) - event := event.NewReadyEvent(task.info.GetID()) + // Use the written (persisted) resolved ts for the ready event, not the + // puller's enqueue watermark: the scan can only serve data that has been + // written to the event store, so reporting the enqueue watermark would + // promise data that is not servable yet. + writtenResolvedTs := c.eventStore.GetSubscriptionWrittenResolvedTs(task.id) + if writtenResolvedTs == 0 { + writtenResolvedTs = task.receivedResolvedTs.Load() + } + event := event.NewReadyEvent(task.info.GetID(), writtenResolvedTs) wrapEvent := newWrapReadyEvent(remoteID, event) c.getMessageCh(task.messageWorkerIndex, common.IsRedoMode(task.info.GetMode())) <- wrapEvent log.Debug("send ready event to dispatcher", diff --git a/pkg/eventservice/event_broker_test.go b/pkg/eventservice/event_broker_test.go index 99498796a7..b7c94b56c0 100644 --- a/pkg/eventservice/event_broker_test.go +++ b/pkg/eventservice/event_broker_test.go @@ -92,6 +92,9 @@ func TestScanRequestCoalescing(t *testing.T) { require.False(t, disp.isScanBusy()) e := <-broker.messageCh[0] require.Equal(t, event.TypeReadyEvent, e.msgType) + ready, ok := e.e.(*event.ReadyEvent) + require.True(t, ok) + require.Equal(t, uint64(102), ready.ResolvedTs) } type scanLifecycleTrackingContext struct { diff --git a/pkg/eventservice/event_service.go b/pkg/eventservice/event_service.go index 5f4656742a..254b481cfb 100644 --- a/pkg/eventservice/event_service.go +++ b/pkg/eventservice/event_service.go @@ -15,6 +15,7 @@ package eventservice import ( "context" + "sync" "time" "github.com/pingcap/log" @@ -80,6 +81,9 @@ type eventService struct { // TODO: use a better way to cache the acceptorInfos dispatcherInfoChan chan DispatcherInfo dispatcherHeartbeat chan *DispatcherHeartBeatWithServerID + handlerMu sync.Mutex + handlerWG sync.WaitGroup + closing bool tz *time.Location } @@ -160,6 +164,11 @@ func (s *eventService) Run(ctx context.Context) error { func (s *eventService) Close(_ context.Context) error { log.Info("event service is closing") + s.handlerMu.Lock() + s.closing = true + s.handlerMu.Unlock() + s.mc.DeRegisterHandler(messaging.EventServiceTopic) + s.handlerWG.Wait() for _, c := range s.brokers { c.close() } @@ -168,6 +177,15 @@ func (s *eventService) Close(_ context.Context) error { } func (s *eventService) handleMessage(ctx context.Context, msg *messaging.TargetMessage) error { + s.handlerMu.Lock() + if s.closing { + s.handlerMu.Unlock() + return nil + } + s.handlerWG.Add(1) + s.handlerMu.Unlock() + defer s.handlerWG.Done() + switch msg.Type { case messaging.TypeDispatcherRequest: infos := msgToDispatcherInfo(msg) diff --git a/pkg/eventservice/event_service_test.go b/pkg/eventservice/event_service_test.go index 445868e0ac..80bb1101d1 100644 --- a/pkg/eventservice/event_service_test.go +++ b/pkg/eventservice/event_service_test.go @@ -201,6 +201,21 @@ func TestHandleMessageIgnoresInvalidSingleMessagePayloads(t *testing.T) { }) } +func TestHandleMessageIgnoredAfterClose(t *testing.T) { + mc := messaging.NewMockMessageCenter() + es := &eventService{ + mc: mc, + brokers: make(map[uint64]*eventBroker), + } + require.NoError(t, es.Close(context.Background())) + + heartbeat := commonEvent.NewDispatcherHeartbeat() + require.NoError(t, es.handleMessage(context.Background(), &messaging.TargetMessage{ + Type: messaging.TypeDispatcherHeartbeat, + Message: []messaging.IOTypeT{heartbeat}, + })) +} + var _ eventstore.EventStore = &mockEventStore{} // mockEventStore is a mock implementation of the EventStore interface @@ -287,6 +302,19 @@ func (m *mockEventStore) Close(ctx context.Context) error { func (m *mockEventStore) UpdateDispatcherCheckpointTs(dispatcherID common.DispatcherID, gcTS uint64) { } +func (m *mockEventStore) DispatcherCount() int { + count := 0 + m.dispatcherMap.Range(func(_, _ any) bool { + count++ + return true + }) + return count +} + +func (m *mockEventStore) GetSubscriptionWrittenResolvedTs(dispatcherID common.DispatcherID) uint64 { + return 0 +} + func (m *mockEventStore) UnregisterDispatcher(changefeedID common.ChangeFeedID, dispatcherID common.DispatcherID) { span, ok := m.dispatcherMap.Load(dispatcherID) if ok { diff --git a/pkg/messaging/remote_target.go b/pkg/messaging/remote_target.go index 852ebe6336..3a108f893b 100644 --- a/pkg/messaging/remote_target.go +++ b/pkg/messaging/remote_target.go @@ -619,31 +619,39 @@ func (s *remoteMessageTarget) handleIncomingMessage(ctx context.Context, stream return err } - mt := IOType(message.Type) - - targetMsg := &TargetMessage{ - From: node.ID(message.From), - To: node.ID(message.To), - Topic: message.Topic, - Type: mt, + targetMsg := s.decodeMessage(message) + if targetMsg != nil { + ch <- targetMsg } + } +} - for _, payload := range message.Payload { - msg, err := decodeIOType(mt, payload) - if err != nil { - log.Error("Failed to decode message", - zap.Error(err), - zap.Stringer("localID", s.messageCenterID), - zap.String("localAddr", s.localAddr), - zap.Stringer("remoteID", s.targetId), - zap.String("remoteAddr", s.targetAddr)) - continue - } - targetMsg.Message = append(targetMsg.Message, msg) - } +func (s *remoteMessageTarget) decodeMessage(message *proto.Message) *TargetMessage { + mt := IOType(message.Type) + targetMsg := &TargetMessage{ + From: node.ID(message.From), + To: node.ID(message.To), + Topic: message.Topic, + Type: mt, + } - ch <- targetMsg + for _, payload := range message.Payload { + msg, err := decodeIOType(mt, payload) + if err != nil { + log.Error("Failed to decode message", + zap.Error(err), + zap.Stringer("localID", s.messageCenterID), + zap.String("localAddr", s.localAddr), + zap.Stringer("remoteID", s.targetId), + zap.String("remoteAddr", s.targetAddr)) + continue + } + targetMsg.Message = append(targetMsg.Message, msg) + } + if len(targetMsg.Message) == 0 { + return nil } + return targetMsg } // Create a new protocol message from target messages diff --git a/pkg/messaging/target_test.go b/pkg/messaging/target_test.go index 6cffb6b114..7b76e3fbc5 100644 --- a/pkg/messaging/target_test.go +++ b/pkg/messaging/target_test.go @@ -18,7 +18,9 @@ import ( "testing" "github.com/pingcap/log" + commonEvent "github.com/pingcap/ticdc/pkg/common/event" "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/ticdc/pkg/messaging/proto" "github.com/pingcap/ticdc/pkg/node" "github.com/stretchr/testify/require" "go.uber.org/zap" @@ -34,6 +36,27 @@ func newRemoteMessageTargetForTest() *remoteMessageTarget { return rt } +func TestDecodeMessageDropsEmptyDecodedPayload(t *testing.T) { + rt := newRemoteMessageTargetForTest() + defer rt.close() + + received := rt.decodeMessage(&proto.Message{ + Type: int32(TypeDispatcherHeartbeat), + Payload: [][]byte{{0}}, + }) + require.Nil(t, received) + + heartbeat := commonEvent.NewDispatcherHeartbeat() + payload, err := heartbeat.Marshal() + require.NoError(t, err) + received = rt.decodeMessage(&proto.Message{ + Type: int32(TypeDispatcherHeartbeat), + Payload: [][]byte{payload}, + }) + require.NotNil(t, received) + require.Len(t, received.Message, 1) +} + func TestRemoteTargetNewMessage(t *testing.T) { rt := newRemoteMessageTargetForTest() defer rt.close()