From 0d1c03e28deec1c778cbb98c68bb03170107dc6f Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Mon, 17 Aug 2026 11:47:31 +0800 Subject: [PATCH 1/6] Revert "kafka: decouple batch size from Kafka message size limit (#5420) (#5772)" This reverts commit 19ff1fc4faef292e51136bcab13c7e80ae86dc73. --- .../sink/cloudstorage/encoder_group_test.go | 1 - downstreamadapter/sink/cloudstorage/sink.go | 8 +- downstreamadapter/sink/helper/helper.go | 9 +- downstreamadapter/sink/kafka/helper.go | 5 +- downstreamadapter/sink/kafka/sink.go | 50 +++- downstreamadapter/sink/kafka/sink_test.go | 18 +- downstreamadapter/sink/pulsar/helper.go | 5 +- .../sink/topicmanager/kafka_topic_manager.go | 74 +++--- .../topicmanager/kafka_topic_manager_test.go | 4 +- pkg/config/large_message.go | 20 +- pkg/config/large_message_test.go | 215 ----------------- .../codec/canal/canal_json_txn_encoder.go | 1 + pkg/sink/codec/common/config.go | 30 +-- pkg/sink/codec/common/config_test.go | 85 ------- pkg/sink/codec/open/encoder.go | 4 +- pkg/sink/codec/open/encoder_test.go | 72 +----- pkg/sink/kafka/admin.go | 21 +- pkg/sink/kafka/claimcheck/claim_check.go | 2 +- pkg/sink/kafka/options.go | 223 +++++++++--------- pkg/sink/kafka/options_test.go | 169 ++++--------- pkg/sink/kafka/sarama_async_producer.go | 15 +- pkg/sink/kafka/sarama_config.go | 76 +++--- pkg/sink/kafka/sarama_config_test.go | 58 ----- pkg/sink/kafka/sarama_factory.go | 45 +--- pkg/sink/kafka/sarama_sync_producer.go | 12 +- tests/integration_tests/_utils/kafka_topic | 12 - .../canal_json_claim_check/run.sh | 9 +- .../canal_json_handle_key_only/run.sh | 1 - .../kafka_big_messages/conf/diff_config.toml | 2 +- .../kafka_big_messages/run.sh | 223 +++--------------- .../kafka_compression/run.sh | 5 + .../kafka_simple_claim_check/data/data.sql | 6 +- .../kafka_simple_claim_check/run.sh | 7 - .../data/data.sql | 6 +- .../kafka_simple_claim_check_avro/run.sh | 7 - .../kafka_simple_handle_key_only/run.sh | 1 - .../kafka_simple_handle_key_only_avro/run.sh | 1 - tests/integration_tests/log_redaction/run.sh | 91 +++++++ .../open_protocol_claim_check/data/data.sql | 1 - .../open_protocol_claim_check/run.sh | 9 +- .../data/data.sql | 1 - .../open_protocol_handle_key_only/run.sh | 1 - tests/integration_tests/run_light_it_in_ci.sh | 2 +- tests/utils/kafka_topic/main.go | 68 ------ 44 files changed, 501 insertions(+), 1174 deletions(-) delete mode 100644 pkg/config/large_message_test.go delete mode 100644 pkg/sink/codec/common/config_test.go delete mode 100755 tests/integration_tests/_utils/kafka_topic delete mode 100644 tests/utils/kafka_topic/main.go diff --git a/downstreamadapter/sink/cloudstorage/encoder_group_test.go b/downstreamadapter/sink/cloudstorage/encoder_group_test.go index f5df7b6454..e77cfc59bb 100644 --- a/downstreamadapter/sink/cloudstorage/encoder_group_test.go +++ b/downstreamadapter/sink/cloudstorage/encoder_group_test.go @@ -230,7 +230,6 @@ func newTestTxnEncoderConfig(t *testing.T) *common.Config { config.ProtocolCsv, replicaConfig.Sink, config.DefaultMaxMessageBytes, - config.DefaultMaxMessageBytes, ) require.NoError(t, err) return encoderConfig diff --git a/downstreamadapter/sink/cloudstorage/sink.go b/downstreamadapter/sink/cloudstorage/sink.go index 0766816315..ff6920307a 100644 --- a/downstreamadapter/sink/cloudstorage/sink.go +++ b/downstreamadapter/sink/cloudstorage/sink.go @@ -86,7 +86,7 @@ func Verify(ctx context.Context, changefeedID common.ChangeFeedID, sinkURI *url. if err != nil { return err } - _, err = helper.GetEncoderConfig(changefeedID, sinkURI, protocol, sinkConfig, math.MaxInt, math.MaxInt) + _, err = helper.GetEncoderConfig(changefeedID, sinkURI, protocol, sinkConfig, math.MaxInt) if err != nil { return err } @@ -116,9 +116,9 @@ func New( } // get cloud storage file extension according to the specific protocol. ext := helper.GetFileExtension(protocol) - // Message size limits are mainly for MQ batch protocols. Cloud storage uses - // max int for both the final message limit and the batch threshold. - encoderConfig, err := helper.GetEncoderConfig(changefeedID, sinkURI, protocol, sinkConfig, math.MaxInt, math.MaxInt) + // the last param maxMsgBytes is mainly to limit the size of a single message for + // batch protocols in mq scenario. In cloud storage sink, we just set it to max int. + encoderConfig, err := helper.GetEncoderConfig(changefeedID, sinkURI, protocol, sinkConfig, math.MaxInt) if err != nil { return nil, err } diff --git a/downstreamadapter/sink/helper/helper.go b/downstreamadapter/sink/helper/helper.go index 3911ea6ab5..47cd949220 100644 --- a/downstreamadapter/sink/helper/helper.go +++ b/downstreamadapter/sink/helper/helper.go @@ -50,16 +50,17 @@ func GetEncoderConfig( sinkURI *url.URL, protocol config.Protocol, sinkConfig *config.SinkConfig, - maxMessageBytes int, - maxBatchedBytes int, + maxMsgBytes int, ) (*common.Config, error) { encoderConfig := common.NewConfig(protocol) if err := encoderConfig.Apply(sinkURI, sinkConfig); err != nil { return nil, errors.WrapError(errors.ErrSinkInvalidConfig, err) } + // Always set encoder's `MaxMessageBytes` equal to producer's `MaxMessageBytes` + // to prevent that the encoder generate batched message too large + // then cause producer meet `message too large`. encoderConfig = encoderConfig. - WithMaxMessageBytes(maxMessageBytes). - WithMaxBatchedBytes(maxBatchedBytes). + WithMaxMessageBytes(maxMsgBytes). WithChangefeedID(changefeedID) tz, err := util.GetTimezone(config.GetGlobalServerConfig().TZ) diff --git a/downstreamadapter/sink/kafka/helper.go b/downstreamadapter/sink/kafka/helper.go index ebd5b915e4..733278a59a 100644 --- a/downstreamadapter/sink/kafka/helper.go +++ b/downstreamadapter/sink/kafka/helper.go @@ -102,10 +102,7 @@ func newKafkaSinkComponent( return comp, protocol, err } - encoderConfig, err := helper.GetEncoderConfig( - changefeedID, sinkURI, protocol, sinkConfig, - options.MaxMessageBytes, options.MaxBatchedBytes, - ) + encoderConfig, err := helper.GetEncoderConfig(changefeedID, sinkURI, protocol, sinkConfig, options.MaxMessageBytes) if err != nil { return comp, protocol, err } diff --git a/downstreamadapter/sink/kafka/sink.go b/downstreamadapter/sink/kafka/sink.go index f6351fb269..76e0253f2e 100644 --- a/downstreamadapter/sink/kafka/sink.go +++ b/downstreamadapter/sink/kafka/sink.go @@ -89,10 +89,7 @@ func Verify(ctx context.Context, changefeedID common.ChangeFeedID, uri *url.URL, } options.Topic = topic - encoderConfig, err := helper.GetEncoderConfig( - changefeedID, uri, protocol, sinkConfig, - options.MaxMessageBytes, options.MaxBatchedBytes, - ) + encoderConfig, err := helper.GetEncoderConfig(changefeedID, uri, protocol, sinkConfig, options.MaxMessageBytes) if err != nil { return err } @@ -238,7 +235,7 @@ func (s *sink) WriteBlockEvent(event commonEvent.BlockEvent) error { case *commonEvent.DDLEvent: err = s.sendDDLEvent(v) default: - log.Error("unsupported kafka sink block event type", + log.Error("kafka sink doesn't support this type of block event", zap.String("namespace", s.changefeedID.Keyspace()), zap.String("changefeed", s.changefeedID.Name()), zap.String("eventType", commonEvent.TypeToString(event.GetType()))) @@ -292,6 +289,9 @@ func (s *sink) calculateKeyPartitions(ctx context.Context) error { default: event, ok := s.eventChan.Get() if !ok { + log.Info("kafka sink event channel closed", + zap.String("keyspace", s.changefeedID.Keyspace()), + zap.String("changefeed", s.changefeedID.Name())) return nil } schema := event.TableInfo.GetSchemaName() @@ -351,6 +351,9 @@ func (s *sink) nonBatchEncodeRun(ctx context.Context) error { default: event, ok := s.rowChan.Get() if !ok { + log.Info("kafka sink event channel closed", + zap.String("keyspace", s.changefeedID.Keyspace()), + zap.String("changefeed", s.changefeedID.Name())) return nil } if err := s.comp.encoderGroup.AddEvents(ctx, event.Key, &event.RowEvent); err != nil { @@ -373,6 +376,10 @@ func (s *sink) batchEncodeRun(ctx context.Context) error { start := time.Now() msgs, err := s.batch(ctx, msgsBuf) if err != nil { + log.Error("kafka sink batch dml events failed", + zap.String("keyspace", s.changefeedID.Keyspace()), + zap.String("changefeed", s.changefeedID.Name()), + zap.Error(err)) return err } if len(msgs) == 0 { @@ -402,6 +409,9 @@ func (s *sink) batch(ctx context.Context, buffer []*commonEvent.MQRowEvent) ([]* default: msgs, ok := s.rowChan.GetMultipleNoGroup(buffer) if !ok { + log.Info("kafka sink event channel closed", + zap.String("keyspace", s.changefeedID.Keyspace()), + zap.String("changefeed", s.changefeedID.Name())) return nil, nil } buffer = buffer[:0] @@ -433,6 +443,9 @@ func (s *sink) sendMessages(ctx context.Context) error { return context.Cause(ctx) case future, ok := <-outCh: if !ok { + log.Info("kafka sink encoder's output channel closed", + zap.String("keyspace", s.changefeedID.Keyspace()), + zap.String("changefeed", s.changefeedID.Name())) return nil } if err = future.Ready(ctx); err != nil { @@ -442,11 +455,16 @@ func (s *sink) sendMessages(ctx context.Context) error { start := time.Now() if err = s.statistics.RecordBatchExecution(func() (int, int64, error) { message.SetPartitionKey(future.Key.PartitionKey) + log.Debug("send message to kafka", zap.String("messageKey", util.RedactBytes(message.Key)), zap.String("messageValue", util.RedactBytes(message.Value))) if err = s.dmlProducer.AsyncSend( ctx, future.Key.Topic, future.Key.Partition, message); err != nil { + log.Error("kafka sink send message failed", + zap.String("keyspace", s.changefeedID.Keyspace()), + zap.String("changefeed", s.changefeedID.Name()), + zap.Error(err)) return 0, 0, err } return message.GetRowsCount(), int64(message.Length()), nil @@ -466,10 +484,9 @@ func (s *sink) sendDDLEvent(event *commonEvent.DDLEvent) error { return err } if message == nil { - log.Info("kafka ddl event skipped", - zap.String("keyspace", s.changefeedID.Keyspace()), zap.String("changefeed", s.changefeedID.Name()), - zap.Uint64("startTs", e.GetStartTs()), zap.Uint64("commitTs", e.GetCommitTs()), - zap.String("query", e.Query)) + log.Info("Skip ddl event", zap.Uint64("startTs", event.GetStartTs()), zap.Uint64("commitTs", e.GetCommitTs()), + zap.String("query", e.Query), + zap.Stringer("changefeed", s.changefeedID)) continue } codecCommon.SetDDLMessageLogInfo(message, e) @@ -495,11 +512,11 @@ func (s *sink) sendDDLEvent(event *commonEvent.DDLEvent) error { if err != nil { return err } - log.Info("kafka ddl event sent", - zap.String("keyspace", s.changefeedID.Keyspace()), zap.String("changefeed", s.changefeedID.Name()), - zap.Uint64("startTs", e.GetStartTs()), zap.Uint64("commitTs", e.GetCommitTs()), - zap.String("query", e.GetDDLQuery())) } + log.Info("kafka sink send DDL event", + zap.String("keyspace", s.changefeedID.Keyspace()), zap.String("changefeed", s.changefeedID.Name()), + zap.Any("startTs", event.GetStartTs()), zap.Any("commitTs", event.GetCommitTs()), zap.Any("event", event.GetDDLQuery()), + zap.String("schema", event.GetSchemaName()), zap.String("table", event.GetTableName())) return nil } @@ -532,6 +549,9 @@ func (s *sink) sendCheckpoint(ctx context.Context) error { return context.Cause(ctx) case ts, ok := <-s.checkpointChan: if !ok { + log.Warn("kafka sink checkpoint channel closed", + zap.String("keyspace", s.changefeedID.Keyspace()), + zap.String("changefeed", s.changefeedID.Name())) return nil } @@ -584,6 +604,10 @@ func (s *sink) SetTableSchemaStore(tableSchemaStore *commonEvent.TableSchemaStor func (s *sink) getAllTableNames(ts uint64) []*commonEvent.SchemaTableName { if s.tableSchemaStore == nil { + log.Warn("kafka sink table schema store is not set", + zap.String("keyspace", s.changefeedID.Keyspace()), + zap.String("changefeed", s.changefeedID.Name()), + zap.Uint64("ts", ts)) return nil } return s.tableSchemaStore.GetAllTableNames(ts) diff --git a/downstreamadapter/sink/kafka/sink_test.go b/downstreamadapter/sink/kafka/sink_test.go index 9c6e49b508..330b981908 100644 --- a/downstreamadapter/sink/kafka/sink_test.go +++ b/downstreamadapter/sink/kafka/sink_test.go @@ -41,19 +41,6 @@ import ( const kafkaSinkTestTopic = "mock_topic" -func TestVerifyValidatesEncoderConfigBeforeKafkaConnection(t *testing.T) { - openProtocol := config.ProtocolOpen.String() - sinkConfig := &config.SinkConfig{Protocol: &openProtocol} - sinkURI, err := url.Parse("kafka://127.0.0.1:1/" + kafkaSinkTestTopic + "?max-batch-size=0") - require.NoError(t, err) - - changefeedID := common.NewChangefeedID4Test("test", "verify-existing-topic") - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - err = Verify(ctx, changefeedID, sinkURI, sinkConfig) - require.ErrorContains(t, err, "invalid max-batch-size 0") -} - func TestSinkWorkersReturnContextError(t *testing.T) { contexts := []struct { name string @@ -197,10 +184,7 @@ func newKafkaSinkForTestWithProducers(ctx context.Context, if err != nil { return nil, err } - encoderConfig, err := helper.GetEncoderConfig( - changefeedID, sinkURI, protocol, sinkConfig, - options.MaxMessageBytes, options.MaxBatchedBytes, - ) + encoderConfig, err := helper.GetEncoderConfig(changefeedID, sinkURI, protocol, sinkConfig, options.MaxMessageBytes) if err != nil { return nil, err } diff --git a/downstreamadapter/sink/pulsar/helper.go b/downstreamadapter/sink/pulsar/helper.go index a2d82f7af6..ffb503a8f3 100644 --- a/downstreamadapter/sink/pulsar/helper.go +++ b/downstreamadapter/sink/pulsar/helper.go @@ -122,10 +122,7 @@ func newPulsarSinkComponentWithFactory(ctx context.Context, return pulsarComponent, protocol, errors.Trace(err) } - encoderConfig, err := helper.GetEncoderConfig( - changefeedID, sinkURI, protocol, sinkConfig, - config.DefaultMaxMessageBytes, config.DefaultMaxMessageBytes, - ) + encoderConfig, err := helper.GetEncoderConfig(changefeedID, sinkURI, protocol, sinkConfig, config.DefaultMaxMessageBytes) if err != nil { return pulsarComponent, protocol, errors.Trace(err) } diff --git a/downstreamadapter/sink/topicmanager/kafka_topic_manager.go b/downstreamadapter/sink/topicmanager/kafka_topic_manager.go index e436e8801a..4ccd04eea8 100644 --- a/downstreamadapter/sink/topicmanager/kafka_topic_manager.go +++ b/downstreamadapter/sink/topicmanager/kafka_topic_manager.go @@ -121,6 +121,10 @@ func (m *kafkaTopicManager) backgroundRefreshMeta(ctx context.Context) { for { select { case <-ctx.Done(): + log.Info("Background refresh Kafka metadata goroutine exit.", + zap.String("keyspace", m.changefeedID.Keyspace()), + zap.String("changefeed", m.changefeedID.Name()), + ) return case <-ticker.C: // We ignore the error here, because the error may be caused by the @@ -140,16 +144,23 @@ func (m *kafkaTopicManager) tryUpdatePartitionsAndLogging(topic string, partitio if oldPartitions.(int32) != partitions { m.topics.Store(topic, partitions) log.Info( - "kafka topic partition count changed", + "update topic partition number", zap.String("keyspace", m.changefeedID.Keyspace()), zap.String("changefeed", m.changefeedID.Name()), zap.String("topic", topic), - zap.Int32("oldPartitionNum", oldPartitions.(int32)), - zap.Int32("newPartitionNum", partitions), + zap.Int32("oldPartitionNumber", oldPartitions.(int32)), + zap.Int32("newPartitionNumber", partitions), ) } } else { m.topics.Store(topic, partitions) + log.Info( + "store topic partition number", + zap.String("keyspace", m.changefeedID.Keyspace()), + zap.String("changefeed", m.changefeedID.Name()), + zap.String("topic", topic), + zap.Int32("partitionNumber", partitions), + ) } } @@ -168,7 +179,7 @@ func (m *kafkaTopicManager) fetchAllTopicsPartitionsNum() (map[string]int32, err numPartitions, err := m.admin.GetTopicsPartitionsNum(topics) if err != nil { log.Warn( - "kafka topic metadata refresh failed", + "Kafka admin client describe topics failed", zap.String("keyspace", m.changefeedID.Keyspace()), zap.String("changefeed", m.changefeedID.Name()), zap.Duration("duration", time.Since(start)), @@ -197,32 +208,33 @@ func (m *kafkaTopicManager) waitUntilTopicVisible( ctx context.Context, topicName string, ) error { - start := time.Now() topics := []string{topicName} err := retry.Do(ctx, func() error { + start := time.Now() // ignoreTopicError is set to false since we just create the topic, // make sure the topic is visible. meta, err := m.admin.GetTopicsMeta(topics, false) if err != nil { + log.Warn("topic not found, retry it", + zap.String("keyspace", m.changefeedID.Keyspace()), + zap.String("changefeed", m.changefeedID.Name()), + zap.Error(err), + zap.Duration("duration", time.Since(start)), + ) return err } - _, ok := meta[topicName] - if !ok { - return errors.ErrKafkaAdminAPI.GenWithStackByArgs("describe-topic", topicName) - } + log.Info("topic found", + zap.String("keyspace", m.changefeedID.Keyspace()), + zap.String("changefeed", m.changefeedID.Name()), + zap.String("topic", topicName), + zap.Int32("partitionNumber", meta[topicName].NumPartitions), + zap.Duration("duration", time.Since(start))) return nil }, retry.WithBackoffBaseDelay(500), retry.WithBackoffMaxDelay(1000), retry.WithMaxTries(6), ) - if err != nil { - log.Warn("kafka topic metadata refresh failed", - zap.String("keyspace", m.changefeedID.Keyspace()), - zap.String("changefeed", m.changefeedID.Name()), - zap.String("topic", topicName), - zap.Duration("duration", time.Since(start)), - zap.Error(err)) - } + return err } @@ -248,11 +260,11 @@ func (m *kafkaTopicManager) createTopic( }) if err != nil { log.Error( - "kafka topic creation failed", + "Kafka admin client create the topic failed", zap.String("keyspace", m.changefeedID.Keyspace()), zap.String("changefeed", m.changefeedID.Name()), zap.String("topic", topicName), - zap.Int32("partitionNum", m.cfg.PartitionNum), + zap.Int32("partitionNumber", m.cfg.PartitionNum), zap.Int16("replicationFactor", m.cfg.ReplicationFactor), zap.Error(err), zap.Duration("duration", time.Since(start)), @@ -260,6 +272,15 @@ func (m *kafkaTopicManager) createTopic( return 0, err } + log.Info( + "Kafka admin client create the topic success", + zap.String("keyspace", m.changefeedID.Keyspace()), + zap.String("changefeed", m.changefeedID.Name()), + zap.String("topic", topicName), + zap.Int32("partitionNumber", m.cfg.PartitionNum), + zap.Int16("replicationFactor", m.cfg.ReplicationFactor), + zap.Duration("duration", time.Since(start)), + ) m.tryUpdatePartitionsAndLogging(topicName, m.cfg.PartitionNum) return m.cfg.PartitionNum, nil @@ -295,7 +316,6 @@ func (m *kafkaTopicManager) CreateTopicAndWaitUntilVisible( return numPartition, nil } - start := time.Now() partitionNum, err := m.createTopic(ctx, topicName) if err != nil { if kafka.IsAdminAuthorizationFailed(err) { @@ -309,16 +329,6 @@ func (m *kafkaTopicManager) CreateTopicAndWaitUntilVisible( return 0, err } - log.Info( - "kafka topic created", - zap.String("keyspace", m.changefeedID.Keyspace()), - zap.String("changefeed", m.changefeedID.Name()), - zap.String("topic", topicName), - zap.Int32("partitionNum", partitionNum), - zap.Int16("replicationFactor", m.cfg.ReplicationFactor), - zap.Duration("duration", time.Since(start)), - ) - return partitionNum, nil } @@ -338,11 +348,11 @@ func (m *kafkaTopicManager) tryStoreTopicMeta( } func (m *kafkaTopicManager) useConfiguredPartitionNum(topicName string, cause error) int32 { - log.Warn("kafka topic creation skipped due to authorization failure", + log.Warn("skip Kafka topic creation because topic authorization failed", zap.String("keyspace", m.changefeedID.Keyspace()), zap.String("changefeed", m.changefeedID.Name()), zap.String("topic", topicName), - zap.Int32("partitionNum", m.cfg.PartitionNum), + zap.Int32("partitionNumber", m.cfg.PartitionNum), zap.Error(cause)) m.tryUpdatePartitionsAndLogging(topicName, m.cfg.PartitionNum) return m.cfg.PartitionNum diff --git a/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go b/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go index b00bf947d0..b8e93c588e 100644 --- a/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go +++ b/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go @@ -234,7 +234,9 @@ func TestEnsureTopicExistsWaitsUntilVisible(t *testing.T) { return nil }), adminClient.EXPECT().GetTopicsMeta([]string{topic}, false).Return( - map[string]kafka.TopicDetail{}, nil), + nil, sarama.ErrUnknownTopicOrPartition), + adminClient.EXPECT().GetTopicsMeta([]string{topic}, false).Return( + nil, sarama.ErrUnknownTopicOrPartition), adminClient.EXPECT().GetTopicsMeta([]string{topic}, false).Return( map[string]kafka.TopicDetail{ topic: { diff --git a/pkg/config/large_message.go b/pkg/config/large_message.go index 6b19afe260..d04584b451 100644 --- a/pkg/config/large_message.go +++ b/pkg/config/large_message.go @@ -15,7 +15,7 @@ package config import ( "github.com/pingcap/ticdc/pkg/compression" - "github.com/pingcap/ticdc/pkg/errors" + cerror "github.com/pingcap/ticdc/pkg/errors" ) const ( @@ -55,39 +55,34 @@ func (c *LargeMessageHandleConfig) AdjustAndValidate(protocol Protocol, enableTi // compression can be enabled independently if !compression.Supported(c.LargeMessageHandleCompression) { - return errors.ErrInvalidReplicaConfig.GenWithStack( + return cerror.ErrInvalidReplicaConfig.GenWithStack( "large message handle compression is not supported, got %s", c.LargeMessageHandleCompression) } if c.LargeMessageHandleOption == LargeMessageHandleOptionNone { return nil } - if c.LargeMessageHandleOption != LargeMessageHandleOptionClaimCheck && - c.LargeMessageHandleOption != LargeMessageHandleOptionHandleKeyOnly { - return errors.ErrInvalidReplicaConfig.GenWithStack( - "unknown large-message-handle-option %s", c.LargeMessageHandleOption) - } switch protocol { case ProtocolOpen, ProtocolSimple: case ProtocolCanalJSON: if !enableTiDBExtension { - return errors.ErrInvalidReplicaConfig.GenWithStack( + return cerror.ErrInvalidReplicaConfig.GenWithStack( "large message handle is set to %s, protocol is %s, but enable-tidb-extension is false", c.LargeMessageHandleOption, protocol.String()) } default: - return errors.ErrInvalidReplicaConfig.GenWithStack( + return cerror.ErrInvalidReplicaConfig.GenWithStack( "large message handle is set to %s, protocol is %s, it's not supported", c.LargeMessageHandleOption, protocol.String()) } if c.LargeMessageHandleOption == LargeMessageHandleOptionClaimCheck { if c.ClaimCheckStorageURI == "" { - return errors.ErrInvalidReplicaConfig.GenWithStack( + return cerror.ErrInvalidReplicaConfig.GenWithStack( "large message handle is set to claim-check, but the claim-check-storage-uri is empty") } if c.ClaimCheckRawValue && protocol == ProtocolOpen { - return errors.ErrInvalidReplicaConfig.GenWithStack( + return cerror.ErrInvalidReplicaConfig.GenWithStack( "large message handle is set to claim-check, raw value is not supported for the open protocol") } } @@ -111,8 +106,7 @@ func (c *LargeMessageHandleConfig) EnableClaimCheck() bool { return c.LargeMessageHandleOption == LargeMessageHandleOptionClaimCheck } -// Disabled returns true only when large message handling is explicitly disabled. -// It returns false for nil and unknown configurations. +// Disabled returns true if disable large message handle. func (c *LargeMessageHandleConfig) Disabled() bool { if c == nil { return false diff --git a/pkg/config/large_message_test.go b/pkg/config/large_message_test.go deleted file mode 100644 index f0721f4b53..0000000000 --- a/pkg/config/large_message_test.go +++ /dev/null @@ -1,215 +0,0 @@ -// Copyright 2023 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package config - -import ( - "testing" - - "github.com/pingcap/ticdc/pkg/compression" - cerror "github.com/pingcap/ticdc/pkg/errors" - "github.com/stretchr/testify/require" -) - -func TestLargeMessageHandle4Compression(t *testing.T) { - t.Parallel() - - largeMessageHandle := NewDefaultLargeMessageHandleConfig() - - // unsupported compression, return error - largeMessageHandle.LargeMessageHandleCompression = "zstd" - - err := largeMessageHandle.AdjustAndValidate(ProtocolCanalJSON, false) - require.ErrorIs(t, err, cerror.ErrInvalidReplicaConfig) - - largeMessageHandle.LargeMessageHandleCompression = compression.LZ4 - err = largeMessageHandle.AdjustAndValidate(ProtocolCanalJSON, false) - require.NoError(t, err) - - largeMessageHandle.LargeMessageHandleCompression = compression.Snappy - err = largeMessageHandle.AdjustAndValidate(ProtocolCanalJSON, false) - require.NoError(t, err) - - largeMessageHandle.LargeMessageHandleCompression = compression.None - err = largeMessageHandle.AdjustAndValidate(ProtocolCanalJSON, false) - require.NoError(t, err) -} - -func TestLargeMessageHandle4NotSupportedProtocol(t *testing.T) { - t.Parallel() - - largeMessageHandle := NewDefaultLargeMessageHandleConfig() - - err := largeMessageHandle.AdjustAndValidate(ProtocolCanal, true) - require.NoError(t, err) - - largeMessageHandle.LargeMessageHandleOption = LargeMessageHandleOptionHandleKeyOnly - err = largeMessageHandle.AdjustAndValidate(ProtocolCanal, true) - require.ErrorIs(t, err, cerror.ErrInvalidReplicaConfig) -} - -func TestLargeMessageHandleRejectsUnknownOption(t *testing.T) { - t.Parallel() - - largeMessageHandle := NewDefaultLargeMessageHandleConfig() - largeMessageHandle.LargeMessageHandleOption = "unknown" - - require.False(t, largeMessageHandle.Disabled()) - err := largeMessageHandle.AdjustAndValidate(ProtocolOpen, false) - require.ErrorIs(t, err, cerror.ErrInvalidReplicaConfig) - require.ErrorContains(t, err, "unknown large-message-handle-option unknown") -} - -func TestHandleKeyOnly4CanalJSON(t *testing.T) { - t.Parallel() - - // large-message-handle not set, always no error - largeMessageHandle := NewDefaultLargeMessageHandleConfig() - - err := largeMessageHandle.AdjustAndValidate(ProtocolCanalJSON, false) - require.NoError(t, err) - require.True(t, largeMessageHandle.Disabled()) - - largeMessageHandle.LargeMessageHandleOption = LargeMessageHandleOptionHandleKeyOnly - - // `enable-tidb-extension` is false, return error - err = largeMessageHandle.AdjustAndValidate(ProtocolCanalJSON, false) - require.ErrorIs(t, err, cerror.ErrInvalidReplicaConfig) - - // `enable-tidb-extension` is true, no error - err = largeMessageHandle.AdjustAndValidate(ProtocolCanalJSON, true) - require.NoError(t, err) - require.Equal(t, LargeMessageHandleOptionHandleKeyOnly, largeMessageHandle.LargeMessageHandleOption) -} - -func TestClaimCheck4CanalJSON(t *testing.T) { - t.Parallel() - - // large-message-handle not set, always no error - largeMessageHandle := NewDefaultLargeMessageHandleConfig() - - err := largeMessageHandle.AdjustAndValidate(ProtocolCanalJSON, false) - require.NoError(t, err) - require.True(t, largeMessageHandle.Disabled()) - - largeMessageHandle.LargeMessageHandleOption = LargeMessageHandleOptionClaimCheck - largeMessageHandle.ClaimCheckStorageURI = "file:///tmp/claim-check" - - for _, rawValue := range []bool{false, true} { - largeMessageHandle.ClaimCheckRawValue = rawValue - // `enable-tidb-extension` is false, return error - err = largeMessageHandle.AdjustAndValidate(ProtocolCanalJSON, false) - require.ErrorIs(t, err, cerror.ErrInvalidReplicaConfig) - - // `enable-tidb-extension` is true, no error - err = largeMessageHandle.AdjustAndValidate(ProtocolCanalJSON, true) - require.NoError(t, err) - require.Equal(t, LargeMessageHandleOptionClaimCheck, largeMessageHandle.LargeMessageHandleOption) - } -} - -func TestHandleKeyOnly4OpenProtocol(t *testing.T) { - t.Parallel() - - // large-message-handle not set, always no error - largeMessageHandle := NewDefaultLargeMessageHandleConfig() - - err := largeMessageHandle.AdjustAndValidate(ProtocolOpen, false) - require.NoError(t, err) - require.True(t, largeMessageHandle.Disabled()) - - largeMessageHandle.LargeMessageHandleOption = LargeMessageHandleOptionHandleKeyOnly - // `enable-tidb-extension` is false, return error - err = largeMessageHandle.AdjustAndValidate(ProtocolOpen, false) - require.NoError(t, err) - - // `enable-tidb-extension` is true, no error - err = largeMessageHandle.AdjustAndValidate(ProtocolOpen, true) - require.NoError(t, err) - require.Equal(t, LargeMessageHandleOptionHandleKeyOnly, largeMessageHandle.LargeMessageHandleOption) -} - -func TestClaimCheck4OpenProtocol(t *testing.T) { - t.Parallel() - - // large-message-handle not set, always no error - largeMessageHandle := NewDefaultLargeMessageHandleConfig() - - err := largeMessageHandle.AdjustAndValidate(ProtocolOpen, false) - require.NoError(t, err) - require.True(t, largeMessageHandle.Disabled()) - - largeMessageHandle.LargeMessageHandleOption = LargeMessageHandleOptionClaimCheck - largeMessageHandle.ClaimCheckStorageURI = "file:///tmp/claim-check" - - // `enable-tidb-extension` is false, return error - err = largeMessageHandle.AdjustAndValidate(ProtocolOpen, false) - require.NoError(t, err) - - // `enable-tidb-extension` is true, no error - err = largeMessageHandle.AdjustAndValidate(ProtocolOpen, true) - require.NoError(t, err) - require.Equal(t, LargeMessageHandleOptionClaimCheck, largeMessageHandle.LargeMessageHandleOption) - - largeMessageHandle.ClaimCheckRawValue = true - err = largeMessageHandle.AdjustAndValidate(ProtocolOpen, true) - require.ErrorIs(t, err, cerror.ErrInvalidReplicaConfig) -} - -func TestHandleKeyOnly4SimpleProtocol(t *testing.T) { - t.Parallel() - - // large-message-handle not set, always no error - largeMessageHandle := NewDefaultLargeMessageHandleConfig() - - err := largeMessageHandle.AdjustAndValidate(ProtocolSimple, false) - require.NoError(t, err) - require.True(t, largeMessageHandle.Disabled()) - - largeMessageHandle.LargeMessageHandleOption = LargeMessageHandleOptionHandleKeyOnly - // `enable-tidb-extension` is false, return error - err = largeMessageHandle.AdjustAndValidate(ProtocolSimple, false) - require.NoError(t, err) - - // `enable-tidb-extension` is true, no error - err = largeMessageHandle.AdjustAndValidate(ProtocolSimple, true) - require.NoError(t, err) - require.Equal(t, LargeMessageHandleOptionHandleKeyOnly, largeMessageHandle.LargeMessageHandleOption) -} - -func TestClaimCheck4SimpleProtocol(t *testing.T) { - t.Parallel() - - // large-message-handle not set, always no error - largeMessageHandle := NewDefaultLargeMessageHandleConfig() - - err := largeMessageHandle.AdjustAndValidate(ProtocolSimple, false) - require.NoError(t, err) - require.True(t, largeMessageHandle.Disabled()) - - largeMessageHandle.LargeMessageHandleOption = LargeMessageHandleOptionClaimCheck - largeMessageHandle.ClaimCheckStorageURI = "file:///tmp/claim-check" - - // `enable-tidb-extension` is false, return error - err = largeMessageHandle.AdjustAndValidate(ProtocolSimple, false) - require.NoError(t, err) - - // `enable-tidb-extension` is true, no error - err = largeMessageHandle.AdjustAndValidate(ProtocolSimple, true) - require.NoError(t, err) - require.Equal(t, LargeMessageHandleOptionClaimCheck, largeMessageHandle.LargeMessageHandleOption) - - largeMessageHandle.ClaimCheckRawValue = true - err = largeMessageHandle.AdjustAndValidate(ProtocolSimple, true) - require.NoError(t, err) -} diff --git a/pkg/sink/codec/canal/canal_json_txn_encoder.go b/pkg/sink/codec/canal/canal_json_txn_encoder.go index 3e10d98229..0af6f4f3f2 100644 --- a/pkg/sink/codec/canal/canal_json_txn_encoder.go +++ b/pkg/sink/codec/canal/canal_json_txn_encoder.go @@ -65,6 +65,7 @@ func (j *JSONTxnEventEncoder) AppendTxnEvent(event *commonEvent.DMLEvent) error return err } length := len(value) + common.MaxRecordOverhead + // For single message that is longer than max-message-bytes, do not send it. if length > j.config.MaxMessageBytes { log.Warn("Single message is too large for canal-json", zap.Int("maxMessageBytes", j.config.MaxMessageBytes), diff --git a/pkg/sink/codec/common/config.go b/pkg/sink/codec/common/config.go index 415e864a05..0033f4acc3 100644 --- a/pkg/sink/codec/common/config.go +++ b/pkg/sink/codec/common/config.go @@ -41,12 +41,9 @@ type Config struct { Protocol config.Protocol + // control batch behavior, only for `open-protocol` and `craft` at the moment. MaxMessageBytes int - - // MaxBatchedBytes controls open-protocol encoder's maximum number of bytes for a batched message. - MaxBatchedBytes int - // MaxBatchedSize controls open-protocol encoder's maximum number of events for a batched message. - MaxBatchSize int + MaxBatchSize int // DeleteOnlyHandleKeyColumns is true, for the delete event only output the handle key columns. DeleteOnlyHandleKeyColumns bool @@ -117,7 +114,6 @@ func NewConfig(protocol config.Protocol) *Config { Protocol: protocol, MaxMessageBytes: config.DefaultMaxMessageBytes, - MaxBatchedBytes: config.DefaultMaxMessageBytes, MaxBatchSize: defaultMaxBatchSize, EnableTiDBExtension: false, @@ -197,7 +193,7 @@ func (c *Config) Apply(sinkURI *url.URL, sinkConfig *config.SinkConfig) error { var err error urlParameter := &urlConfig{} if err = binding.Query.Bind(req, urlParameter); err != nil { - return errors.WrapError(errors.ErrSinkInvalidConfig, err) + return errors.WrapError(errors.ErrMySQLInvalidConfig, err) } if urlParameter, err = mergeConfig(sinkConfig, urlParameter); err != nil { return err @@ -348,12 +344,6 @@ func (c *Config) WithMaxMessageBytes(bytes int) *Config { return c } -// WithMaxBatchedBytes sets the maximum batched message bytes. -func (c *Config) WithMaxBatchedBytes(bytes int) *Config { - c.MaxBatchedBytes = bytes - return c -} - // WithChangefeedID set the `changefeedID` func (c *Config) WithChangefeedID(id common.ChangeFeedID) *Config { c.ChangefeedID = id @@ -421,17 +411,15 @@ func (c *Config) Validate() error { } if c.MaxMessageBytes <= 0 { - return errors.ErrCodecInvalidConfig.GenWithStack("invalid max-message-bytes %d", c.MaxMessageBytes) - } - if c.MaxBatchedBytes < 0 { - return errors.ErrCodecInvalidConfig.GenWithStack("invalid max-batch-message-bytes %d", c.MaxBatchedBytes) - } - if c.MaxBatchedBytes > c.MaxMessageBytes { - return errors.ErrCodecInvalidConfig.GenWithStack("max-batch-message-bytes %d cannot be greater than max-message-bytes %d", c.MaxBatchedBytes, c.MaxMessageBytes) + return errors.ErrCodecInvalidConfig.Wrap( + errors.Errorf("invalid max-message-bytes %d", c.MaxMessageBytes), + ) } if c.MaxBatchSize <= 0 { - return errors.ErrCodecInvalidConfig.GenWithStack("invalid max-batch-size %d", c.MaxBatchSize) + return errors.ErrCodecInvalidConfig.Wrap( + errors.Errorf("invalid max-batch-size %d", c.MaxBatchSize), + ) } if c.LargeMessageHandle != nil { diff --git a/pkg/sink/codec/common/config_test.go b/pkg/sink/codec/common/config_test.go deleted file mode 100644 index 8a49fb3ff1..0000000000 --- a/pkg/sink/codec/common/config_test.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package common - -import ( - "net/url" - "testing" - - "github.com/pingcap/ticdc/pkg/config" - "github.com/pingcap/ticdc/pkg/errors" - "github.com/stretchr/testify/require" -) - -func TestApplyReturnsSinkInvalidConfigForQueryBindingError(t *testing.T) { - cfg := NewConfig(config.ProtocolOpen) - sinkURI, err := url.Parse("kafka://127.0.0.1:9092/topic?max-batch-size=invalid") - require.NoError(t, err) - - err = cfg.Apply(sinkURI, config.GetDefaultReplicaConfig().Sink) - errCode, ok := errors.RFCCode(err) - require.True(t, ok, err) - require.Equal(t, errors.ErrSinkInvalidConfig.RFCCode(), errCode) -} - -func TestValidateMaxBatchMessageBytes(t *testing.T) { - tests := []struct { - name string - adjust func(*Config) - expected string - }{ - { - name: "non-positive max message bytes", - adjust: func(cfg *Config) { - cfg.MaxMessageBytes = 0 - }, - expected: "invalid max-message-bytes 0", - }, - { - name: "negative max batched bytes", - adjust: func(cfg *Config) { - cfg.MaxBatchedBytes = -1 - }, - expected: "invalid max-batch-message-bytes -1", - }, - { - name: "max batched bytes exceeds max message bytes", - adjust: func(cfg *Config) { - cfg.MaxMessageBytes = 100 - cfg.MaxBatchedBytes = 101 - }, - expected: "max-batch-message-bytes 101 cannot be greater than max-message-bytes 100", - }, - { - name: "non-positive max batch size", - adjust: func(cfg *Config) { - cfg.MaxBatchSize = 0 - }, - expected: "invalid max-batch-size 0", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - cfg := NewConfig(config.ProtocolOpen) - test.adjust(cfg) - - err := cfg.Validate() - require.ErrorContains(t, err, test.expected) - errCode, ok := errors.RFCCode(err) - require.True(t, ok, err) - require.Equal(t, errors.ErrCodecInvalidConfig.RFCCode(), errCode) - }) - } -} diff --git a/pkg/sink/codec/open/encoder.go b/pkg/sink/codec/open/encoder.go index b8e608a298..567ce8b60f 100644 --- a/pkg/sink/codec/open/encoder.go +++ b/pkg/sink/codec/open/encoder.go @@ -38,7 +38,7 @@ var ( ) // batchEncoder for open protocol will batch multiple row changed events into a single message. -// One message can contain at most MaxBatchSize events, and the total size of the message cannot exceed MaxBatchedBytes. +// One message can contain at most MaxBatchSize events, and the total size of the message cannot exceed MaxMessageBytes. type batchEncoder struct { messages []*common.Message // buff the callback of the latest message @@ -164,7 +164,7 @@ func (d *batchEncoder) pushMessage(key, value []byte, callback func()) { binary.BigEndian.PutUint64(keyLenByte[:], uint64(len(key))) binary.BigEndian.PutUint64(valueLenByte[:], uint64(len(value))) - if len(d.messages) == 0 || d.messages[len(d.messages)-1].Length()+length > d.config.MaxBatchedBytes || d.messages[len(d.messages)-1].GetRowsCount() >= d.config.MaxBatchSize { + if len(d.messages) == 0 || d.messages[len(d.messages)-1].Length()+length > d.config.MaxMessageBytes || d.messages[len(d.messages)-1].GetRowsCount() >= d.config.MaxBatchSize { d.finalizeCallback() // create a new message versionHead := make([]byte, 8) diff --git a/pkg/sink/codec/open/encoder_test.go b/pkg/sink/codec/open/encoder_test.go index 026190d2e2..678c4fb539 100644 --- a/pkg/sink/codec/open/encoder_test.go +++ b/pkg/sink/codec/open/encoder_test.go @@ -189,7 +189,7 @@ func TestFloatTypes(t *testing.T) { helper.Tk().MustExec("use test") job := helper.DDL2Job(`create table test.t( id int primary key auto_increment, - a float, b float(10, 3), c float(10), + a float, b float(10, 3), c float(10), d double, e double(20, 3))`) dmlEvent := helper.DML2Event("test", "t", `insert into test.t(a,b,c,d,e) values (1.23, 4.56, 7.89, 10.11, 12.13)`) @@ -337,7 +337,7 @@ func TestBlobTypes(t *testing.T) { helper.Tk().MustExec("use test") job := helper.DDL2Job(`create table test.t( - id int primary key auto_increment, + id int primary key auto_increment, a tinyblob, b blob, c mediumblob, d longblob)`) dmlEvent := helper.DML2Event("test", "t", `insert into test.t(a,b,c,d) values (0x010201,0x010202,0x010203,0x010204)`) @@ -533,17 +533,17 @@ func TestOtherTypes(t *testing.T) { helper.Tk().MustExec("use test") job := helper.DDL2Job(`create table test.t( - id int primary key auto_increment, + id int primary key auto_increment, a bool, b bool, c year, - d bit(10), e json, - f decimal(10,2), + d bit(10), e json, + f decimal(10,2), g enum('a','b','c'), h set('a','b','c'))`) tableInfo := helper.GetTableInfo(job) dmlEvent := helper.DML2Event("test", "t", `insert into test.t(a, b, c, d, e, f, g, h) values ( - true, false, 2000, - 0b0101010101, '{"key1": "value1"}', - 153.123, + true, false, 2000, + 0b0101010101, '{"key1": "value1"}', + 153.123, 'a', 'a,b')`) require.NotNil(t, dmlEvent) @@ -778,9 +778,7 @@ func TestEncoderMultipleMessage(t *testing.T) { `insert into test.t values (3, 333)`) ctx := context.Background() - codecConfig := common.NewConfig(config.ProtocolOpen). - WithMaxMessageBytes(1000). - WithMaxBatchedBytes(400) + codecConfig := common.NewConfig(config.ProtocolOpen).WithMaxMessageBytes(400) encoder, err := NewBatchEncoder(codecConfig, nil) require.NoError(t, err) @@ -810,13 +808,11 @@ func TestEncoderMultipleMessage(t *testing.T) { require.Equal(t, 2, len(messages)) require.Equal(t, 2, messages[0].GetRowsCount()) require.Equal(t, 1, messages[1].GetRowsCount()) - require.LessOrEqual(t, messages[0].Length(), codecConfig.MaxBatchedBytes) - require.LessOrEqual(t, messages[1].Length(), codecConfig.MaxBatchedBytes) - require.Equal(t, 0, count) - messages[0].Callback() - require.Equal(t, 2, count) - messages[1].Callback() + for _, message := range messages { + message.Callback() + } + require.Equal(t, 3, count) decoder, err := NewDecoder(ctx, 0, codecConfig, nil) @@ -889,48 +885,6 @@ func TestMessageTooLarge(t *testing.T) { require.Equal(t, count, 0) } -func TestMessageLargerThanBatchLimit(t *testing.T) { - ctx := context.Background() - codecConfig := common.NewConfig(config.ProtocolOpen). - WithMaxMessageBytes(400). - WithMaxBatchedBytes(100) - encoder, err := NewBatchEncoder(codecConfig, nil) - require.NoError(t, err) - - helper := commonEvent.NewEventTestHelper(t) - defer helper.Close() - helper.Tk().MustExec("use test") - - job := helper.DDL2Job(`create table test.t(a tinyint primary key, b int)`) - tableInfo := helper.GetTableInfo(job) - dmlEvent := helper.DML2Event("test", "t", `insert into test.t values (1, 123)`) - require.NotNil(t, dmlEvent) - insertRow, ok := dmlEvent.GetNextRow() - require.True(t, ok) - - count := 0 - insertRowEvent := &commonEvent.RowEvent{ - TableInfo: tableInfo, - CommitTs: dmlEvent.GetCommitTs(), - Event: insertRow, - ColumnSelector: columnselector.NewDefaultColumnSelector(), - Callback: func() { count++ }, - } - - err = encoder.AppendRowChangedEvent(ctx, "", insertRowEvent) - require.NoError(t, err) - - messages := encoder.Build() - require.Len(t, messages, 1) - require.Equal(t, 1, messages[0].GetRowsCount()) - require.Greater(t, messages[0].Length(), codecConfig.MaxBatchedBytes) - require.LessOrEqual(t, messages[0].Length(), codecConfig.MaxMessageBytes) - require.Equal(t, 0, count) - - messages[0].Callback() - require.Equal(t, 1, count) -} - func TestLargeMessageWithHandleEnableHandleKeyOnly(t *testing.T) { helper := commonEvent.NewEventTestHelper(t) defer helper.Close() diff --git a/pkg/sink/kafka/admin.go b/pkg/sink/kafka/admin.go index 8ad44cf5ea..6e7f2930b3 100644 --- a/pkg/sink/kafka/admin.go +++ b/pkg/sink/kafka/admin.go @@ -81,6 +81,11 @@ func (a *saramaAdminClient) GetBrokerConfig(configName string) (string, bool, er return entry.Value, true, nil } } + + log.Warn("Kafka config item not found", + zap.String("keyspace", a.changefeed.Keyspace()), + zap.String("changefeed", a.changefeed.Name()), + zap.String("configName", configName)) return "", false, nil } @@ -99,9 +104,19 @@ func (a *saramaAdminClient) GetTopicConfig(topicName string, configName string) // 2. Kop returns all configs. for _, entry := range configEntries { if entry.Name == configName { + log.Info("Kafka config item found", + zap.String("keyspace", a.changefeed.Keyspace()), + zap.String("changefeed", a.changefeed.Name()), + zap.String("configName", configName), + zap.String("configValue", entry.Value)) return entry.Value, true, nil } } + + log.Warn("Kafka config item not found", + zap.String("keyspace", a.changefeed.Keyspace()), + zap.String("changefeed", a.changefeed.Name()), + zap.String("configName", configName)) return "", false, nil } @@ -121,7 +136,7 @@ func (a *saramaAdminClient) GetTopicsMeta(topics []string, ignoreTopicError bool if !ignoreTopicError { return nil, errors.WrapError(errors.ErrKafkaAdminAPI, meta.Err, "describe-topic", meta.Name) } - log.Warn("kafka topic metadata refresh failed", + log.Warn("fetch topic meta failed", zap.String("keyspace", a.changefeed.Keyspace()), zap.String("changefeed", a.changefeed.Name()), zap.String("topic", meta.Name), @@ -175,7 +190,7 @@ func (a *saramaAdminClient) Close() { // only when admin is unexpectedly nil. if a.admin != nil { if err := a.admin.Close(); err != nil { - log.Warn("kafka admin client close failed", + log.Warn("close admin client meet error", zap.String("keyspace", a.changefeed.Keyspace()), zap.String("changefeed", a.changefeed.Name()), zap.Error(err)) @@ -184,7 +199,7 @@ func (a *saramaAdminClient) Close() { } if a.client != nil { if err := a.client.Close(); err != nil { - log.Warn("kafka client close failed", + log.Warn("close kafka client meet error", zap.String("keyspace", a.changefeed.Keyspace()), zap.String("changefeed", a.changefeed.Name()), zap.Error(err)) diff --git a/pkg/sink/kafka/claimcheck/claim_check.go b/pkg/sink/kafka/claimcheck/claim_check.go index 7c49bd1bbe..4f8ecc2a42 100644 --- a/pkg/sink/kafka/claimcheck/claim_check.go +++ b/pkg/sink/kafka/claimcheck/claim_check.go @@ -52,7 +52,7 @@ func New(ctx context.Context, config *config.LargeMessageHandleConfig, changefee start := time.Now() externalStorage, err := util.GetExternalStorageWithDefaultTimeout(ctx, config.ClaimCheckStorageURI) if err != nil { - log.Error("external storage creation failed", + log.Error("create external storage failed", zap.String("keyspace", changefeedID.Keyspace()), zap.String("changefeed", changefeedID.Name()), zap.String("storageURI", util.MaskSensitiveDataInURI(config.ClaimCheckStorageURI)), diff --git a/pkg/sink/kafka/options.go b/pkg/sink/kafka/options.go index 1e64238f9b..5bc933a3bf 100644 --- a/pkg/sink/kafka/options.go +++ b/pkg/sink/kafka/options.go @@ -14,6 +14,7 @@ package kafka import ( + "context" "encoding/base64" "fmt" "net/http" @@ -38,8 +39,13 @@ const ( defaultPartitionNum = 3 // defaultMaxRetry is the default retry budget for Kafka producers. defaultMaxRetry = 5 - // defaultTimeout is the default timeout for Kafka connections. - defaultTimeout = 10 * time.Second + + // the `max-message-bytes` is set equal to topic's `max.message.bytes`, and is used to check + // whether the message is larger than the max size limit. It's found some message pass the message + // size limit check at the client side and failed at the broker side since message enlarged during + // the network transmission. so we set the `max-message-bytes` to a smaller value to avoid this problem. + // maxMessageBytesOverhead is used to reduce the `max-message-bytes`. + maxMessageBytesOverhead = 128 ) const ( @@ -151,15 +157,14 @@ type options struct { Version string IsAssignedVersion bool RequestVersion int16 - // MaxMessageBytes controls the byte size limit of the producer and encoded messages. - MaxMessageBytes int - // MaxBatchedBytes controls the byte size limit when batching messages. - MaxBatchedBytes int - - MaxRetry int - Compression string - ClientID string - RequiredAcks RequiredAcks + MaxMessageBytes int + MaxRetry int + Compression string + ClientID string + RequiredAcks RequiredAcks + // Only for test. User can not set this value. + // The current prod default value is 0. + MaxMessages int // Credential is used to connect to kafka cluster. EnableTLS bool @@ -176,9 +181,9 @@ type options struct { // NewOptions returns a default Kafka configuration func NewOptions() *options { return &options{ - Version: "2.4.0", + Version: "2.4.0", + // MaxMessageBytes will be used to initialize producer MaxMessageBytes: config.DefaultMaxMessageBytes, - MaxBatchedBytes: config.DefaultMaxMessageBytes, MaxRetry: defaultMaxRetry, ReplicationFactor: 1, Compression: "none", @@ -187,25 +192,27 @@ func NewOptions() *options { InsecureSkipVerify: false, SASL: &security.SASL{}, AutoCreate: true, - DialTimeout: defaultTimeout, - WriteTimeout: defaultTimeout, - ReadTimeout: defaultTimeout, + DialTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + ReadTimeout: 10 * time.Second, } } // setPartitionNum set the partition-num by the topic's partition count. -func (o *options) setPartitionNum(changefeedID common.ChangeFeedID, realPartitionCount int32) error { +func (o *options) setPartitionNum(realPartitionCount int32) error { // user does not specify the `partition-num` in the sink-uri if o.PartitionNum == 0 { o.PartitionNum = realPartitionCount + log.Info("partitionNum is not set, set by topic's partition-num", + zap.Int32("partitionNum", realPartitionCount)) return nil } if o.PartitionNum < realPartitionCount { - log.Warn("configured kafka partition count is lower than topic partition count", - zap.String("namespace", changefeedID.Keyspace()), zap.String("changefeed", changefeedID.Name()), - zap.Int32("configuredPartitionNum", o.PartitionNum), - zap.Int32("topicPartitionNum", realPartitionCount)) + log.Warn("number of partition specified in sink-uri is less than that of the actual topic. "+ + "Some partitions will not have messages dispatched to", + zap.Int32("sinkUriPartitions", o.PartitionNum), + zap.Int32("topicPartitions", realPartitionCount)) return nil } @@ -255,13 +262,8 @@ func (o *options) Apply(changefeedID common.ChangeFeedID, } if urlParameter.MaxMessageBytes != nil { - if *urlParameter.MaxMessageBytes <= 0 { - return errors.ErrKafkaInvalidConfig.GenWithStack( - "invalid max-message-bytes %d", *urlParameter.MaxMessageBytes) - } o.MaxMessageBytes = *urlParameter.MaxMessageBytes } - o.MaxBatchedBytes = o.MaxMessageBytes if urlParameter.MaxRetry != nil && *urlParameter.MaxRetry >= 0 { o.MaxRetry = *urlParameter.MaxRetry @@ -293,9 +295,6 @@ func (o *options) Apply(changefeedID common.ChangeFeedID, if err != nil { return errors.WrapError(errors.ErrKafkaInvalidConfig, err) } - if a <= 0 { - return errors.ErrKafkaInvalidConfig.GenWithStack("dial-timeout must be greater than zero") - } o.DialTimeout = a } @@ -304,9 +303,6 @@ func (o *options) Apply(changefeedID common.ChangeFeedID, if err != nil { return errors.WrapError(errors.ErrKafkaInvalidConfig, err) } - if a <= 0 { - return errors.ErrKafkaInvalidConfig.GenWithStack("write-timeout must be greater than zero") - } o.WriteTimeout = a } @@ -315,9 +311,6 @@ func (o *options) Apply(changefeedID common.ChangeFeedID, if err != nil { return errors.WrapError(errors.ErrKafkaInvalidConfig, err) } - if a <= 0 { - return errors.ErrKafkaInvalidConfig.GenWithStack("read-timeout must be greater than zero") - } o.ReadTimeout = a } @@ -501,6 +494,7 @@ func (o *options) applySASL(urlParameter *urlConfig, sinkConfig *config.SinkConf // BASE64 decode the client secret decodedClientSecret, err := base64.StdEncoding.DecodeString(clientSecret) if err != nil { + log.Error("OAuth2 client secret is not base64 encoded", zap.Error(err)) return errors.ErrKafkaInvalidConfig.GenWithStack("OAuth2 client secret is not base64 encoded") } o.SASL.OAuth2.ClientSecret = string(decodedClientSecret) @@ -570,14 +564,14 @@ func (c *AutoCreateTopicConfig) ValidateReplicationFactor(admin ClusterAdminClie raw, found, err := admin.GetBrokerConfig(MinInsyncReplicasConfigName) if err != nil { - log.Warn("kafka broker configuration lookup failed, skipping replication factor validation", + log.Warn("cannot get Kafka broker configuration, assume replication factor is valid", zap.String("configName", MinInsyncReplicasConfigName), zap.Int16("replicationFactor", c.ReplicationFactor), zap.Error(err)) return nil } if !found { - log.Warn("kafka broker configuration not found, skipping replication factor validation", + log.Warn("Kafka broker configuration not found, assume replication factor is valid", zap.String("configName", MinInsyncReplicasConfigName), zap.Int16("replicationFactor", c.ReplicationFactor)) return nil @@ -622,11 +616,9 @@ func NewKafkaClientID(captureAddr string, return } -// adjustOptions adjusts options with Kafka runtime metadata. -// It overwrites MaxMessageBytes with the final producer message limit derived -// from the topic or broker configuration. +// adjustOptions adjust the `options` and `sarama.Config` by condition. func adjustOptions( - changefeedID common.ChangeFeedID, + ctx context.Context, admin ClusterAdminClient, options *options, topic string, @@ -640,98 +632,90 @@ func adjustOptions( // once we have found the topic, no matter `auto-create-topic`, // make sure user input parameters are valid. if exists { - err = adjustExistingTopicOption(changefeedID, admin, options, info) - } else { - adjustNewTopicOptions(changefeedID, admin, options) - } - if err != nil { - return err - } - - options.MaxBatchedBytes = min(options.MaxBatchedBytes, options.MaxMessageBytes) - return nil -} + // make sure that producer's `MaxMessageBytes` smaller than topic's `max.message.bytes` + topicMaxMessageBytesStr, found, err := getTopicConfig( + ctx, admin, info.Name, + TopicMaxMessageBytesConfigName, + BrokerMessageMaxBytesConfigName, + ) + if err != nil { + return err + } + if !found { + return errors.ErrKafkaAdminAPI.GenWithStack( + "Kafka configuration %s not found in topic %s or broker", + TopicMaxMessageBytesConfigName, info.Name) + } + topicMaxMessageBytes, err := strconv.Atoi(topicMaxMessageBytesStr) + if err != nil { + return errors.WrapError(errors.ErrKafkaAdminAPI, err, + "parse-config", TopicMaxMessageBytesConfigName) + } -func adjustExistingTopicOption( - changefeedID common.ChangeFeedID, - admin ClusterAdminClient, - options *options, - info TopicDetail, -) error { - maxMessageBytes, found, err := getTopicMaxMessageBytes(admin, info.Name) - if err != nil || !found { - log.Warn("kafka topic `max.message.bytes` unavailable, using configured value", - zap.String("namespace", changefeedID.Keyspace()), zap.String("changefeed", changefeedID.Name()), - zap.Int("maxMessageBytes", options.MaxMessageBytes), zap.Error(err)) - maxMessageBytes = options.MaxMessageBytes - } - options.MaxMessageBytes = maxMessageBytes + maxMessageBytes := topicMaxMessageBytes - maxMessageBytesOverhead + if topicMaxMessageBytes <= options.MaxMessageBytes { + log.Warn("topic's `max.message.bytes` less than the `max-message-bytes`,"+ + "use topic's `max.message.bytes` to initialize the Kafka producer", + zap.Int("max.message.bytes", topicMaxMessageBytes), + zap.Int("max-message-bytes", options.MaxMessageBytes), + zap.Int("real-max-message-bytes", maxMessageBytes)) + options.MaxMessageBytes = maxMessageBytes + } else if maxMessageBytes < options.MaxMessageBytes { + options.MaxMessageBytes = maxMessageBytes + } - if err = options.setPartitionNum(changefeedID, info.NumPartitions); err != nil { - return err - } - return nil -} + // no need to create the topic, + // but we would have to log user if they found enter wrong topic name later + if options.AutoCreate { + log.Warn("topic already exist, TiCDC will not create the topic", + zap.String("topic", topic), zap.Any("detail", info)) + } -func adjustNewTopicOptions( - changefeedID common.ChangeFeedID, - admin ClusterAdminClient, - options *options, -) { - // when create the topic, `max.message.bytes` is decided by the broker, - // it would use broker's `message.max.bytes` to set topic's `max.message.bytes`. - messageMaxBytes, found, err := getBrokerMaxMessageBytes(admin) - if err != nil || !found { - log.Warn("kafka broker `message.max.bytes` unavailable, using configured value", - zap.String("namespace", changefeedID.Keyspace()), zap.String("changefeed", changefeedID.Name()), - zap.Int("maxMessageBytes", options.MaxMessageBytes), zap.Error(err)) - messageMaxBytes = options.MaxMessageBytes - } - options.MaxMessageBytes = messageMaxBytes + if err = options.setPartitionNum(info.NumPartitions); err != nil { + return err + } - // topic not exists yet, and user does not specify the `partition-num` in the sink uri. - if options.PartitionNum == 0 { - options.PartitionNum = defaultPartitionNum + return nil } -} -func getTopicMaxMessageBytes( - admin ClusterAdminClient, - topic string, -) (int, bool, error) { - raw, found, err := getTopicConfig( - admin, topic, - TopicMaxMessageBytesConfigName, - BrokerMessageMaxBytesConfigName, - ) + brokerMessageMaxBytesStr, found, err := admin.GetBrokerConfig(BrokerMessageMaxBytesConfigName) if err != nil { - return 0, false, err + log.Warn("TiCDC cannot find `message.max.bytes` from broker's configuration") + return err } if !found { - return 0, false, nil + return errors.ErrKafkaAdminAPI.GenWithStack( + "Kafka broker configuration %s not found", BrokerMessageMaxBytesConfigName) } - maxMessageBytes, err := strconv.Atoi(raw) + brokerMessageMaxBytes, err := strconv.Atoi(brokerMessageMaxBytesStr) if err != nil { - return 0, false, errors.WrapError( - errors.ErrKafkaAdminAPI, err, "parse-config", TopicMaxMessageBytesConfigName) + return errors.WrapError(errors.ErrKafkaAdminAPI, err, + "parse-config", BrokerMessageMaxBytesConfigName) } - return maxMessageBytes, true, nil -} -func getBrokerMaxMessageBytes(admin ClusterAdminClient) (int, bool, error) { - raw, found, err := admin.GetBrokerConfig(BrokerMessageMaxBytesConfigName) - if err != nil { - return 0, false, err - } - if !found { - return 0, false, nil + // when create the topic, `max.message.bytes` is decided by the broker, + // it would use broker's `message.max.bytes` to set topic's `max.message.bytes`. + // TiCDC need to make sure that the producer's `MaxMessageBytes` won't larger than + // broker's `message.max.bytes`. + maxMessageBytes := brokerMessageMaxBytes - maxMessageBytesOverhead + if brokerMessageMaxBytes <= options.MaxMessageBytes { + log.Warn("broker's `message.max.bytes` less than the `max-message-bytes`,"+ + "use broker's `message.max.bytes` to initialize the Kafka producer", + zap.Int("message.max.bytes", brokerMessageMaxBytes), + zap.Int("max-message-bytes", options.MaxMessageBytes), + zap.Int("real-max-message-bytes", maxMessageBytes)) + options.MaxMessageBytes = maxMessageBytes + } else if maxMessageBytes < options.MaxMessageBytes { + options.MaxMessageBytes = maxMessageBytes } - messageMaxBytes, err := strconv.Atoi(raw) - if err != nil { - return 0, false, errors.WrapError( - errors.ErrKafkaAdminAPI, err, "parse-config", BrokerMessageMaxBytesConfigName) + + // topic not exists yet, and user does not specify the `partition-num` in the sink uri. + if options.PartitionNum == 0 { + options.PartitionNum = defaultPartitionNum + log.Warn("partition-num is not set, use the default partition count", + zap.String("topic", topic), zap.Int32("partitions", options.PartitionNum)) } - return messageMaxBytes, true, nil + return nil } // getTopicConfig gets topic config by name. @@ -739,6 +723,7 @@ func getBrokerMaxMessageBytes(admin ClusterAdminClient) (int, bool, error) { // we will try to get it from the broker's configuration. // NOTICE: The configuration names of topic and broker may be different for the same configuration. func getTopicConfig( + _ context.Context, admin ClusterAdminClient, topicName string, topicConfigName string, @@ -749,5 +734,7 @@ func getTopicConfig( return c, true, nil } + log.Info("kafka sink cannot get the configuration from topic, try to get it from broker", + zap.String("topic", topicName), zap.String("config", topicConfigName), zap.Error(err)) return admin.GetBrokerConfig(brokerConfigName) } diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index 9c0ca3c6ed..d75b9d64ea 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/config" "github.com/pingcap/ticdc/pkg/errors" + codecCommon "github.com/pingcap/ticdc/pkg/sink/codec/common" "github.com/stretchr/testify/require" ) @@ -154,6 +155,14 @@ func (f *kafkaAdminFixture) setMessageMaxBytes(brokerValue, topicValue string) { f.topicConfig[defaultMockTopicName][TopicMaxMessageBytesConfigName] = topicValue } +func expectedAdjustedMaxMessageBytes(configuredMaxMessageBytes, sourceMaxMessageBytes int) int { + sourceMaxMessageBytes -= maxMessageBytesOverhead + if configuredMaxMessageBytes < sourceMaxMessageBytes { + return configuredMaxMessageBytes + } + return sourceMaxMessageBytes +} + func (f *kafkaAdminFixture) setMinInsyncReplicas(minInsyncReplicas string) { f.topicConfig[defaultMockTopicName][MinInsyncReplicasConfigName] = minInsyncReplicas f.brokerConfig[MinInsyncReplicasConfigName] = minInsyncReplicas @@ -181,7 +190,6 @@ func TestCompleteOptions(t *testing.T) { require.Equal(t, int16(3), options.ReplicationFactor) require.Equal(t, "2.6.0", options.Version) require.Equal(t, 4096, options.MaxMessageBytes) - require.Equal(t, 4096, options.MaxBatchedBytes) require.Equal(t, WaitForLocal, options.RequiredAcks) require.Equal(t, defaultMaxRetry, options.MaxRetry) @@ -290,74 +298,19 @@ func TestCompleteOptions(t *testing.T) { require.Equal(t, defaultMaxRetry, options.MaxRetry) } -func TestApplyRejectsNonPositiveMaxMessageBytes(t *testing.T) { - tests := []struct { - name string - uri string - configValue *int - expected int - }{ - { - name: "zero from URI", - uri: "kafka://127.0.0.1:9092/test-topic?max-message-bytes=0", - expected: 0, - }, - { - name: "negative from URI", - uri: "kafka://127.0.0.1:9092/test-topic?max-message-bytes=-1", - expected: -1, - }, - { - name: "zero from sink config", - uri: "kafka://127.0.0.1:9092/test-topic", - configValue: aws.Int(0), - expected: 0, - }, - { - name: "negative from sink config", - uri: "kafka://127.0.0.1:9092/test-topic", - configValue: aws.Int(-1), - expected: -1, - }, - } - - changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test") - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - sinkURI, err := url.Parse(test.uri) - require.NoError(t, err) - - sinkConfig := config.GetDefaultReplicaConfig().Sink - if test.configValue != nil { - sinkConfig.KafkaConfig = &config.KafkaConfig{ - MaxMessageBytes: test.configValue, - } - } - - options := NewOptions() - err = options.Apply(changefeedID, sinkURI, sinkConfig) - require.ErrorContains(t, err, fmt.Sprintf("invalid max-message-bytes %d", test.expected)) - errCode, ok := errors.RFCCode(err) - require.True(t, ok) - require.Equal(t, errors.ErrKafkaInvalidConfig.RFCCode(), errCode) - }) - } -} - func TestSetPartitionNum(t *testing.T) { options := NewOptions() - changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test") - err := options.setPartitionNum(changefeedID, 2) + err := options.setPartitionNum(2) require.NoError(t, err) require.Equal(t, int32(2), options.PartitionNum) options.PartitionNum = 1 - err = options.setPartitionNum(changefeedID, 2) + err = options.setPartitionNum(2) require.NoError(t, err) require.Equal(t, int32(1), options.PartitionNum) options.PartitionNum = 3 - err = options.setPartitionNum(changefeedID, 2) + err = options.setPartitionNum(2) require.True(t, errors.ErrKafkaInvalidConfig.Equal(err)) } @@ -425,43 +378,19 @@ func TestTimeout(t *testing.T) { require.Equal(t, 2*time.Minute, options.WriteTimeout) } -func TestApplyRejectsNonPositiveTimeout(t *testing.T) { - t.Parallel() - - changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test") - for _, parameter := range []string{"dial-timeout", "read-timeout", "write-timeout"} { - for _, value := range []string{"0s", "-1s"} { - t.Run(parameter+"="+value, func(t *testing.T) { - t.Parallel() - - sinkURI, err := url.Parse( - "kafka://127.0.0.1:9092/kafka-test?" + parameter + "=" + value) - require.NoError(t, err) - - err = NewOptions().Apply( - changefeedID, sinkURI, config.GetDefaultReplicaConfig().Sink) - require.ErrorContains(t, err, parameter+" must be greater than zero") - errCode, ok := errors.RFCCode(err) - require.True(t, ok) - require.Equal(t, errors.ErrKafkaInvalidConfig.RFCCode(), errCode) - }) - } - } -} - func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *testing.T) { tests := []struct { name string configuredMaxMessageBytes func(*kafkaAdminFixture) int }{ { - name: "uses broker limit when configured value is below broker", + name: "keeps configured value below broker limit", configuredMaxMessageBytes: func(*kafkaAdminFixture) int { return 1024 }, }, { - name: "uses broker limit when configured value is below broker by one byte", + name: "uses broker limit when configured value is within overhead", configuredMaxMessageBytes: func(f *kafkaAdminFixture) int { return f.brokerMessageMaxBytes() - 1 }, @@ -475,7 +404,6 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t } topicName := "test-topic" - changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test") for _, test := range tests { t.Run(test.name, func(t *testing.T) { adminFixture := newKafkaAdminFixture(t) @@ -488,38 +416,23 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t err := adminClient.CreateTopic(detail) require.NoError(t, err) - configuredMaxMessageBytes := test.configuredMaxMessageBytes(adminFixture) - sinkURI, err := url.Parse(fmt.Sprintf( - "kafka://127.0.0.1:9092/%s?max-message-bytes=%d", - topicName, configuredMaxMessageBytes, - )) - require.NoError(t, err) - options := NewOptions() - err = options.Apply( - changefeedID, - sinkURI, - config.GetDefaultReplicaConfig().Sink, + options.BrokerEndpoints = []string{"127.0.0.1:9092"} + options.MaxMessageBytes = test.configuredMaxMessageBytes(adminFixture) + expectedMaxMessageBytes := expectedAdjustedMaxMessageBytes( + options.MaxMessageBytes, + adminFixture.brokerMessageMaxBytes(), ) - require.NoError(t, err) - require.Equal(t, configuredMaxMessageBytes, options.MaxMessageBytes) - require.Equal(t, configuredMaxMessageBytes, options.MaxBatchedBytes) - expectedProducerLimit := adminFixture.brokerMessageMaxBytes() ctx := context.Background() - err = adjustOptions(changefeedID, adminClient, options, topicName) + err = adjustOptions(ctx, adminClient, options, topicName) require.NoError(t, err) saramaConfig, err := newSaramaConfig(ctx, options) require.NoError(t, err) - require.Equal(t, expectedProducerLimit, options.MaxMessageBytes) - require.Equal( - t, - min(configuredMaxMessageBytes, expectedProducerLimit), - options.MaxBatchedBytes, - ) - require.Equal(t, expectedProducerLimit, saramaConfig.Producer.MaxMessageBytes) + require.Equal(t, expectedMaxMessageBytes, options.MaxMessageBytes) + require.Equal(t, expectedMaxMessageBytes, saramaConfig.Producer.MaxMessageBytes) }) } } @@ -612,7 +525,7 @@ func TestConfigurationCombinations(t *testing.T) { mockTopicMessageMaxBytes, }, { - "new topic broker below user", + "new topic broker overhead below user", "kafka://127.0.0.1:9092/%s?max-message-bytes=%s", []any{"not-created-topic", strconv.Itoa(1024*1024 + 1)}, mockBrokerMessageMaxBytes, @@ -678,7 +591,7 @@ func TestConfigurationCombinations(t *testing.T) { strconv.Itoa(config.DefaultMaxMessageBytes + 1), }, { - "existing topic topic below user", + "existing topic topic overhead below user", "kafka://127.0.0.1:9092/%s?max-message-bytes=%s", []any{defaultMockTopicName, strconv.Itoa(1024*1024 + 1)}, mockBrokerMessageMaxBytes, @@ -729,7 +642,6 @@ func TestConfigurationCombinations(t *testing.T) { options := NewOptions() err = options.Apply(common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) require.Nil(t, err) - configuredMaxMessageBytes := options.MaxMessageBytes topic, ok := a.uriParams[0].(string) require.True(t, ok) @@ -739,15 +651,30 @@ func TestConfigurationCombinations(t *testing.T) { if _, exists := adminFixture.topics[topic]; exists { sourceMaxMessageBytes = adminFixture.topicMaxMessageBytes(topic) } - changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test") - err = adjustOptions(changefeedID, adminClient, options, topic) + expectedMaxMessageBytes := expectedAdjustedMaxMessageBytes(options.MaxMessageBytes, sourceMaxMessageBytes) + + err = adjustOptions(context.Background(), adminClient, options, topic) require.Nil(t, err) - require.Equal(t, sourceMaxMessageBytes, options.MaxMessageBytes) - require.Equal( - t, - min(configuredMaxMessageBytes, sourceMaxMessageBytes), - options.MaxBatchedBytes, - ) + require.Equal(t, expectedMaxMessageBytes, options.MaxMessageBytes) + + saramaConfig, err := newSaramaConfig(context.Background(), options) + require.Nil(t, err) + require.Equal(t, expectedMaxMessageBytes, saramaConfig.Producer.MaxMessageBytes) + + encoderConfig := codecCommon.NewConfig(config.ProtocolOpen) + err = encoderConfig.Apply(sinkURI, &config.SinkConfig{ + KafkaConfig: &config.KafkaConfig{ + LargeMessageHandle: config.NewDefaultLargeMessageHandleConfig(), + }, + }) + require.Nil(t, err) + encoderConfig.WithMaxMessageBytes(options.MaxMessageBytes) + + err = encoderConfig.Validate() + require.Nil(t, err) + + // producer's `MaxMessageBytes` = encoder's `MaxMessageBytes`. + require.Equal(t, expectedMaxMessageBytes, encoderConfig.MaxMessageBytes) adminClient.Close() }) @@ -792,7 +719,6 @@ func TestMerge(t *testing.T) { require.Equal(t, int16(5), c.ReplicationFactor) require.Equal(t, "3.1.2", c.Version) require.Equal(t, 1024*1024, c.MaxMessageBytes) - require.Equal(t, 1024*1024, c.MaxBatchedBytes) require.Equal(t, "gzip", c.Compression) require.Equal(t, "test-id", c.ClientID) require.Equal(t, true, c.AutoCreate) @@ -874,7 +800,6 @@ func TestMerge(t *testing.T) { require.Equal(t, int16(5), c.ReplicationFactor) require.Equal(t, "3.1.2", c.Version) require.Equal(t, 1024*1024, c.MaxMessageBytes) - require.Equal(t, 1024*1024, c.MaxBatchedBytes) require.Equal(t, "gzip", c.Compression) require.Equal(t, "test-id", c.ClientID) require.Equal(t, true, c.AutoCreate) diff --git a/pkg/sink/kafka/sarama_async_producer.go b/pkg/sink/kafka/sarama_async_producer.go index d3e1781c71..f0c0f6b5d1 100644 --- a/pkg/sink/kafka/sarama_async_producer.go +++ b/pkg/sink/kafka/sarama_async_producer.go @@ -62,13 +62,13 @@ func (p *saramaAsyncProducer) Close() { // To prevent the scenario mentioned above, close the client first. start := time.Now() if err := p.client.Close(); err != nil { - log.Warn("kafka async producer client close failed", + log.Warn("Close kafka async producer client error", zap.String("keyspace", p.changefeedID.Keyspace()), zap.String("changefeed", p.changefeedID.Name()), zap.Duration("duration", time.Since(start)), zap.Error(err)) } else { - log.Info("kafka async producer client closed", + log.Info("Close kafka async producer client success", zap.String("keyspace", p.changefeedID.Keyspace()), zap.String("changefeed", p.changefeedID.Name()), zap.Duration("duration", time.Since(start))) @@ -76,13 +76,13 @@ func (p *saramaAsyncProducer) Close() { start = time.Now() if err := p.producer.Close(); err != nil { - log.Warn("kafka async producer close failed", + log.Warn("Close kafka async producer error", zap.String("keyspace", p.changefeedID.Keyspace()), zap.String("changefeed", p.changefeedID.Name()), zap.Duration("duration", time.Since(start)), zap.Error(err)) } else { - log.Info("kafka async producer closed", + log.Info("Close kafka async producer success", zap.String("keyspace", p.changefeedID.Keyspace()), zap.String("changefeed", p.changefeedID.Name()), zap.Duration("duration", time.Since(start))) @@ -97,6 +97,9 @@ func (p *saramaAsyncProducer) AsyncRunCallback( for { select { case <-ctx.Done(): + log.Info("async producer exit since context is done", + zap.String("keyspace", p.changefeedID.Keyspace()), + zap.String("changefeed", p.changefeedID.Name())) return context.Cause(ctx) case ack := <-p.producer.Successes(): if ack != nil { @@ -106,7 +109,7 @@ func (p *saramaAsyncProducer) AsyncRunCallback( meta.callback() } default: - log.Error("kafka producer received unknown message metadata type", + log.Error("unknown message metadata type in async producer", zap.Any("metadata", ack.Metadata)) } } @@ -125,7 +128,7 @@ func (p *saramaAsyncProducer) AsyncRunCallback( } func (p *saramaAsyncProducer) handleProducerError(err *sarama.ProducerError) error { - log.Error("kafka message send failed", + log.Error("send message to kafka failed", zap.String("keyspace", p.changefeedID.Keyspace()), zap.String("changefeed", p.changefeedID.Name()), zap.String("eventContext", BuildEventLogContext( diff --git a/pkg/sink/kafka/sarama_config.go b/pkg/sink/kafka/sarama_config.go index 51dbd2384e..4988c79c52 100644 --- a/pkg/sink/kafka/sarama_config.go +++ b/pkg/sink/kafka/sarama_config.go @@ -62,7 +62,7 @@ func newSaramaConfig(ctx context.Context, o *options) (*sarama.Config, error) { config.Producer.Flush.Bytes = 0 config.Producer.Flush.Messages = 0 config.Producer.Flush.Frequency = time.Duration(0) - config.Producer.Flush.MaxMessages = 0 + config.Producer.Flush.MaxMessages = o.MaxMessages config.Net.MaxOpenRequests = 1 config.Net.DialTimeout = o.DialTimeout @@ -87,9 +87,12 @@ func newSaramaConfig(ctx context.Context, o *options) (*sarama.Config, error) { case "zstd": config.Producer.Compression = sarama.CompressionZSTD default: - log.Warn("unsupported kafka compression algorithm", zap.String("compression", o.Compression)) + log.Warn("Unsupported compression algorithm", zap.String("compression", o.Compression)) config.Producer.Compression = sarama.CompressionNone } + if config.Producer.Compression != sarama.CompressionNone { + log.Info("Kafka producer uses " + compression + " compression algorithm") + } if o.EnableTLS { // for SSL encryption with a trust CA certificate, we must populate the @@ -117,27 +120,27 @@ func newSaramaConfig(ctx context.Context, o *options) (*sarama.Config, error) { return nil, err } - err = completeSaramaKafkaVersion(config, o) + kafkaVersion, err := getKafkaVersion(config, o) if err != nil { - return nil, err + log.Warn("Can't get Kafka version by broker. ticdc will use default version", + zap.String("defaultVersion", kafkaVersion.String())) } - return config, nil -} + config.Version = kafkaVersion -func completeSaramaKafkaVersion(config *sarama.Config, o *options) error { - detectedVersion, err := detectKafkaVersion(config, o) - if err != nil { - log.Warn("kafka version detection failed, using fallback version", - zap.Strings("brokers", o.BrokerEndpoints), - zap.String("fallbackVersion", detectedVersion.String()), - zap.Error(err)) - } - kafkaVersion, err := selectKafkaVersion(detectedVersion, o) - if err != nil { - return err + if o.IsAssignedVersion { + version, err := sarama.ParseKafkaVersion(o.Version) + if err != nil { + return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) + } + config.Version = version + if !version.IsAtLeast(maxKafkaVersion) && version.String() != kafkaVersion.String() { + log.Warn("The Kafka version you assigned may not be correct. "+ + "Please assign a version equal to or less than the specified version", + zap.String("assignedVersion", version.String()), + zap.String("desiredVersion", kafkaVersion.String())) + } } - config.Version = kafkaVersion - return nil + return config, nil } func completeSaramaSASLConfig(ctx context.Context, config *sarama.Config, o *options) error { @@ -183,7 +186,7 @@ func completeSaramaSASLConfig(ctx context.Context, config *sarama.Config, o *opt return nil } -func detectKafkaVersion(config *sarama.Config, o *options) (sarama.KafkaVersion, error) { +func getKafkaVersion(config *sarama.Config, o *options) (sarama.KafkaVersion, error) { addrs := o.BrokerEndpoints if len(addrs) > 1 { // Shuffle the list of addresses to randomize the order in which @@ -205,26 +208,25 @@ func detectKafkaVersion(config *sarama.Config, o *options) (sarama.KafkaVersion, } } if err != nil { + log.Warn("kafka sink use the default kafka version since cannot find it from the brokers", + zap.String("defaultVersion", defaultKafkaVersion.String())) targetVersion = defaultKafkaVersion } - return targetVersion, err -} -func selectKafkaVersion(detectedVersion sarama.KafkaVersion, o *options) (sarama.KafkaVersion, error) { - if !o.IsAssignedVersion { - return detectedVersion, nil - } - assignedVersion, err := sarama.ParseKafkaVersion(o.Version) - if err != nil { - return assignedVersion, errors.WrapError(errors.ErrKafkaInvalidConfig, err) - } - if !assignedVersion.IsAtLeast(maxKafkaVersion) && - assignedVersion.String() != detectedVersion.String() { - log.Warn("configured kafka version differs from detected version", - zap.String("assignedVersion", assignedVersion.String()), - zap.String("desiredVersion", detectedVersion.String())) + if o.IsAssignedVersion { + assignedVersion, err := sarama.ParseKafkaVersion(o.Version) + if err != nil { + return assignedVersion, errors.WrapError(errors.ErrKafkaInvalidConfig, err) + } + if !assignedVersion.IsAtLeast(maxKafkaVersion) && assignedVersion.String() != targetVersion.String() { + log.Warn("The Kafka version you assigned may not be correct. "+ + "Please assign a version equal to or less than the specified version", + zap.String("assignedVersion", assignedVersion.String()), + zap.String("desiredVersion", targetVersion.String())) + } + targetVersion = assignedVersion } - return assignedVersion, nil + return targetVersion, nil } func getKafkaVersionFromBroker(config *sarama.Config, requestVersion int16, addr string) (sarama.KafkaVersion, error) { @@ -235,10 +237,12 @@ func getKafkaVersionFromBroker(config *sarama.Config, requestVersion int16, addr _ = broker.Close() }() if err != nil { + log.Warn("Kafka fail to open broker", zap.String("addr", addr), zap.Error(err)) return KafkaVersion, err } apiResponse, err := broker.ApiVersions(&sarama.ApiVersionsRequest{Version: requestVersion}) if err != nil { + log.Warn("Kafka fail to get ApiVersions", zap.String("addr", addr), zap.Error(err)) return KafkaVersion, err } // ApiKey method diff --git a/pkg/sink/kafka/sarama_config_test.go b/pkg/sink/kafka/sarama_config_test.go index f85d688edc..ca3adbbdcb 100644 --- a/pkg/sink/kafka/sarama_config_test.go +++ b/pkg/sink/kafka/sarama_config_test.go @@ -58,7 +58,6 @@ func TestNewSaramaConfig(t *testing.T) { cfg, err := newSaramaConfig(ctx, options) require.NoError(t, err) require.Equal(t, defaultMaxRetry, cfg.Producer.Retry.Max) - require.Equal(t, options.MaxMessageBytes, cfg.Producer.MaxMessageBytes) options.EnableTLS = true options.Credential = &security.Credential{ @@ -86,63 +85,6 @@ func TestNewSaramaConfig(t *testing.T) { require.Equal(t, sarama.SASLMechanism("SCRAM-SHA-256"), cfg.Net.SASL.Mechanism) } -func TestSelectKafkaVersion(t *testing.T) { - tests := []struct { - name string - detectedVersion sarama.KafkaVersion - assignedVersion string - expectedVersion sarama.KafkaVersion - expectedErr error - }{ - { - name: "use detected version", - detectedVersion: sarama.V2_4_0_0, - expectedVersion: sarama.V2_4_0_0, - }, - { - name: "use fallback version", - detectedVersion: defaultKafkaVersion, - expectedVersion: defaultKafkaVersion, - }, - { - name: "assigned version overrides detected version", - detectedVersion: sarama.V2_4_0_0, - assignedVersion: "2.6.0", - expectedVersion: sarama.V2_6_0_0, - }, - { - name: "assigned version overrides fallback version", - detectedVersion: defaultKafkaVersion, - assignedVersion: "2.6.0", - expectedVersion: sarama.V2_6_0_0, - }, - { - name: "reject invalid assigned version", - detectedVersion: sarama.V2_4_0_0, - assignedVersion: "invalid", - expectedErr: errors.ErrKafkaInvalidConfig, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - options := NewOptions() - if test.assignedVersion != "" { - options.IsAssignedVersion = true - options.Version = test.assignedVersion - } - - version, err := selectKafkaVersion(test.detectedVersion, options) - if test.expectedErr != nil { - require.ErrorIs(t, err, test.expectedErr) - return - } - require.NoError(t, err) - require.Equal(t, test.expectedVersion, version) - }) - } -} - func TestNewSaramaConfigInvalidOAuthTokenURL(t *testing.T) { options := NewOptions() options.SASL = &security.SASL{ diff --git a/pkg/sink/kafka/sarama_factory.go b/pkg/sink/kafka/sarama_factory.go index 8f73ca70b5..57bf5ef27c 100644 --- a/pkg/sink/kafka/sarama_factory.go +++ b/pkg/sink/kafka/sarama_factory.go @@ -40,12 +40,10 @@ func NewSaramaFactory( ) (Factory, error) { start := time.Now() config, err := newSaramaConfig(ctx, o) - duration := time.Since(start) - if duration > 2*time.Second { - log.Warn("kafka configuration initialization is slow", - zap.String("keyspace", changefeedID.Keyspace()), - zap.String("changefeed", changefeedID.Name()), - zap.Duration("duration", duration)) + duration := time.Since(start).Seconds() + if duration > 2 { + log.Warn("new sarama config cost too much time", + zap.Stringer("changefeedID", changefeedID), zap.Any("duration", duration)) } if err != nil { return nil, err @@ -59,22 +57,9 @@ func NewSaramaFactory( admin.Close() }() - if err = adjustOptions(changefeedID, admin, o, o.Topic); err != nil { + if err = adjustOptions(ctx, admin, o, o.Topic); err != nil { return nil, err } - log.Info("kafka sink configuration resolved", - zap.String("namespace", changefeedID.Keyspace()), - zap.String("changefeed", changefeedID.Name()), - zap.String("topic", o.Topic), - zap.Int32("partitionNum", o.PartitionNum), - zap.Int("maxMessageBytes", o.MaxMessageBytes), - zap.Int("maxBatchedBytes", o.MaxBatchedBytes), - zap.String("compression", config.Producer.Compression.String()), - zap.Int16("requiredAcks", int16(o.RequiredAcks)), - zap.Int("maxRetry", o.MaxRetry), - zap.Duration("dialTimeout", o.DialTimeout), - zap.Duration("readTimeout", o.ReadTimeout), - zap.Duration("writeTimeout", o.WriteTimeout)) return &saramaFactory{ changefeedID: changefeedID, @@ -86,12 +71,10 @@ func NewSaramaFactory( func newAdminClient(changefeedID common.ChangeFeedID, endpoints []string, config *sarama.Config) (ClusterAdminClient, error) { start := time.Now() client, err := sarama.NewClient(endpoints, config) - duration := time.Since(start) - if duration > 2*time.Second { - log.Warn("kafka client initialization is slow", - zap.String("keyspace", changefeedID.Keyspace()), - zap.String("changefeed", changefeedID.Name()), - zap.Duration("duration", duration)) + duration := time.Since(start).Seconds() + if duration > 2 { + log.Warn("new sarama client cost too much time", + zap.Any("duration", duration), zap.Stringer("changefeedID", changefeedID)) } if err != nil { return nil, errors.WrapError(errors.ErrNewKafkaSink, err) @@ -99,12 +82,10 @@ func newAdminClient(changefeedID common.ChangeFeedID, endpoints []string, config start = time.Now() admin, err := sarama.NewClusterAdminFromClient(client) - duration = time.Since(start) - if duration > 2*time.Second { - log.Warn("kafka admin client initialization is slow", - zap.String("keyspace", changefeedID.Keyspace()), - zap.String("changefeed", changefeedID.Name()), - zap.Duration("duration", duration)) + duration = time.Since(start).Seconds() + if duration > 2 { + log.Warn("new sarama cluster admin cost too much time", + zap.Any("duration", duration), zap.Stringer("changefeedID", changefeedID)) } if err != nil { // `sarama.NewClusterAdminFromClient` does not take ownership of the client, diff --git a/pkg/sink/kafka/sarama_sync_producer.go b/pkg/sink/kafka/sarama_sync_producer.go index fcf1c9c258..754d1db9e1 100644 --- a/pkg/sink/kafka/sarama_sync_producer.go +++ b/pkg/sink/kafka/sarama_sync_producer.go @@ -58,7 +58,7 @@ func (p *saramaSyncProducer) SendMessage(topic string, partitionNum int32, messa if err == nil { return nil } - log.Error("kafka message send failed", + log.Error("send message to kafka failed", zap.String("keyspace", p.id.Keyspace()), zap.String("changefeed", p.id.Name()), zap.String("eventContext", BuildEventLogContext(p.id.Keyspace(), p.id.Name(), message.LogInfo)), @@ -84,7 +84,7 @@ func (p *saramaSyncProducer) SendMessages(topic string, partitionNum int32, mess if err == nil { return nil } - log.Error("kafka message send failed", + log.Error("send message to kafka failed", zap.String("keyspace", p.id.Keyspace()), zap.String("changefeed", p.id.Name()), zap.String("eventContext", BuildEventLogContext(p.id.Keyspace(), p.id.Name(), message.LogInfo)), @@ -94,7 +94,7 @@ func (p *saramaSyncProducer) SendMessages(topic string, partitionNum int32, mess func (p *saramaSyncProducer) Close() { if p.closed.Load() { - log.Warn("kafka ddl producer already closed", + log.Warn("kafka DDL producer already closed", zap.String("keyspace", p.id.Keyspace()), zap.String("changefeed", p.id.Name())) return @@ -106,7 +106,7 @@ func (p *saramaSyncProducer) Close() { // so producer.Close() alone won't release the underlying client resources. if p.client != nil { if err := p.client.Close(); err != nil { - log.Warn("kafka ddl producer client close failed", + log.Warn("Close Kafka DDL producer client with error", zap.String("keyspace", p.id.Keyspace()), zap.String("changefeed", p.id.Name()), zap.Duration("duration", time.Since(start)), @@ -115,7 +115,7 @@ func (p *saramaSyncProducer) Close() { } if p.producer != nil { if err := p.producer.Close(); err != nil { - log.Error("kafka ddl producer close failed", + log.Error("Close Kafka DDL producer with error", zap.String("keyspace", p.id.Keyspace()), zap.String("changefeed", p.id.Name()), zap.Duration("duration", time.Since(start)), @@ -123,7 +123,7 @@ func (p *saramaSyncProducer) Close() { return } } - log.Info("kafka ddl producer closed", + log.Info("Kafka DDL producer closed", zap.String("keyspace", p.id.Keyspace()), zap.String("changefeed", p.id.Name()), zap.Duration("duration", time.Since(start))) diff --git a/tests/integration_tests/_utils/kafka_topic b/tests/integration_tests/_utils/kafka_topic deleted file mode 100755 index 9f9a79a1cd..0000000000 --- a/tests/integration_tests/_utils/kafka_topic +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -set -eu - -CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -UTILITY_DIR="$CUR/../../utils/kafka_topic" - -if [ ! -f "$UTILITY_DIR/kafka_topic" ]; then - (cd "$UTILITY_DIR" && GO111MODULE=on go build) -fi - -"$UTILITY_DIR/kafka_topic" "$@" diff --git a/tests/integration_tests/canal_json_claim_check/run.sh b/tests/integration_tests/canal_json_claim_check/run.sh index a354dc589a..10f822c98b 100755 --- a/tests/integration_tests/canal_json_claim_check/run.sh +++ b/tests/integration_tests/canal_json_claim_check/run.sh @@ -18,10 +18,7 @@ function run() { start_tidb_cluster --workdir $WORK_DIR - TOPIC_NAME="canal-json-claim-check-$RANDOM" - CLAIM_CHECK_DIR="/tmp/canal-json-claim-check" - rm -rf "$CLAIM_CHECK_DIR" - kafka_topic --topic "$TOPIC_NAME" --max-message-bytes 1000 + TOPIC_NAME="canal-json-claim-check" # record tso before we create tables to skip the system table DDLs start_ts=$(run_cdc_cli_tso_query ${UP_PD_HOST_1} ${UP_PD_PORT_1}) @@ -39,10 +36,6 @@ function run() { # sync_diff can't check non-exist table, so we check expected tables are created in downstream first check_table_exists test.finish_mark ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 200 check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml - if ! find "$CLAIM_CHECK_DIR" -type f -print -quit | grep -q .; then - echo "claim-check did not write any file to $CLAIM_CHECK_DIR" - exit 1 - fi cleanup_process $CDC_BINARY } diff --git a/tests/integration_tests/canal_json_handle_key_only/run.sh b/tests/integration_tests/canal_json_handle_key_only/run.sh index 93ae5adfba..372dff24fe 100755 --- a/tests/integration_tests/canal_json_handle_key_only/run.sh +++ b/tests/integration_tests/canal_json_handle_key_only/run.sh @@ -19,7 +19,6 @@ function run() { start_tidb_cluster --workdir $WORK_DIR TOPIC_NAME="canal-json-handle-key-only-$RANDOM" - kafka_topic --topic "$TOPIC_NAME" --max-message-bytes 1000 # record tso before we create tables to skip the system table DDLs start_ts=$(run_cdc_cli_tso_query ${UP_PD_HOST_1} ${UP_PD_PORT_1}) diff --git a/tests/integration_tests/kafka_big_messages/conf/diff_config.toml b/tests/integration_tests/kafka_big_messages/conf/diff_config.toml index bf73beb595..0082a37028 100644 --- a/tests/integration_tests/kafka_big_messages/conf/diff_config.toml +++ b/tests/integration_tests/kafka_big_messages/conf/diff_config.toml @@ -13,7 +13,7 @@ source-instances = ["mysql1"] target-instance = "tidb0" -target-check-tables = ["database_name.*"] +target-check-tables = ["kafka_big_messages.test"] [data-sources] [data-sources.mysql1] diff --git a/tests/integration_tests/kafka_big_messages/run.sh b/tests/integration_tests/kafka_big_messages/run.sh index b9a4b646fb..0628eaa92e 100755 --- a/tests/integration_tests/kafka_big_messages/run.sh +++ b/tests/integration_tests/kafka_big_messages/run.sh @@ -3,212 +3,53 @@ set -eu CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -source "$CUR/../_utils/test_prepare" +source $CUR/../_utils/test_prepare WORK_DIR=$OUT_DIR/$TEST_NAME CDC_BINARY=cdc.test SINK_TYPE=$1 -STATE_WAIT_TIMEOUT_SECONDS=30 -STATE_CHECK_INTERVAL_SECONDS=1 -TABLE_CHECK_RETRIES=15 -BATCH_LIMIT=262144 -SMALL_TOPIC_LIMIT=524288 -LARGE_TOPIC_LIMIT=2097152 -ROW_BYTES=1048576 -SCHEMA_REGISTRY_URI=http://127.0.0.1:8088 -GENERATOR_DIR=$CUR/../../utils/gen_kafka_big_messages -consumer_pid="" - -function start_schema_registry() { - if curl -o /dev/null -s "$SCHEMA_REGISTRY_URI"; then +function run() { + # test kafka sink only in this case + if [ "$SINK_TYPE" != "kafka" ]; then return fi + rm -rf $WORK_DIR && mkdir -p $WORK_DIR - echo "Starting schema registry..." - ./bin/bin/schema-registry-start -daemon ./bin/etc/schema-registry/schema-registry.properties - local i=0 - while ! curl -o /dev/null -s "$SCHEMA_REGISTRY_URI"; do - i=$((i + 1)) - if [ "$i" -gt 30 ]; then - echo "Failed to start schema registry" - exit 1 - fi - sleep 2 - done - curl -X PUT -H "Content-Type: application/vnd.schemaregistry.v1+json" --data '{"compatibility": "NONE"}' "$SCHEMA_REGISTRY_URI/config" -} + start_tidb_cluster --workdir $WORK_DIR -function build_message_generator() { - if [ ! -f "$GENERATOR_DIR/gen_kafka_big_messages" ]; then - (cd "$GENERATOR_DIR" && GO111MODULE=on go build) - fi -} + TOPIC_NAME="big-message-test-$RANDOM" -function kafka_sink_uri() { - local topic_name=$1 - local protocol=$2 - local extra_params=$3 - local sink_uri="kafka://127.0.0.1:9092/${topic_name}?protocol=${protocol}&partition-num=1&kafka-version=${KAFKA_VERSION}&max-message-bytes=${BATCH_LIMIT}" - if [ "$extra_params" != "" ]; then - sink_uri="${sink_uri}&${extra_params}" - fi - echo "$sink_uri" -} + # record tso before we create tables to skip the system table DDLs + start_ts=$(run_cdc_cli_tso_query $UP_PD_HOST_1 $UP_PD_PORT_1) -function start_kafka_consumer() { - local work_dir=$1 - local sink_uri=$2 - local schema_registry_uri=$3 - local protocol_case=$4 - local downstream_uri="mysql://root@${DOWN_TIDB_HOST}:${DOWN_TIDB_PORT}/?safe-mode=true&batch-dml-enable=false&enable-ddl-ts=false" - local args=( - --log-file "$work_dir/cdc_kafka_consumer.log" - --log-level debug - --upstream-uri "$sink_uri" - --downstream-uri "$downstream_uri" - ) - if [ "$schema_registry_uri" != "" ]; then - args+=(--schema-registry-uri "$schema_registry_uri") - fi - if [[ "$protocol_case" == simple_* ]]; then - args+=(--upstream-tidb-dsn "root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?") - fi + run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY - cdc_kafka_consumer "${args[@]}" >>"$work_dir/cdc_kafka_consumer_stdout.log" 2>&1 & - consumer_pid=$! -} + # Use a max-message-bytes parameter that is larger than the kafka topic max message bytes. + # Test if TiCDC automatically uses the max-message-bytes of the topic. + # See: https://github.com/PingCAP-QE/ci/blob/ddde195ebf4364a0028d53405d1194aa37a4d853/jenkins/pipelines/ci/ticdc/cdc_ghpr_kafka_integration_test.groovy#L178 + # Use a topic that has already been created. + # See: https://github.com/PingCAP-QE/ci/blob/ddde195ebf4364a0028d53405d1194aa37a4d853/jenkins/pipelines/ci/ticdc/cdc_ghpr_kafka_integration_test.groovy#L180 + SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=open-protocol&partition-num=1&kafka-version=${KAFKA_VERSION}&max-message-bytes=12582912" + cdc_cli_changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI" + run_kafka_consumer $WORK_DIR "kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=open-protocol&partition-num=1&version=${KAFKA_VERSION}" -function stop_kafka_consumer() { - if [ "$consumer_pid" != "" ]; then - kill -9 "$consumer_pid" 2>/dev/null || true - wait "$consumer_pid" 2>/dev/null || true - consumer_pid="" + echo "Starting generate kafka big messages..." + cd $CUR/../../utils/gen_kafka_big_messages + if [ ! -f ./gen_kafka_big_messages ]; then + GO111MODULE=on go build fi -} - -function wait_changefeed_state() { - local pd_addr=$1 - local changefeed_id=$2 - local expected_state=$3 - local expected_error=$4 - local deadline=$((SECONDS + STATE_WAIT_TIMEOUT_SECONDS)) - - while true; do - if check_changefeed_state "$pd_addr" "$changefeed_id" "$expected_state" "$expected_error" ""; then - return - fi - if [ "$SECONDS" -ge "$deadline" ]; then - echo "changefeed $changefeed_id did not reach state $expected_state within ${STATE_WAIT_TIMEOUT_SECONDS}s" - return 1 - fi - sleep "$STATE_CHECK_INTERVAL_SECONDS" - done -} - -function render_diff_config() { - local work_dir=$1 - local database_name=$2 - local diff_config=$3 - - sed -e "s/database_name/${database_name}/g" \ - -e "s|/tmp/tidb_cdc_test/kafka_big_messages/sync_diff/output|${work_dir}/sync_diff/output|g" \ - "$CUR/conf/diff_config.toml" >"$diff_config" -} - -function run_protocol_case() { - local protocol_case=$1 - local protocol=$2 - local schema_registry_uri=$3 - local extra_params=$4 - local topic_case=${protocol_case//_/-} - local topic_name="big-message-${topic_case}-${RANDOM}" - local changefeed_id="kafka-big-messages-${topic_case}" - local database_name="kafka_big_messages_${protocol_case}" - local work_dir="$WORK_DIR/$protocol_case" - local sql_file="$work_dir/test.sql" - local diff_config="$work_dir/diff_config.toml" - local pd_addr="http://${UP_PD_HOST_1}:${UP_PD_PORT_1}" - local sink_uri - local initial_topic_limit=$SMALL_TOPIC_LIMIT - local expected_error=ErrMessageTooLarge - if [ "$protocol_case" = "async_error" ]; then - initial_topic_limit=$LARGE_TOPIC_LIMIT - expected_error=ErrKafkaSendMessage - fi - - mkdir -p "$work_dir" - render_diff_config "$work_dir" "$database_name" "$diff_config" - kafka_topic --topic "$topic_name" --max-message-bytes "$initial_topic_limit" - local start_ts - start_ts=$(run_cdc_cli_tso_query "$UP_PD_HOST_1" "$UP_PD_PORT_1") - sink_uri=$(kafka_sink_uri "$topic_name" "$protocol" "$extra_params") - - if [ "$schema_registry_uri" != "" ]; then - cdc_cli_changefeed create --start-ts="$start_ts" --sink-uri="$sink_uri" --schema-registry="$schema_registry_uri" -c "$changefeed_id" - else - cdc_cli_changefeed create --start-ts="$start_ts" --sink-uri="$sink_uri" -c "$changefeed_id" - fi - start_kafka_consumer "$work_dir" "$sink_uri" "$schema_registry_uri" "$protocol_case" - wait_changefeed_state "$pd_addr" "$changefeed_id" "normal" "null" - - # Lower the topic limit after the producer has started. The encoder and - # producer still accept the message, then Kafka rejects it asynchronously. - if [ "$protocol_case" = "async_error" ]; then - local ready_database="${database_name}_ready" - run_sql "CREATE DATABASE ${ready_database}; CREATE TABLE ${ready_database}.ready(id INT PRIMARY KEY); INSERT INTO ${ready_database}.ready VALUES (1)" "$UP_TIDB_HOST" "$UP_TIDB_PORT" - ensure "$TABLE_CHECK_RETRIES" "run_sql 'SELECT id FROM ${ready_database}.ready' '$DOWN_TIDB_HOST' '$DOWN_TIDB_PORT' && check_contains 'id: 1'" - kafka_topic --topic "$topic_name" --max-message-bytes "$SMALL_TOPIC_LIMIT" --alter - fi - - "$GENERATOR_DIR/gen_kafka_big_messages" --row-bytes="$ROW_BYTES" --row-count=1 --database-name="$database_name" --table-name=test --sql-file-path="$sql_file" - run_sql_file "$sql_file" "$UP_TIDB_HOST" "$UP_TIDB_PORT" - run_sql "CREATE TABLE ${database_name}.finish_mark(id INT PRIMARY KEY)" "$UP_TIDB_HOST" "$UP_TIDB_PORT" - - wait_changefeed_state "$pd_addr" "$changefeed_id" "warning" "$expected_error" - - # Only increase Kafka's topic limit. TiCDC must recreate the sink, read the - # new limit, and resume without updating, pausing, or resuming the changefeed. - kafka_topic --topic "$topic_name" --max-message-bytes "$LARGE_TOPIC_LIMIT" --alter - wait_changefeed_state "$pd_addr" "$changefeed_id" "normal" "null" - check_table_exists "${database_name}.finish_mark" "$DOWN_TIDB_HOST" "$DOWN_TIDB_PORT" "$TABLE_CHECK_RETRIES" - check_sync_diff "$work_dir" "$diff_config" - - cdc_cli_changefeed remove -c "$changefeed_id" - stop_kafka_consumer -} - -function run() { - # Test Kafka sink only in this case. - if [ "$SINK_TYPE" != "kafka" ]; then - return - fi - - local cases=( - "canal_json|canal-json||enable-tidb-extension=true" - "open_protocol|open-protocol||" - "async_error|open-protocol||max-retry=0" - "simple_json|simple||" - "simple_avro|simple||encoding-format=avro" - "avro|avro|$SCHEMA_REGISTRY_URI|enable-tidb-extension=true&avro-enable-watermark=true&avro-decimal-handling-mode=string&avro-bigint-unsigned-handling-mode=string" - ) - - rm -rf "$WORK_DIR" && mkdir -p "$WORK_DIR" - start_schema_registry - build_message_generator - start_tidb_cluster --workdir "$WORK_DIR" - run_cdc_server --workdir "$WORK_DIR" --binary "$CDC_BINARY" + # Generate data larger than kafka broker max.message.bytes. We can send this data correctly. + ./gen_kafka_big_messages --row-count=15 --sql-file-path=$CUR/test.sql - local case_entry - for case_entry in "${cases[@]}"; do - local protocol_case protocol schema_registry_uri extra_params - IFS='|' read -r protocol_case protocol schema_registry_uri extra_params <<<"$case_entry" - run_protocol_case "$protocol_case" "$protocol" "$schema_registry_uri" "$extra_params" - done + run_sql_file $CUR/test.sql ${UP_TIDB_HOST} ${UP_TIDB_PORT} + table="kafka_big_messages.test" + check_table_exists $table ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} + check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml - cleanup_process "$CDC_BINARY" + cleanup_process $CDC_BINARY } -trap 'stop_kafka_consumer; stop_test "$WORK_DIR"' EXIT -run "$@" -check_logs "$WORK_DIR" +trap 'stop_test $WORK_DIR' EXIT +run $* +check_logs $WORK_DIR echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>" diff --git a/tests/integration_tests/kafka_compression/run.sh b/tests/integration_tests/kafka_compression/run.sh index a57cd937b1..e0f85df648 100755 --- a/tests/integration_tests/kafka_compression/run.sh +++ b/tests/integration_tests/kafka_compression/run.sh @@ -18,6 +18,11 @@ function test_compression() { run_kafka_consumer $WORK_DIR "kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=canal-json&version=${KAFKA_VERSION}&enable-tidb-extension=true" run_sql_file $CUR/data/$1_data.sql ${UP_TIDB_HOST} ${UP_TIDB_PORT} + compression_algorithm=$(grep "Kafka producer uses $1 compression algorithm" "$WORK_DIR/cdc.log") + if [[ "$compression_algorithm" -ne 1 ]]; then + echo "can't found producer compression algorithm" + exit 1 + fi check_table_exists test.$1_finish_mark ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 200 check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml cdc_cli_changefeed pause -c $1 diff --git a/tests/integration_tests/kafka_simple_claim_check/data/data.sql b/tests/integration_tests/kafka_simple_claim_check/data/data.sql index c753734f2e..2730d96d71 100644 --- a/tests/integration_tests/kafka_simple_claim_check/data/data.sql +++ b/tests/integration_tests/kafka_simple_claim_check/data/data.sql @@ -1,6 +1,4 @@ use test; --- Keep the encoded row larger than max-message-bytes after Snappy compression, --- so this case exercises claim-check instead of sending the full row to Kafka. insert into t values ( 1, 1, 2, 3, 4, 5, @@ -9,7 +7,7 @@ insert into t values ( 3.1415, 2.7182, 8000, 179394.233, '2020-02-20', '2020-02-20 02:20:20', '2020-02-20 02:20:20', '02:20:20', '2020', '89504E470D0A1A0A', '89504E470D0A1A0A', '89504E470D0A1A0A', '89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A', - x'89504E470D0A1A0A', RANDOM_BYTES(1024), RANDOM_BYTES(1024), RANDOM_BYTES(1024), + x'89504E470D0A1A0A', x'89504E470D0A1A0A', x'89504E470D0A1A0A', x'89504E470D0A1A0A', '89504E470D0A1A0A', '89504E470D0A1A0A', x'89504E470D0A1A0A', x'89504E470D0A1A0A', 'b', 'b,c', b'1000001', '{ "key1": "value1", @@ -30,7 +28,7 @@ insert into t values ( 3.1415, 2.7182, 8000, 179394.233, '2020-02-20', '2020-02-20 02:20:20', '2020-02-20 02:20:20', '02:20:20', '2020', '89504E470D0A1A0A', '89504E470D0A1A0A', '89504E470D0A1A0A', '89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A', - x'89504E470D0A1A0A', RANDOM_BYTES(1024), RANDOM_BYTES(1024), RANDOM_BYTES(1024), + x'89504E470D0A1A0A', x'89504E470D0A1A0A', x'89504E470D0A1A0A', x'89504E470D0A1A0A', '89504E470D0A1A0A', '89504E470D0A1A0A', x'89504E470D0A1A0A', x'89504E470D0A1A0A', 'b', 'b,c', b'1000001', '{ "key1": "value1", diff --git a/tests/integration_tests/kafka_simple_claim_check/run.sh b/tests/integration_tests/kafka_simple_claim_check/run.sh index a0ff8de268..c8414aee56 100755 --- a/tests/integration_tests/kafka_simple_claim_check/run.sh +++ b/tests/integration_tests/kafka_simple_claim_check/run.sh @@ -24,8 +24,6 @@ function run() { run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY TOPIC_NAME="kafka-simple-claim-check-$RANDOM" - CLAIM_CHECK_DIR="/tmp/kafka-simple-claim-check" - rm -rf "$CLAIM_CHECK_DIR" # record tso before we create tables to skip the system table DDLs start_ts=$(run_cdc_cli_tso_query ${UP_PD_HOST_1} ${UP_PD_PORT_1}) @@ -39,7 +37,6 @@ function run() { cdc_cli_changefeed pause -c ${changefeed_id} - kafka_topic --topic "$TOPIC_NAME" --max-message-bytes 2048 --alter SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=simple&max-message-bytes=2048" cdc_cli_changefeed update -c ${changefeed_id} --sink-uri="$SINK_URI" --config="$CUR/conf/changefeed.toml" --no-confirm cdc_cli_changefeed resume -c ${changefeed_id} @@ -51,10 +48,6 @@ function run() { # sync_diff can't check non-exist table, so we check expected tables are created in downstream first check_table_exists test.finish_mark ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 200 check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml - if ! find "$CLAIM_CHECK_DIR" -type f -print -quit | grep -q .; then - echo "claim-check did not write any file to $CLAIM_CHECK_DIR" - exit 1 - fi cleanup_process $CDC_BINARY } diff --git a/tests/integration_tests/kafka_simple_claim_check_avro/data/data.sql b/tests/integration_tests/kafka_simple_claim_check_avro/data/data.sql index c753734f2e..2730d96d71 100644 --- a/tests/integration_tests/kafka_simple_claim_check_avro/data/data.sql +++ b/tests/integration_tests/kafka_simple_claim_check_avro/data/data.sql @@ -1,6 +1,4 @@ use test; --- Keep the encoded row larger than max-message-bytes after Snappy compression, --- so this case exercises claim-check instead of sending the full row to Kafka. insert into t values ( 1, 1, 2, 3, 4, 5, @@ -9,7 +7,7 @@ insert into t values ( 3.1415, 2.7182, 8000, 179394.233, '2020-02-20', '2020-02-20 02:20:20', '2020-02-20 02:20:20', '02:20:20', '2020', '89504E470D0A1A0A', '89504E470D0A1A0A', '89504E470D0A1A0A', '89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A', - x'89504E470D0A1A0A', RANDOM_BYTES(1024), RANDOM_BYTES(1024), RANDOM_BYTES(1024), + x'89504E470D0A1A0A', x'89504E470D0A1A0A', x'89504E470D0A1A0A', x'89504E470D0A1A0A', '89504E470D0A1A0A', '89504E470D0A1A0A', x'89504E470D0A1A0A', x'89504E470D0A1A0A', 'b', 'b,c', b'1000001', '{ "key1": "value1", @@ -30,7 +28,7 @@ insert into t values ( 3.1415, 2.7182, 8000, 179394.233, '2020-02-20', '2020-02-20 02:20:20', '2020-02-20 02:20:20', '02:20:20', '2020', '89504E470D0A1A0A', '89504E470D0A1A0A', '89504E470D0A1A0A', '89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A', - x'89504E470D0A1A0A', RANDOM_BYTES(1024), RANDOM_BYTES(1024), RANDOM_BYTES(1024), + x'89504E470D0A1A0A', x'89504E470D0A1A0A', x'89504E470D0A1A0A', x'89504E470D0A1A0A', '89504E470D0A1A0A', '89504E470D0A1A0A', x'89504E470D0A1A0A', x'89504E470D0A1A0A', 'b', 'b,c', b'1000001', '{ "key1": "value1", diff --git a/tests/integration_tests/kafka_simple_claim_check_avro/run.sh b/tests/integration_tests/kafka_simple_claim_check_avro/run.sh index ac3626634e..259d526d59 100755 --- a/tests/integration_tests/kafka_simple_claim_check_avro/run.sh +++ b/tests/integration_tests/kafka_simple_claim_check_avro/run.sh @@ -24,8 +24,6 @@ function run() { run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY TOPIC_NAME="kafka-simple-claim-check-avro-$RANDOM" - CLAIM_CHECK_DIR="/tmp/kafka-simple-avro-claim-check" - rm -rf "$CLAIM_CHECK_DIR" # record tso before we create tables to skip the system table DDLs start_ts=$(run_cdc_cli_tso_query ${UP_PD_HOST_1} ${UP_PD_PORT_1}) @@ -39,7 +37,6 @@ function run() { cdc_cli_changefeed pause -c ${changefeed_id} - kafka_topic --topic "$TOPIC_NAME" --max-message-bytes 2048 --alter SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=simple&encoding-format=avro&max-message-bytes=2048" cdc_cli_changefeed update -c ${changefeed_id} --sink-uri="$SINK_URI" --config="$CUR/conf/changefeed.toml" --no-confirm cdc_cli_changefeed resume -c ${changefeed_id} @@ -51,10 +48,6 @@ function run() { # sync_diff can't check non-exist table, so we check expected tables are created in downstream first check_table_exists test.finish_mark ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 200 check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml - if ! find "$CLAIM_CHECK_DIR" -type f -print -quit | grep -q .; then - echo "claim-check did not write any file to $CLAIM_CHECK_DIR" - exit 1 - fi cleanup_process $CDC_BINARY } diff --git a/tests/integration_tests/kafka_simple_handle_key_only/run.sh b/tests/integration_tests/kafka_simple_handle_key_only/run.sh index e7b9f38884..32f7ecc6e3 100755 --- a/tests/integration_tests/kafka_simple_handle_key_only/run.sh +++ b/tests/integration_tests/kafka_simple_handle_key_only/run.sh @@ -36,7 +36,6 @@ function run() { cdc_cli_changefeed pause -c ${changefeed_id} - kafka_topic --topic "$TOPIC_NAME" --max-message-bytes 700 --alter SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=simple&max-message-bytes=700" cdc_cli_changefeed update -c ${changefeed_id} --sink-uri="$SINK_URI" --config="$CUR/conf/changefeed.toml" --no-confirm cdc_cli_changefeed resume -c ${changefeed_id} diff --git a/tests/integration_tests/kafka_simple_handle_key_only_avro/run.sh b/tests/integration_tests/kafka_simple_handle_key_only_avro/run.sh index 94ea60d97b..717d3924d3 100755 --- a/tests/integration_tests/kafka_simple_handle_key_only_avro/run.sh +++ b/tests/integration_tests/kafka_simple_handle_key_only_avro/run.sh @@ -36,7 +36,6 @@ function run() { cdc_cli_changefeed pause -c ${changefeed_id} - kafka_topic --topic "$TOPIC_NAME" --max-message-bytes 650 --alter SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=simple&encoding-format=avro&max-message-bytes=650" cdc_cli_changefeed update -c ${changefeed_id} --sink-uri="$SINK_URI" --config="$CUR/conf/changefeed.toml" --no-confirm cdc_cli_changefeed resume -c ${changefeed_id} diff --git a/tests/integration_tests/log_redaction/run.sh b/tests/integration_tests/log_redaction/run.sh index 71b7e1e656..b99fe5da90 100755 --- a/tests/integration_tests/log_redaction/run.sh +++ b/tests/integration_tests/log_redaction/run.sh @@ -372,6 +372,97 @@ function run() { echo "[$(date)] ✓ MySQL sink: All redaction modes validated" fi + # ========================================================================== + # Test 4b: Kafka sink validation (tests Kafka-specific redaction) + # ========================================================================== + if [ "$SINK_TYPE" = "kafka" ]; then + echo "" + echo "=== Test 4b: Kafka sink redaction validation ===" + + # Kafka sink logs message key/value at DEBUG level + # Log message: "send message to kafka" with messageKey and messageValue fields + + # Test ON mode with Kafka sink (most important - full redaction) + echo " [4b-1] ON mode with Kafka sink:" + run_sql "DROP DATABASE IF EXISTS log_redaction_test;" + run_sql "CREATE DATABASE log_redaction_test;" + + KAFKA_TOPIC="log-redaction-test-$RANDOM" + KAFKA_SINK_URI="kafka://127.0.0.1:9092/$KAFKA_TOPIC?protocol=open-protocol" + + run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY --redact-info-log on --logsuffix "_on_kafka" + + cdc_cli_changefeed create --sink-uri="$KAFKA_SINK_URI" --changefeed-id="kafka-on-test" --config=$CUR/conf/changefeed.toml + + run_sql_file $CUR/data/test.sql ${UP_TIDB_HOST} ${UP_TIDB_PORT} + + echo " Waiting for Kafka sink to process events..." + wait_for_log_content "$WORK_DIR/cdc_on_kafka.log" "send message to kafka" "Kafka message logs" 30 + + echo " [Validation] ON mode with Kafka sink:" + echo "" + + # Capture Kafka logs once for all validations + captured_logs=$(grep "send message to kafka" "$WORK_DIR/cdc_on_kafka.log" 2>/dev/null || echo "") + log_raw_content "Kafka message logs (ON mode)" "$captured_logs" + + # STRICT POSITIVE VALIDATION: messageKey and messageValue must show redacted format + echo " [1/2] Verifying Kafka logs show redacted '?' placeholder:" + require_log_pattern "$WORK_DIR/cdc_on_kafka.log" \ + "send message to kafka.*messageKey.*\?.*messageValue.*\?" \ + "Kafka messageKey and messageValue redacted to '?'" \ + "ON mode should redact both messageKey and messageValue" + + # STRICT NEGATIVE VALIDATION: No sensitive data should leak in Kafka logs + echo " [2/2] Verifying NO sensitive data leaks in Kafka logs:" + sensitive_patterns=("Password1!" "SecretPass1!" "user1@example.com" "4532-1000-1000") + for pattern in "${sensitive_patterns[@]}"; do + require_no_log_pattern "$WORK_DIR/cdc_on_kafka.log" \ + "$pattern" \ + "No leak of sensitive value in Kafka logs: $pattern" + done + + captured_logs="" + cleanup_process $CDC_BINARY + + # Test MARKER mode with Kafka sink + echo "" + echo " [4b-2] MARKER mode with Kafka sink:" + run_sql "DROP DATABASE IF EXISTS log_redaction_test;" + run_sql "CREATE DATABASE log_redaction_test;" + + KAFKA_TOPIC="log-redaction-marker-$RANDOM" + KAFKA_SINK_URI="kafka://127.0.0.1:9092/$KAFKA_TOPIC?protocol=open-protocol" + + run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY --redact-info-log marker --logsuffix "_marker_kafka" + + cdc_cli_changefeed create --sink-uri="$KAFKA_SINK_URI" --changefeed-id="kafka-marker-test" --config=$CUR/conf/changefeed.toml + + run_sql_file $CUR/data/test.sql ${UP_TIDB_HOST} ${UP_TIDB_PORT} + + echo " Waiting for Kafka sink to process events..." + wait_for_log_content "$WORK_DIR/cdc_marker_kafka.log" "send message to kafka" "Kafka message logs" 30 + + echo " [Validation] MARKER mode with Kafka sink:" + echo "" + + # Capture Kafka logs once for all validations + captured_logs=$(grep "send message to kafka" "$WORK_DIR/cdc_marker_kafka.log" 2>/dev/null || echo "") + log_raw_content "Kafka message logs (MARKER mode)" "$captured_logs" + + # STRICT POSITIVE VALIDATION: messageKey and messageValue must have markers + echo " [1/1] Verifying Kafka logs have ‹› markers:" + require_log_pattern "$WORK_DIR/cdc_marker_kafka.log" \ + "send message to kafka.*‹" \ + "Kafka message values wrapped with ‹› markers" \ + "MARKER mode should wrap Kafka message data with ‹› markers" + + captured_logs="" + cleanup_process $CDC_BINARY + + echo "[$(date)] ✓ Kafka sink: Redaction modes validated" + fi + # ========================================================================== # Test 5: API mode switching # ========================================================================== diff --git a/tests/integration_tests/open_protocol_claim_check/data/data.sql b/tests/integration_tests/open_protocol_claim_check/data/data.sql index dbacc55cf2..14ae2db77e 100644 --- a/tests/integration_tests/open_protocol_claim_check/data/data.sql +++ b/tests/integration_tests/open_protocol_claim_check/data/data.sql @@ -93,7 +93,6 @@ insert into t values ( ); update t set c_float = 3.1415, c_double = 2.7182, c_decimal = 8000, c_decimal_2 = 179394.233 where id = 2; -update t set c_longblob = concat(random_bytes(1024), random_bytes(1024), random_bytes(1024)) where id = 2; begin; diff --git a/tests/integration_tests/open_protocol_claim_check/run.sh b/tests/integration_tests/open_protocol_claim_check/run.sh index 7f9f94e3be..2b262fd3df 100755 --- a/tests/integration_tests/open_protocol_claim_check/run.sh +++ b/tests/integration_tests/open_protocol_claim_check/run.sh @@ -18,10 +18,7 @@ function run() { start_tidb_cluster --workdir $WORK_DIR - TOPIC_NAME="open-protocol-claim-check-$RANDOM" - CLAIM_CHECK_DIR="/tmp/open-protocol-claim-check" - rm -rf "$CLAIM_CHECK_DIR" - kafka_topic --topic "$TOPIC_NAME" --max-message-bytes 2048 + TOPIC_NAME="open-protocol-claim-check" # record tso before we create tables to skip the system table DDLs start_ts=$(run_cdc_cli_tso_query ${UP_PD_HOST_1} ${UP_PD_PORT_1}) @@ -43,10 +40,6 @@ function run() { # sync_diff can't check non-exist table, so we check expected tables are created in downstream first check_table_exists test.finish_mark ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 200 check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml - if ! find "$CLAIM_CHECK_DIR" -type f -print -quit | grep -q .; then - echo "claim-check did not write any file to $CLAIM_CHECK_DIR" - exit 1 - fi cleanup_process $CDC_BINARY } diff --git a/tests/integration_tests/open_protocol_handle_key_only/data/data.sql b/tests/integration_tests/open_protocol_handle_key_only/data/data.sql index 2c79413baf..2977b9aa12 100644 --- a/tests/integration_tests/open_protocol_handle_key_only/data/data.sql +++ b/tests/integration_tests/open_protocol_handle_key_only/data/data.sql @@ -93,7 +93,6 @@ insert into t values ( ); update t set c_float = 3.1415, c_double = 2.7182, c_decimal = 8000, c_decimal_2 = 179394.233 where id = 2; -update t set c_longblob = concat(random_bytes(1024), random_bytes(1024), random_bytes(1024)) where id = 2; create table finish_mark ( diff --git a/tests/integration_tests/open_protocol_handle_key_only/run.sh b/tests/integration_tests/open_protocol_handle_key_only/run.sh index 01aab001dc..a416274e1d 100755 --- a/tests/integration_tests/open_protocol_handle_key_only/run.sh +++ b/tests/integration_tests/open_protocol_handle_key_only/run.sh @@ -19,7 +19,6 @@ function run() { start_tidb_cluster --workdir $WORK_DIR TOPIC_NAME="open-protocol-handle-key-only-$RANDOM" - kafka_topic --topic "$TOPIC_NAME" --max-message-bytes 2048 # record tso before we create tables to skip the system table DDLs start_ts=$(run_cdc_cli_tso_query ${UP_PD_HOST_1} ${UP_PD_PORT_1}) diff --git a/tests/integration_tests/run_light_it_in_ci.sh b/tests/integration_tests/run_light_it_in_ci.sh index 22a85bb36a..647fd77f04 100755 --- a/tests/integration_tests/run_light_it_in_ci.sh +++ b/tests/integration_tests/run_light_it_in_ci.sh @@ -93,7 +93,7 @@ kafka_groups=( # G08 'capture_session_done_during_task fail_over_ddl_I table_route' # G09 - 'cdc_server_tips ddl_sequence fail_over_ddl_J' + 'cdc_server_tips ddl_sequence log_redaction fail_over_ddl_J' # G10 'changefeed_error batch_add_table fail_over_ddl_K split_table_check' # G11 diff --git a/tests/utils/kafka_topic/main.go b/tests/utils/kafka_topic/main.go deleted file mode 100644 index 6227492cf9..0000000000 --- a/tests/utils/kafka_topic/main.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "flag" - "log" - "strconv" - "strings" - - "github.com/IBM/sarama" -) - -func main() { - brokers := flag.String("brokers", "127.0.0.1:9092", "Comma-separated Kafka broker addresses.") - topic := flag.String("topic", "", "Kafka topic name.") - maxMessageBytes := flag.Int("max-message-bytes", 0, "Topic max.message.bytes value.") - alter := flag.Bool("alter", false, "Alter an existing topic instead of creating it.") - flag.Parse() - - if *topic == "" { - log.Fatal("topic must not be empty") - } - if *maxMessageBytes <= 0 { - log.Fatal("max-message-bytes must be greater than zero") - } - - value := strconv.Itoa(*maxMessageBytes) - config := sarama.NewConfig() - config.ClientID = "ticdc-integration-test-kafka-topic" - admin, err := sarama.NewClusterAdmin(strings.Split(*brokers, ","), config) - if err != nil { - log.Fatalf("create Kafka admin client: %v", err) - } - defer func() { - if err := admin.Close(); err != nil { - log.Printf("close Kafka admin client: %v", err) - } - }() - - configEntries := map[string]*string{"max.message.bytes": &value} - if *alter { - if err := admin.AlterConfig(sarama.TopicResource, *topic, configEntries, false); err != nil { - log.Fatalf("alter Kafka topic %s: %v", *topic, err) - } - return - } - - detail := &sarama.TopicDetail{ - NumPartitions: 1, - ReplicationFactor: 1, - ConfigEntries: configEntries, - } - if err := admin.CreateTopic(*topic, detail, false); err != nil { - log.Fatalf("create Kafka topic %s: %v", *topic, err) - } -} From a24012c068a999132cec2d185a03e6f6bb24188d Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Mon, 17 Aug 2026 12:19:44 +0800 Subject: [PATCH 2/6] kafka: remove unnecessary revert logs --- downstreamadapter/sink/kafka/sink.go | 15 --------------- pkg/sink/kafka/options.go | 1 - 2 files changed, 16 deletions(-) diff --git a/downstreamadapter/sink/kafka/sink.go b/downstreamadapter/sink/kafka/sink.go index 76e0253f2e..96430a5a1d 100644 --- a/downstreamadapter/sink/kafka/sink.go +++ b/downstreamadapter/sink/kafka/sink.go @@ -289,9 +289,6 @@ func (s *sink) calculateKeyPartitions(ctx context.Context) error { default: event, ok := s.eventChan.Get() if !ok { - log.Info("kafka sink event channel closed", - zap.String("keyspace", s.changefeedID.Keyspace()), - zap.String("changefeed", s.changefeedID.Name())) return nil } schema := event.TableInfo.GetSchemaName() @@ -351,9 +348,6 @@ func (s *sink) nonBatchEncodeRun(ctx context.Context) error { default: event, ok := s.rowChan.Get() if !ok { - log.Info("kafka sink event channel closed", - zap.String("keyspace", s.changefeedID.Keyspace()), - zap.String("changefeed", s.changefeedID.Name())) return nil } if err := s.comp.encoderGroup.AddEvents(ctx, event.Key, &event.RowEvent); err != nil { @@ -409,9 +403,6 @@ func (s *sink) batch(ctx context.Context, buffer []*commonEvent.MQRowEvent) ([]* default: msgs, ok := s.rowChan.GetMultipleNoGroup(buffer) if !ok { - log.Info("kafka sink event channel closed", - zap.String("keyspace", s.changefeedID.Keyspace()), - zap.String("changefeed", s.changefeedID.Name())) return nil, nil } buffer = buffer[:0] @@ -443,9 +434,6 @@ func (s *sink) sendMessages(ctx context.Context) error { return context.Cause(ctx) case future, ok := <-outCh: if !ok { - log.Info("kafka sink encoder's output channel closed", - zap.String("keyspace", s.changefeedID.Keyspace()), - zap.String("changefeed", s.changefeedID.Name())) return nil } if err = future.Ready(ctx); err != nil { @@ -549,9 +537,6 @@ func (s *sink) sendCheckpoint(ctx context.Context) error { return context.Cause(ctx) case ts, ok := <-s.checkpointChan: if !ok { - log.Warn("kafka sink checkpoint channel closed", - zap.String("keyspace", s.changefeedID.Keyspace()), - zap.String("changefeed", s.changefeedID.Name())) return nil } diff --git a/pkg/sink/kafka/options.go b/pkg/sink/kafka/options.go index 5bc933a3bf..f783816176 100644 --- a/pkg/sink/kafka/options.go +++ b/pkg/sink/kafka/options.go @@ -494,7 +494,6 @@ func (o *options) applySASL(urlParameter *urlConfig, sinkConfig *config.SinkConf // BASE64 decode the client secret decodedClientSecret, err := base64.StdEncoding.DecodeString(clientSecret) if err != nil { - log.Error("OAuth2 client secret is not base64 encoded", zap.Error(err)) return errors.ErrKafkaInvalidConfig.GenWithStack("OAuth2 client secret is not base64 encoded") } o.SASL.OAuth2.ClientSecret = string(decodedClientSecret) From e9932606f3d988bdda8aae60299df09aae10244d Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Mon, 17 Aug 2026 13:38:22 +0800 Subject: [PATCH 3/6] kafka: preserve post-revert fixes --- .../sink/topicmanager/kafka_topic_manager.go | 27 ++- .../topicmanager/kafka_topic_manager_test.go | 72 +++--- pkg/config/large_message.go | 20 +- pkg/config/large_message_test.go | 215 ++++++++++++++++++ pkg/sink/codec/open/encoder_test.go | 16 +- 5 files changed, 288 insertions(+), 62 deletions(-) create mode 100644 pkg/config/large_message_test.go diff --git a/downstreamadapter/sink/topicmanager/kafka_topic_manager.go b/downstreamadapter/sink/topicmanager/kafka_topic_manager.go index 4ccd04eea8..eddd0cfe6c 100644 --- a/downstreamadapter/sink/topicmanager/kafka_topic_manager.go +++ b/downstreamadapter/sink/topicmanager/kafka_topic_manager.go @@ -208,33 +208,32 @@ func (m *kafkaTopicManager) waitUntilTopicVisible( ctx context.Context, topicName string, ) error { + start := time.Now() topics := []string{topicName} err := retry.Do(ctx, func() error { - start := time.Now() // ignoreTopicError is set to false since we just create the topic, // make sure the topic is visible. meta, err := m.admin.GetTopicsMeta(topics, false) if err != nil { - log.Warn("topic not found, retry it", - zap.String("keyspace", m.changefeedID.Keyspace()), - zap.String("changefeed", m.changefeedID.Name()), - zap.Error(err), - zap.Duration("duration", time.Since(start)), - ) return err } - log.Info("topic found", - zap.String("keyspace", m.changefeedID.Keyspace()), - zap.String("changefeed", m.changefeedID.Name()), - zap.String("topic", topicName), - zap.Int32("partitionNumber", meta[topicName].NumPartitions), - zap.Duration("duration", time.Since(start))) + _, ok := meta[topicName] + if !ok { + return errors.ErrKafkaAdminAPI.GenWithStackByArgs("describe-topic", topicName) + } return nil }, retry.WithBackoffBaseDelay(500), retry.WithBackoffMaxDelay(1000), retry.WithMaxTries(6), ) - + if err != nil { + log.Warn("kafka topic metadata refresh failed", + zap.String("keyspace", m.changefeedID.Keyspace()), + zap.String("changefeed", m.changefeedID.Name()), + zap.String("topic", topicName), + zap.Duration("duration", time.Since(start)), + zap.Error(err)) + } return err } diff --git a/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go b/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go index b8e93c588e..13b371f697 100644 --- a/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go +++ b/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go @@ -212,44 +212,50 @@ func TestEnsureTopicExistsWaitsUntilVisible(t *testing.T) { ctrl := gomock.NewController(t) adminClient := kafka.NewMockClusterAdminClient(ctrl) - cfg := &kafka.AutoCreateTopicConfig{ - AutoCreate: true, - PartitionNum: 2, - ReplicationFactor: 1, - } - - topic := "delayed-topic" - gomock.InOrder( - adminClient.EXPECT().GetTopicsMeta([]string{topic}, true).Return( - map[string]kafka.TopicDetail{}, nil), - adminClient.EXPECT().GetTopicsMeta([]string{topic}, false).Return( - map[string]kafka.TopicDetail{}, nil), - adminClient.EXPECT().CreateTopic(gomock.Any()).DoAndReturn( - func(detail *kafka.TopicDetail) error { - require.Equal(t, &kafka.TopicDetail{ - Name: topic, - NumPartitions: 2, - ReplicationFactor: 1, - }, detail) - return nil - }), - adminClient.EXPECT().GetTopicsMeta([]string{topic}, false).Return( - nil, sarama.ErrUnknownTopicOrPartition), - adminClient.EXPECT().GetTopicsMeta([]string{topic}, false).Return( - nil, sarama.ErrUnknownTopicOrPartition), - adminClient.EXPECT().GetTopicsMeta([]string{topic}, false).Return( - map[string]kafka.TopicDetail{ - topic: { - Name: topic, + created := false + postCreateDescribeCount := 0 + adminClient.EXPECT().GetTopicsMeta([]string{"delayed-topic"}, true).Return(map[string]kafka.TopicDetail{}, nil) + adminClient.EXPECT().GetTopicsMeta([]string{"delayed-topic"}, false).DoAndReturn( + func([]string, bool) (map[string]kafka.TopicDetail, error) { + if !created { + return map[string]kafka.TopicDetail{}, nil + } + postCreateDescribeCount++ + if postCreateDescribeCount == 1 { + return map[string]kafka.TopicDetail{}, nil + } + return map[string]kafka.TopicDetail{ + "delayed-topic": { + Name: "delayed-topic", NumPartitions: 2, }, - }, nil), + }, nil + }).Times(3) + adminClient.EXPECT().CreateTopic(gomock.Any()).DoAndReturn( + func(detail *kafka.TopicDetail) error { + require.Equal(t, &kafka.TopicDetail{ + Name: "delayed-topic", + NumPartitions: 2, + ReplicationFactor: 1, + }, detail) + created = true + return nil + }) + + err := EnsureTopic( + context.Background(), + common.NewChangefeedID4Test("test", "test"), + "delayed-topic", + &kafka.AutoCreateTopicConfig{ + AutoCreate: true, + PartitionNum: 2, + ReplicationFactor: 1, + }, + adminClient, ) - ctx := context.Background() - changefeedID := common.NewChangefeedID4Test("test", "test") - err := EnsureTopic(ctx, changefeedID, topic, cfg, adminClient) require.NoError(t, err) + require.Equal(t, 2, postCreateDescribeCount) } func TestGetTopicManagerStartsBackgroundRefreshAfterTopicReady(t *testing.T) { diff --git a/pkg/config/large_message.go b/pkg/config/large_message.go index d04584b451..6b19afe260 100644 --- a/pkg/config/large_message.go +++ b/pkg/config/large_message.go @@ -15,7 +15,7 @@ package config import ( "github.com/pingcap/ticdc/pkg/compression" - cerror "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/errors" ) const ( @@ -55,34 +55,39 @@ func (c *LargeMessageHandleConfig) AdjustAndValidate(protocol Protocol, enableTi // compression can be enabled independently if !compression.Supported(c.LargeMessageHandleCompression) { - return cerror.ErrInvalidReplicaConfig.GenWithStack( + return errors.ErrInvalidReplicaConfig.GenWithStack( "large message handle compression is not supported, got %s", c.LargeMessageHandleCompression) } if c.LargeMessageHandleOption == LargeMessageHandleOptionNone { return nil } + if c.LargeMessageHandleOption != LargeMessageHandleOptionClaimCheck && + c.LargeMessageHandleOption != LargeMessageHandleOptionHandleKeyOnly { + return errors.ErrInvalidReplicaConfig.GenWithStack( + "unknown large-message-handle-option %s", c.LargeMessageHandleOption) + } switch protocol { case ProtocolOpen, ProtocolSimple: case ProtocolCanalJSON: if !enableTiDBExtension { - return cerror.ErrInvalidReplicaConfig.GenWithStack( + return errors.ErrInvalidReplicaConfig.GenWithStack( "large message handle is set to %s, protocol is %s, but enable-tidb-extension is false", c.LargeMessageHandleOption, protocol.String()) } default: - return cerror.ErrInvalidReplicaConfig.GenWithStack( + return errors.ErrInvalidReplicaConfig.GenWithStack( "large message handle is set to %s, protocol is %s, it's not supported", c.LargeMessageHandleOption, protocol.String()) } if c.LargeMessageHandleOption == LargeMessageHandleOptionClaimCheck { if c.ClaimCheckStorageURI == "" { - return cerror.ErrInvalidReplicaConfig.GenWithStack( + return errors.ErrInvalidReplicaConfig.GenWithStack( "large message handle is set to claim-check, but the claim-check-storage-uri is empty") } if c.ClaimCheckRawValue && protocol == ProtocolOpen { - return cerror.ErrInvalidReplicaConfig.GenWithStack( + return errors.ErrInvalidReplicaConfig.GenWithStack( "large message handle is set to claim-check, raw value is not supported for the open protocol") } } @@ -106,7 +111,8 @@ func (c *LargeMessageHandleConfig) EnableClaimCheck() bool { return c.LargeMessageHandleOption == LargeMessageHandleOptionClaimCheck } -// Disabled returns true if disable large message handle. +// Disabled returns true only when large message handling is explicitly disabled. +// It returns false for nil and unknown configurations. func (c *LargeMessageHandleConfig) Disabled() bool { if c == nil { return false diff --git a/pkg/config/large_message_test.go b/pkg/config/large_message_test.go new file mode 100644 index 0000000000..f0721f4b53 --- /dev/null +++ b/pkg/config/large_message_test.go @@ -0,0 +1,215 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "testing" + + "github.com/pingcap/ticdc/pkg/compression" + cerror "github.com/pingcap/ticdc/pkg/errors" + "github.com/stretchr/testify/require" +) + +func TestLargeMessageHandle4Compression(t *testing.T) { + t.Parallel() + + largeMessageHandle := NewDefaultLargeMessageHandleConfig() + + // unsupported compression, return error + largeMessageHandle.LargeMessageHandleCompression = "zstd" + + err := largeMessageHandle.AdjustAndValidate(ProtocolCanalJSON, false) + require.ErrorIs(t, err, cerror.ErrInvalidReplicaConfig) + + largeMessageHandle.LargeMessageHandleCompression = compression.LZ4 + err = largeMessageHandle.AdjustAndValidate(ProtocolCanalJSON, false) + require.NoError(t, err) + + largeMessageHandle.LargeMessageHandleCompression = compression.Snappy + err = largeMessageHandle.AdjustAndValidate(ProtocolCanalJSON, false) + require.NoError(t, err) + + largeMessageHandle.LargeMessageHandleCompression = compression.None + err = largeMessageHandle.AdjustAndValidate(ProtocolCanalJSON, false) + require.NoError(t, err) +} + +func TestLargeMessageHandle4NotSupportedProtocol(t *testing.T) { + t.Parallel() + + largeMessageHandle := NewDefaultLargeMessageHandleConfig() + + err := largeMessageHandle.AdjustAndValidate(ProtocolCanal, true) + require.NoError(t, err) + + largeMessageHandle.LargeMessageHandleOption = LargeMessageHandleOptionHandleKeyOnly + err = largeMessageHandle.AdjustAndValidate(ProtocolCanal, true) + require.ErrorIs(t, err, cerror.ErrInvalidReplicaConfig) +} + +func TestLargeMessageHandleRejectsUnknownOption(t *testing.T) { + t.Parallel() + + largeMessageHandle := NewDefaultLargeMessageHandleConfig() + largeMessageHandle.LargeMessageHandleOption = "unknown" + + require.False(t, largeMessageHandle.Disabled()) + err := largeMessageHandle.AdjustAndValidate(ProtocolOpen, false) + require.ErrorIs(t, err, cerror.ErrInvalidReplicaConfig) + require.ErrorContains(t, err, "unknown large-message-handle-option unknown") +} + +func TestHandleKeyOnly4CanalJSON(t *testing.T) { + t.Parallel() + + // large-message-handle not set, always no error + largeMessageHandle := NewDefaultLargeMessageHandleConfig() + + err := largeMessageHandle.AdjustAndValidate(ProtocolCanalJSON, false) + require.NoError(t, err) + require.True(t, largeMessageHandle.Disabled()) + + largeMessageHandle.LargeMessageHandleOption = LargeMessageHandleOptionHandleKeyOnly + + // `enable-tidb-extension` is false, return error + err = largeMessageHandle.AdjustAndValidate(ProtocolCanalJSON, false) + require.ErrorIs(t, err, cerror.ErrInvalidReplicaConfig) + + // `enable-tidb-extension` is true, no error + err = largeMessageHandle.AdjustAndValidate(ProtocolCanalJSON, true) + require.NoError(t, err) + require.Equal(t, LargeMessageHandleOptionHandleKeyOnly, largeMessageHandle.LargeMessageHandleOption) +} + +func TestClaimCheck4CanalJSON(t *testing.T) { + t.Parallel() + + // large-message-handle not set, always no error + largeMessageHandle := NewDefaultLargeMessageHandleConfig() + + err := largeMessageHandle.AdjustAndValidate(ProtocolCanalJSON, false) + require.NoError(t, err) + require.True(t, largeMessageHandle.Disabled()) + + largeMessageHandle.LargeMessageHandleOption = LargeMessageHandleOptionClaimCheck + largeMessageHandle.ClaimCheckStorageURI = "file:///tmp/claim-check" + + for _, rawValue := range []bool{false, true} { + largeMessageHandle.ClaimCheckRawValue = rawValue + // `enable-tidb-extension` is false, return error + err = largeMessageHandle.AdjustAndValidate(ProtocolCanalJSON, false) + require.ErrorIs(t, err, cerror.ErrInvalidReplicaConfig) + + // `enable-tidb-extension` is true, no error + err = largeMessageHandle.AdjustAndValidate(ProtocolCanalJSON, true) + require.NoError(t, err) + require.Equal(t, LargeMessageHandleOptionClaimCheck, largeMessageHandle.LargeMessageHandleOption) + } +} + +func TestHandleKeyOnly4OpenProtocol(t *testing.T) { + t.Parallel() + + // large-message-handle not set, always no error + largeMessageHandle := NewDefaultLargeMessageHandleConfig() + + err := largeMessageHandle.AdjustAndValidate(ProtocolOpen, false) + require.NoError(t, err) + require.True(t, largeMessageHandle.Disabled()) + + largeMessageHandle.LargeMessageHandleOption = LargeMessageHandleOptionHandleKeyOnly + // `enable-tidb-extension` is false, return error + err = largeMessageHandle.AdjustAndValidate(ProtocolOpen, false) + require.NoError(t, err) + + // `enable-tidb-extension` is true, no error + err = largeMessageHandle.AdjustAndValidate(ProtocolOpen, true) + require.NoError(t, err) + require.Equal(t, LargeMessageHandleOptionHandleKeyOnly, largeMessageHandle.LargeMessageHandleOption) +} + +func TestClaimCheck4OpenProtocol(t *testing.T) { + t.Parallel() + + // large-message-handle not set, always no error + largeMessageHandle := NewDefaultLargeMessageHandleConfig() + + err := largeMessageHandle.AdjustAndValidate(ProtocolOpen, false) + require.NoError(t, err) + require.True(t, largeMessageHandle.Disabled()) + + largeMessageHandle.LargeMessageHandleOption = LargeMessageHandleOptionClaimCheck + largeMessageHandle.ClaimCheckStorageURI = "file:///tmp/claim-check" + + // `enable-tidb-extension` is false, return error + err = largeMessageHandle.AdjustAndValidate(ProtocolOpen, false) + require.NoError(t, err) + + // `enable-tidb-extension` is true, no error + err = largeMessageHandle.AdjustAndValidate(ProtocolOpen, true) + require.NoError(t, err) + require.Equal(t, LargeMessageHandleOptionClaimCheck, largeMessageHandle.LargeMessageHandleOption) + + largeMessageHandle.ClaimCheckRawValue = true + err = largeMessageHandle.AdjustAndValidate(ProtocolOpen, true) + require.ErrorIs(t, err, cerror.ErrInvalidReplicaConfig) +} + +func TestHandleKeyOnly4SimpleProtocol(t *testing.T) { + t.Parallel() + + // large-message-handle not set, always no error + largeMessageHandle := NewDefaultLargeMessageHandleConfig() + + err := largeMessageHandle.AdjustAndValidate(ProtocolSimple, false) + require.NoError(t, err) + require.True(t, largeMessageHandle.Disabled()) + + largeMessageHandle.LargeMessageHandleOption = LargeMessageHandleOptionHandleKeyOnly + // `enable-tidb-extension` is false, return error + err = largeMessageHandle.AdjustAndValidate(ProtocolSimple, false) + require.NoError(t, err) + + // `enable-tidb-extension` is true, no error + err = largeMessageHandle.AdjustAndValidate(ProtocolSimple, true) + require.NoError(t, err) + require.Equal(t, LargeMessageHandleOptionHandleKeyOnly, largeMessageHandle.LargeMessageHandleOption) +} + +func TestClaimCheck4SimpleProtocol(t *testing.T) { + t.Parallel() + + // large-message-handle not set, always no error + largeMessageHandle := NewDefaultLargeMessageHandleConfig() + + err := largeMessageHandle.AdjustAndValidate(ProtocolSimple, false) + require.NoError(t, err) + require.True(t, largeMessageHandle.Disabled()) + + largeMessageHandle.LargeMessageHandleOption = LargeMessageHandleOptionClaimCheck + largeMessageHandle.ClaimCheckStorageURI = "file:///tmp/claim-check" + + // `enable-tidb-extension` is false, return error + err = largeMessageHandle.AdjustAndValidate(ProtocolSimple, false) + require.NoError(t, err) + + // `enable-tidb-extension` is true, no error + err = largeMessageHandle.AdjustAndValidate(ProtocolSimple, true) + require.NoError(t, err) + require.Equal(t, LargeMessageHandleOptionClaimCheck, largeMessageHandle.LargeMessageHandleOption) + + largeMessageHandle.ClaimCheckRawValue = true + err = largeMessageHandle.AdjustAndValidate(ProtocolSimple, true) + require.NoError(t, err) +} diff --git a/pkg/sink/codec/open/encoder_test.go b/pkg/sink/codec/open/encoder_test.go index 678c4fb539..acd79528e2 100644 --- a/pkg/sink/codec/open/encoder_test.go +++ b/pkg/sink/codec/open/encoder_test.go @@ -189,7 +189,7 @@ func TestFloatTypes(t *testing.T) { helper.Tk().MustExec("use test") job := helper.DDL2Job(`create table test.t( id int primary key auto_increment, - a float, b float(10, 3), c float(10), + a float, b float(10, 3), c float(10), d double, e double(20, 3))`) dmlEvent := helper.DML2Event("test", "t", `insert into test.t(a,b,c,d,e) values (1.23, 4.56, 7.89, 10.11, 12.13)`) @@ -337,7 +337,7 @@ func TestBlobTypes(t *testing.T) { helper.Tk().MustExec("use test") job := helper.DDL2Job(`create table test.t( - id int primary key auto_increment, + id int primary key auto_increment, a tinyblob, b blob, c mediumblob, d longblob)`) dmlEvent := helper.DML2Event("test", "t", `insert into test.t(a,b,c,d) values (0x010201,0x010202,0x010203,0x010204)`) @@ -533,17 +533,17 @@ func TestOtherTypes(t *testing.T) { helper.Tk().MustExec("use test") job := helper.DDL2Job(`create table test.t( - id int primary key auto_increment, + id int primary key auto_increment, a bool, b bool, c year, - d bit(10), e json, - f decimal(10,2), + d bit(10), e json, + f decimal(10,2), g enum('a','b','c'), h set('a','b','c'))`) tableInfo := helper.GetTableInfo(job) dmlEvent := helper.DML2Event("test", "t", `insert into test.t(a, b, c, d, e, f, g, h) values ( - true, false, 2000, - 0b0101010101, '{"key1": "value1"}', - 153.123, + true, false, 2000, + 0b0101010101, '{"key1": "value1"}', + 153.123, 'a', 'a,b')`) require.NotNil(t, dmlEvent) From 64ad5dc1a3de53793d3299398c798c7880d7225b Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Mon, 17 Aug 2026 14:19:16 +0800 Subject: [PATCH 4/6] revert some more changes from 5818 --- downstreamadapter/sink/kafka/sink.go | 30 ++---- .../sink/topicmanager/kafka_topic_manager.go | 47 ++++------ pkg/sink/kafka/admin.go | 21 +---- pkg/sink/kafka/claimcheck/claim_check.go | 2 +- pkg/sink/kafka/options.go | 50 +++++----- pkg/sink/kafka/options_test.go | 37 +++++++- pkg/sink/kafka/sarama_async_producer.go | 15 ++- pkg/sink/kafka/sarama_config.go | 74 +++++++-------- pkg/sink/kafka/sarama_config_test.go | 57 ++++++++++++ pkg/sink/kafka/sarama_factory.go | 45 ++++++--- pkg/sink/kafka/sarama_sync_producer.go | 12 +-- .../kafka_compression/run.sh | 5 - tests/integration_tests/log_redaction/run.sh | 91 ------------------- tests/integration_tests/run_light_it_in_ci.sh | 2 +- 14 files changed, 225 insertions(+), 263 deletions(-) diff --git a/downstreamadapter/sink/kafka/sink.go b/downstreamadapter/sink/kafka/sink.go index 96430a5a1d..6433f40d7e 100644 --- a/downstreamadapter/sink/kafka/sink.go +++ b/downstreamadapter/sink/kafka/sink.go @@ -235,7 +235,7 @@ func (s *sink) WriteBlockEvent(event commonEvent.BlockEvent) error { case *commonEvent.DDLEvent: err = s.sendDDLEvent(v) default: - log.Error("kafka sink doesn't support this type of block event", + log.Error("unsupported kafka sink block event type", zap.String("namespace", s.changefeedID.Keyspace()), zap.String("changefeed", s.changefeedID.Name()), zap.String("eventType", commonEvent.TypeToString(event.GetType()))) @@ -370,10 +370,6 @@ func (s *sink) batchEncodeRun(ctx context.Context) error { start := time.Now() msgs, err := s.batch(ctx, msgsBuf) if err != nil { - log.Error("kafka sink batch dml events failed", - zap.String("keyspace", s.changefeedID.Keyspace()), - zap.String("changefeed", s.changefeedID.Name()), - zap.Error(err)) return err } if len(msgs) == 0 { @@ -443,16 +439,11 @@ func (s *sink) sendMessages(ctx context.Context) error { start := time.Now() if err = s.statistics.RecordBatchExecution(func() (int, int64, error) { message.SetPartitionKey(future.Key.PartitionKey) - log.Debug("send message to kafka", zap.String("messageKey", util.RedactBytes(message.Key)), zap.String("messageValue", util.RedactBytes(message.Value))) if err = s.dmlProducer.AsyncSend( ctx, future.Key.Topic, future.Key.Partition, message); err != nil { - log.Error("kafka sink send message failed", - zap.String("keyspace", s.changefeedID.Keyspace()), - zap.String("changefeed", s.changefeedID.Name()), - zap.Error(err)) return 0, 0, err } return message.GetRowsCount(), int64(message.Length()), nil @@ -472,9 +463,10 @@ func (s *sink) sendDDLEvent(event *commonEvent.DDLEvent) error { return err } if message == nil { - log.Info("Skip ddl event", zap.Uint64("startTs", event.GetStartTs()), zap.Uint64("commitTs", e.GetCommitTs()), - zap.String("query", e.Query), - zap.Stringer("changefeed", s.changefeedID)) + log.Info("kafka ddl event skipped", + zap.String("keyspace", s.changefeedID.Keyspace()), zap.String("changefeed", s.changefeedID.Name()), + zap.Uint64("startTs", e.GetStartTs()), zap.Uint64("commitTs", e.GetCommitTs()), + zap.String("query", e.Query)) continue } codecCommon.SetDDLMessageLogInfo(message, e) @@ -500,11 +492,11 @@ func (s *sink) sendDDLEvent(event *commonEvent.DDLEvent) error { if err != nil { return err } + log.Info("kafka ddl event sent", + zap.String("keyspace", s.changefeedID.Keyspace()), zap.String("changefeed", s.changefeedID.Name()), + zap.Uint64("startTs", e.GetStartTs()), zap.Uint64("commitTs", e.GetCommitTs()), + zap.String("query", e.GetDDLQuery())) } - log.Info("kafka sink send DDL event", - zap.String("keyspace", s.changefeedID.Keyspace()), zap.String("changefeed", s.changefeedID.Name()), - zap.Any("startTs", event.GetStartTs()), zap.Any("commitTs", event.GetCommitTs()), zap.Any("event", event.GetDDLQuery()), - zap.String("schema", event.GetSchemaName()), zap.String("table", event.GetTableName())) return nil } @@ -589,10 +581,6 @@ func (s *sink) SetTableSchemaStore(tableSchemaStore *commonEvent.TableSchemaStor func (s *sink) getAllTableNames(ts uint64) []*commonEvent.SchemaTableName { if s.tableSchemaStore == nil { - log.Warn("kafka sink table schema store is not set", - zap.String("keyspace", s.changefeedID.Keyspace()), - zap.String("changefeed", s.changefeedID.Name()), - zap.Uint64("ts", ts)) return nil } return s.tableSchemaStore.GetAllTableNames(ts) diff --git a/downstreamadapter/sink/topicmanager/kafka_topic_manager.go b/downstreamadapter/sink/topicmanager/kafka_topic_manager.go index eddd0cfe6c..e436e8801a 100644 --- a/downstreamadapter/sink/topicmanager/kafka_topic_manager.go +++ b/downstreamadapter/sink/topicmanager/kafka_topic_manager.go @@ -121,10 +121,6 @@ func (m *kafkaTopicManager) backgroundRefreshMeta(ctx context.Context) { for { select { case <-ctx.Done(): - log.Info("Background refresh Kafka metadata goroutine exit.", - zap.String("keyspace", m.changefeedID.Keyspace()), - zap.String("changefeed", m.changefeedID.Name()), - ) return case <-ticker.C: // We ignore the error here, because the error may be caused by the @@ -144,23 +140,16 @@ func (m *kafkaTopicManager) tryUpdatePartitionsAndLogging(topic string, partitio if oldPartitions.(int32) != partitions { m.topics.Store(topic, partitions) log.Info( - "update topic partition number", + "kafka topic partition count changed", zap.String("keyspace", m.changefeedID.Keyspace()), zap.String("changefeed", m.changefeedID.Name()), zap.String("topic", topic), - zap.Int32("oldPartitionNumber", oldPartitions.(int32)), - zap.Int32("newPartitionNumber", partitions), + zap.Int32("oldPartitionNum", oldPartitions.(int32)), + zap.Int32("newPartitionNum", partitions), ) } } else { m.topics.Store(topic, partitions) - log.Info( - "store topic partition number", - zap.String("keyspace", m.changefeedID.Keyspace()), - zap.String("changefeed", m.changefeedID.Name()), - zap.String("topic", topic), - zap.Int32("partitionNumber", partitions), - ) } } @@ -179,7 +168,7 @@ func (m *kafkaTopicManager) fetchAllTopicsPartitionsNum() (map[string]int32, err numPartitions, err := m.admin.GetTopicsPartitionsNum(topics) if err != nil { log.Warn( - "Kafka admin client describe topics failed", + "kafka topic metadata refresh failed", zap.String("keyspace", m.changefeedID.Keyspace()), zap.String("changefeed", m.changefeedID.Name()), zap.Duration("duration", time.Since(start)), @@ -259,11 +248,11 @@ func (m *kafkaTopicManager) createTopic( }) if err != nil { log.Error( - "Kafka admin client create the topic failed", + "kafka topic creation failed", zap.String("keyspace", m.changefeedID.Keyspace()), zap.String("changefeed", m.changefeedID.Name()), zap.String("topic", topicName), - zap.Int32("partitionNumber", m.cfg.PartitionNum), + zap.Int32("partitionNum", m.cfg.PartitionNum), zap.Int16("replicationFactor", m.cfg.ReplicationFactor), zap.Error(err), zap.Duration("duration", time.Since(start)), @@ -271,15 +260,6 @@ func (m *kafkaTopicManager) createTopic( return 0, err } - log.Info( - "Kafka admin client create the topic success", - zap.String("keyspace", m.changefeedID.Keyspace()), - zap.String("changefeed", m.changefeedID.Name()), - zap.String("topic", topicName), - zap.Int32("partitionNumber", m.cfg.PartitionNum), - zap.Int16("replicationFactor", m.cfg.ReplicationFactor), - zap.Duration("duration", time.Since(start)), - ) m.tryUpdatePartitionsAndLogging(topicName, m.cfg.PartitionNum) return m.cfg.PartitionNum, nil @@ -315,6 +295,7 @@ func (m *kafkaTopicManager) CreateTopicAndWaitUntilVisible( return numPartition, nil } + start := time.Now() partitionNum, err := m.createTopic(ctx, topicName) if err != nil { if kafka.IsAdminAuthorizationFailed(err) { @@ -328,6 +309,16 @@ func (m *kafkaTopicManager) CreateTopicAndWaitUntilVisible( return 0, err } + log.Info( + "kafka topic created", + zap.String("keyspace", m.changefeedID.Keyspace()), + zap.String("changefeed", m.changefeedID.Name()), + zap.String("topic", topicName), + zap.Int32("partitionNum", partitionNum), + zap.Int16("replicationFactor", m.cfg.ReplicationFactor), + zap.Duration("duration", time.Since(start)), + ) + return partitionNum, nil } @@ -347,11 +338,11 @@ func (m *kafkaTopicManager) tryStoreTopicMeta( } func (m *kafkaTopicManager) useConfiguredPartitionNum(topicName string, cause error) int32 { - log.Warn("skip Kafka topic creation because topic authorization failed", + log.Warn("kafka topic creation skipped due to authorization failure", zap.String("keyspace", m.changefeedID.Keyspace()), zap.String("changefeed", m.changefeedID.Name()), zap.String("topic", topicName), - zap.Int32("partitionNumber", m.cfg.PartitionNum), + zap.Int32("partitionNum", m.cfg.PartitionNum), zap.Error(cause)) m.tryUpdatePartitionsAndLogging(topicName, m.cfg.PartitionNum) return m.cfg.PartitionNum diff --git a/pkg/sink/kafka/admin.go b/pkg/sink/kafka/admin.go index 6e7f2930b3..8ad44cf5ea 100644 --- a/pkg/sink/kafka/admin.go +++ b/pkg/sink/kafka/admin.go @@ -81,11 +81,6 @@ func (a *saramaAdminClient) GetBrokerConfig(configName string) (string, bool, er return entry.Value, true, nil } } - - log.Warn("Kafka config item not found", - zap.String("keyspace", a.changefeed.Keyspace()), - zap.String("changefeed", a.changefeed.Name()), - zap.String("configName", configName)) return "", false, nil } @@ -104,19 +99,9 @@ func (a *saramaAdminClient) GetTopicConfig(topicName string, configName string) // 2. Kop returns all configs. for _, entry := range configEntries { if entry.Name == configName { - log.Info("Kafka config item found", - zap.String("keyspace", a.changefeed.Keyspace()), - zap.String("changefeed", a.changefeed.Name()), - zap.String("configName", configName), - zap.String("configValue", entry.Value)) return entry.Value, true, nil } } - - log.Warn("Kafka config item not found", - zap.String("keyspace", a.changefeed.Keyspace()), - zap.String("changefeed", a.changefeed.Name()), - zap.String("configName", configName)) return "", false, nil } @@ -136,7 +121,7 @@ func (a *saramaAdminClient) GetTopicsMeta(topics []string, ignoreTopicError bool if !ignoreTopicError { return nil, errors.WrapError(errors.ErrKafkaAdminAPI, meta.Err, "describe-topic", meta.Name) } - log.Warn("fetch topic meta failed", + log.Warn("kafka topic metadata refresh failed", zap.String("keyspace", a.changefeed.Keyspace()), zap.String("changefeed", a.changefeed.Name()), zap.String("topic", meta.Name), @@ -190,7 +175,7 @@ func (a *saramaAdminClient) Close() { // only when admin is unexpectedly nil. if a.admin != nil { if err := a.admin.Close(); err != nil { - log.Warn("close admin client meet error", + log.Warn("kafka admin client close failed", zap.String("keyspace", a.changefeed.Keyspace()), zap.String("changefeed", a.changefeed.Name()), zap.Error(err)) @@ -199,7 +184,7 @@ func (a *saramaAdminClient) Close() { } if a.client != nil { if err := a.client.Close(); err != nil { - log.Warn("close kafka client meet error", + log.Warn("kafka client close failed", zap.String("keyspace", a.changefeed.Keyspace()), zap.String("changefeed", a.changefeed.Name()), zap.Error(err)) diff --git a/pkg/sink/kafka/claimcheck/claim_check.go b/pkg/sink/kafka/claimcheck/claim_check.go index 4f8ecc2a42..7c49bd1bbe 100644 --- a/pkg/sink/kafka/claimcheck/claim_check.go +++ b/pkg/sink/kafka/claimcheck/claim_check.go @@ -52,7 +52,7 @@ func New(ctx context.Context, config *config.LargeMessageHandleConfig, changefee start := time.Now() externalStorage, err := util.GetExternalStorageWithDefaultTimeout(ctx, config.ClaimCheckStorageURI) if err != nil { - log.Error("create external storage failed", + log.Error("external storage creation failed", zap.String("keyspace", changefeedID.Keyspace()), zap.String("changefeed", changefeedID.Name()), zap.String("storageURI", util.MaskSensitiveDataInURI(config.ClaimCheckStorageURI)), diff --git a/pkg/sink/kafka/options.go b/pkg/sink/kafka/options.go index f783816176..5d42c58dd5 100644 --- a/pkg/sink/kafka/options.go +++ b/pkg/sink/kafka/options.go @@ -14,7 +14,6 @@ package kafka import ( - "context" "encoding/base64" "fmt" "net/http" @@ -39,6 +38,8 @@ const ( defaultPartitionNum = 3 // defaultMaxRetry is the default retry budget for Kafka producers. defaultMaxRetry = 5 + // defaultTimeout is the default timeout for Kafka connections. + defaultTimeout = 10 * time.Second // the `max-message-bytes` is set equal to topic's `max.message.bytes`, and is used to check // whether the message is larger than the max size limit. It's found some message pass the message @@ -192,27 +193,25 @@ func NewOptions() *options { InsecureSkipVerify: false, SASL: &security.SASL{}, AutoCreate: true, - DialTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, - ReadTimeout: 10 * time.Second, + DialTimeout: defaultTimeout, + WriteTimeout: defaultTimeout, + ReadTimeout: defaultTimeout, } } // setPartitionNum set the partition-num by the topic's partition count. -func (o *options) setPartitionNum(realPartitionCount int32) error { +func (o *options) setPartitionNum(changefeedID common.ChangeFeedID, realPartitionCount int32) error { // user does not specify the `partition-num` in the sink-uri if o.PartitionNum == 0 { o.PartitionNum = realPartitionCount - log.Info("partitionNum is not set, set by topic's partition-num", - zap.Int32("partitionNum", realPartitionCount)) return nil } if o.PartitionNum < realPartitionCount { - log.Warn("number of partition specified in sink-uri is less than that of the actual topic. "+ - "Some partitions will not have messages dispatched to", - zap.Int32("sinkUriPartitions", o.PartitionNum), - zap.Int32("topicPartitions", realPartitionCount)) + log.Warn("configured kafka partition count is lower than topic partition count", + zap.String("namespace", changefeedID.Keyspace()), zap.String("changefeed", changefeedID.Name()), + zap.Int32("configuredPartitionNum", o.PartitionNum), + zap.Int32("topicPartitionNum", realPartitionCount)) return nil } @@ -295,6 +294,9 @@ func (o *options) Apply(changefeedID common.ChangeFeedID, if err != nil { return errors.WrapError(errors.ErrKafkaInvalidConfig, err) } + if a <= 0 { + return errors.ErrKafkaInvalidConfig.GenWithStack("dial-timeout must be greater than zero") + } o.DialTimeout = a } @@ -303,6 +305,9 @@ func (o *options) Apply(changefeedID common.ChangeFeedID, if err != nil { return errors.WrapError(errors.ErrKafkaInvalidConfig, err) } + if a <= 0 { + return errors.ErrKafkaInvalidConfig.GenWithStack("write-timeout must be greater than zero") + } o.WriteTimeout = a } @@ -311,6 +316,9 @@ func (o *options) Apply(changefeedID common.ChangeFeedID, if err != nil { return errors.WrapError(errors.ErrKafkaInvalidConfig, err) } + if a <= 0 { + return errors.ErrKafkaInvalidConfig.GenWithStack("read-timeout must be greater than zero") + } o.ReadTimeout = a } @@ -563,14 +571,14 @@ func (c *AutoCreateTopicConfig) ValidateReplicationFactor(admin ClusterAdminClie raw, found, err := admin.GetBrokerConfig(MinInsyncReplicasConfigName) if err != nil { - log.Warn("cannot get Kafka broker configuration, assume replication factor is valid", + log.Warn("kafka broker configuration lookup failed, skipping replication factor validation", zap.String("configName", MinInsyncReplicasConfigName), zap.Int16("replicationFactor", c.ReplicationFactor), zap.Error(err)) return nil } if !found { - log.Warn("Kafka broker configuration not found, assume replication factor is valid", + log.Warn("kafka broker configuration not found, skipping replication factor validation", zap.String("configName", MinInsyncReplicasConfigName), zap.Int16("replicationFactor", c.ReplicationFactor)) return nil @@ -617,7 +625,7 @@ func NewKafkaClientID(captureAddr string, // adjustOptions adjust the `options` and `sarama.Config` by condition. func adjustOptions( - ctx context.Context, + changefeedID common.ChangeFeedID, admin ClusterAdminClient, options *options, topic string, @@ -633,7 +641,7 @@ func adjustOptions( if exists { // make sure that producer's `MaxMessageBytes` smaller than topic's `max.message.bytes` topicMaxMessageBytesStr, found, err := getTopicConfig( - ctx, admin, info.Name, + admin, info.Name, TopicMaxMessageBytesConfigName, BrokerMessageMaxBytesConfigName, ) @@ -665,12 +673,7 @@ func adjustOptions( // no need to create the topic, // but we would have to log user if they found enter wrong topic name later - if options.AutoCreate { - log.Warn("topic already exist, TiCDC will not create the topic", - zap.String("topic", topic), zap.Any("detail", info)) - } - - if err = options.setPartitionNum(info.NumPartitions); err != nil { + if err = options.setPartitionNum(changefeedID, info.NumPartitions); err != nil { return err } @@ -711,8 +714,6 @@ func adjustOptions( // topic not exists yet, and user does not specify the `partition-num` in the sink uri. if options.PartitionNum == 0 { options.PartitionNum = defaultPartitionNum - log.Warn("partition-num is not set, use the default partition count", - zap.String("topic", topic), zap.Int32("partitions", options.PartitionNum)) } return nil } @@ -722,7 +723,6 @@ func adjustOptions( // we will try to get it from the broker's configuration. // NOTICE: The configuration names of topic and broker may be different for the same configuration. func getTopicConfig( - _ context.Context, admin ClusterAdminClient, topicName string, topicConfigName string, @@ -733,7 +733,5 @@ func getTopicConfig( return c, true, nil } - log.Info("kafka sink cannot get the configuration from topic, try to get it from broker", - zap.String("topic", topicName), zap.String("config", topicConfigName), zap.Error(err)) return admin.GetBrokerConfig(brokerConfigName) } diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index d75b9d64ea..66d6e6cee1 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -300,17 +300,18 @@ func TestCompleteOptions(t *testing.T) { func TestSetPartitionNum(t *testing.T) { options := NewOptions() - err := options.setPartitionNum(2) + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test") + err := options.setPartitionNum(changefeedID, 2) require.NoError(t, err) require.Equal(t, int32(2), options.PartitionNum) options.PartitionNum = 1 - err = options.setPartitionNum(2) + err = options.setPartitionNum(changefeedID, 2) require.NoError(t, err) require.Equal(t, int32(1), options.PartitionNum) options.PartitionNum = 3 - err = options.setPartitionNum(2) + err = options.setPartitionNum(changefeedID, 2) require.True(t, errors.ErrKafkaInvalidConfig.Equal(err)) } @@ -378,6 +379,30 @@ func TestTimeout(t *testing.T) { require.Equal(t, 2*time.Minute, options.WriteTimeout) } +func TestApplyRejectsNonPositiveTimeout(t *testing.T) { + t.Parallel() + + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test") + for _, parameter := range []string{"dial-timeout", "read-timeout", "write-timeout"} { + for _, value := range []string{"0s", "-1s"} { + t.Run(parameter+"="+value, func(t *testing.T) { + t.Parallel() + + sinkURI, err := url.Parse( + "kafka://127.0.0.1:9092/kafka-test?" + parameter + "=" + value) + require.NoError(t, err) + + err = NewOptions().Apply( + changefeedID, sinkURI, config.GetDefaultReplicaConfig().Sink) + require.ErrorContains(t, err, parameter+" must be greater than zero") + errCode, ok := errors.RFCCode(err) + require.True(t, ok) + require.Equal(t, errors.ErrKafkaInvalidConfig.RFCCode(), errCode) + }) + } + } +} + func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *testing.T) { tests := []struct { name string @@ -404,6 +429,7 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t } topicName := "test-topic" + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test") for _, test := range tests { t.Run(test.name, func(t *testing.T) { adminFixture := newKafkaAdminFixture(t) @@ -425,7 +451,7 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t ) ctx := context.Background() - err = adjustOptions(ctx, adminClient, options, topicName) + err = adjustOptions(changefeedID, adminClient, options, topicName) require.NoError(t, err) saramaConfig, err := newSaramaConfig(ctx, options) @@ -653,7 +679,8 @@ func TestConfigurationCombinations(t *testing.T) { } expectedMaxMessageBytes := expectedAdjustedMaxMessageBytes(options.MaxMessageBytes, sourceMaxMessageBytes) - err = adjustOptions(context.Background(), adminClient, options, topic) + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test") + err = adjustOptions(changefeedID, adminClient, options, topic) require.Nil(t, err) require.Equal(t, expectedMaxMessageBytes, options.MaxMessageBytes) diff --git a/pkg/sink/kafka/sarama_async_producer.go b/pkg/sink/kafka/sarama_async_producer.go index f0c0f6b5d1..d3e1781c71 100644 --- a/pkg/sink/kafka/sarama_async_producer.go +++ b/pkg/sink/kafka/sarama_async_producer.go @@ -62,13 +62,13 @@ func (p *saramaAsyncProducer) Close() { // To prevent the scenario mentioned above, close the client first. start := time.Now() if err := p.client.Close(); err != nil { - log.Warn("Close kafka async producer client error", + log.Warn("kafka async producer client close failed", zap.String("keyspace", p.changefeedID.Keyspace()), zap.String("changefeed", p.changefeedID.Name()), zap.Duration("duration", time.Since(start)), zap.Error(err)) } else { - log.Info("Close kafka async producer client success", + log.Info("kafka async producer client closed", zap.String("keyspace", p.changefeedID.Keyspace()), zap.String("changefeed", p.changefeedID.Name()), zap.Duration("duration", time.Since(start))) @@ -76,13 +76,13 @@ func (p *saramaAsyncProducer) Close() { start = time.Now() if err := p.producer.Close(); err != nil { - log.Warn("Close kafka async producer error", + log.Warn("kafka async producer close failed", zap.String("keyspace", p.changefeedID.Keyspace()), zap.String("changefeed", p.changefeedID.Name()), zap.Duration("duration", time.Since(start)), zap.Error(err)) } else { - log.Info("Close kafka async producer success", + log.Info("kafka async producer closed", zap.String("keyspace", p.changefeedID.Keyspace()), zap.String("changefeed", p.changefeedID.Name()), zap.Duration("duration", time.Since(start))) @@ -97,9 +97,6 @@ func (p *saramaAsyncProducer) AsyncRunCallback( for { select { case <-ctx.Done(): - log.Info("async producer exit since context is done", - zap.String("keyspace", p.changefeedID.Keyspace()), - zap.String("changefeed", p.changefeedID.Name())) return context.Cause(ctx) case ack := <-p.producer.Successes(): if ack != nil { @@ -109,7 +106,7 @@ func (p *saramaAsyncProducer) AsyncRunCallback( meta.callback() } default: - log.Error("unknown message metadata type in async producer", + log.Error("kafka producer received unknown message metadata type", zap.Any("metadata", ack.Metadata)) } } @@ -128,7 +125,7 @@ func (p *saramaAsyncProducer) AsyncRunCallback( } func (p *saramaAsyncProducer) handleProducerError(err *sarama.ProducerError) error { - log.Error("send message to kafka failed", + log.Error("kafka message send failed", zap.String("keyspace", p.changefeedID.Keyspace()), zap.String("changefeed", p.changefeedID.Name()), zap.String("eventContext", BuildEventLogContext( diff --git a/pkg/sink/kafka/sarama_config.go b/pkg/sink/kafka/sarama_config.go index 4988c79c52..8ca31e5025 100644 --- a/pkg/sink/kafka/sarama_config.go +++ b/pkg/sink/kafka/sarama_config.go @@ -87,12 +87,9 @@ func newSaramaConfig(ctx context.Context, o *options) (*sarama.Config, error) { case "zstd": config.Producer.Compression = sarama.CompressionZSTD default: - log.Warn("Unsupported compression algorithm", zap.String("compression", o.Compression)) + log.Warn("unsupported kafka compression algorithm", zap.String("compression", o.Compression)) config.Producer.Compression = sarama.CompressionNone } - if config.Producer.Compression != sarama.CompressionNone { - log.Info("Kafka producer uses " + compression + " compression algorithm") - } if o.EnableTLS { // for SSL encryption with a trust CA certificate, we must populate the @@ -120,27 +117,27 @@ func newSaramaConfig(ctx context.Context, o *options) (*sarama.Config, error) { return nil, err } - kafkaVersion, err := getKafkaVersion(config, o) + err = completeSaramaKafkaVersion(config, o) if err != nil { - log.Warn("Can't get Kafka version by broker. ticdc will use default version", - zap.String("defaultVersion", kafkaVersion.String())) + return nil, err } - config.Version = kafkaVersion + return config, nil +} - if o.IsAssignedVersion { - version, err := sarama.ParseKafkaVersion(o.Version) - if err != nil { - return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) - } - config.Version = version - if !version.IsAtLeast(maxKafkaVersion) && version.String() != kafkaVersion.String() { - log.Warn("The Kafka version you assigned may not be correct. "+ - "Please assign a version equal to or less than the specified version", - zap.String("assignedVersion", version.String()), - zap.String("desiredVersion", kafkaVersion.String())) - } +func completeSaramaKafkaVersion(config *sarama.Config, o *options) error { + detectedVersion, err := detectKafkaVersion(config, o) + if err != nil { + log.Warn("kafka version detection failed, using fallback version", + zap.Strings("brokers", o.BrokerEndpoints), + zap.String("fallbackVersion", detectedVersion.String()), + zap.Error(err)) } - return config, nil + kafkaVersion, err := selectKafkaVersion(detectedVersion, o) + if err != nil { + return err + } + config.Version = kafkaVersion + return nil } func completeSaramaSASLConfig(ctx context.Context, config *sarama.Config, o *options) error { @@ -186,7 +183,7 @@ func completeSaramaSASLConfig(ctx context.Context, config *sarama.Config, o *opt return nil } -func getKafkaVersion(config *sarama.Config, o *options) (sarama.KafkaVersion, error) { +func detectKafkaVersion(config *sarama.Config, o *options) (sarama.KafkaVersion, error) { addrs := o.BrokerEndpoints if len(addrs) > 1 { // Shuffle the list of addresses to randomize the order in which @@ -208,25 +205,26 @@ func getKafkaVersion(config *sarama.Config, o *options) (sarama.KafkaVersion, er } } if err != nil { - log.Warn("kafka sink use the default kafka version since cannot find it from the brokers", - zap.String("defaultVersion", defaultKafkaVersion.String())) targetVersion = defaultKafkaVersion } + return targetVersion, err +} - if o.IsAssignedVersion { - assignedVersion, err := sarama.ParseKafkaVersion(o.Version) - if err != nil { - return assignedVersion, errors.WrapError(errors.ErrKafkaInvalidConfig, err) - } - if !assignedVersion.IsAtLeast(maxKafkaVersion) && assignedVersion.String() != targetVersion.String() { - log.Warn("The Kafka version you assigned may not be correct. "+ - "Please assign a version equal to or less than the specified version", - zap.String("assignedVersion", assignedVersion.String()), - zap.String("desiredVersion", targetVersion.String())) - } - targetVersion = assignedVersion +func selectKafkaVersion(detectedVersion sarama.KafkaVersion, o *options) (sarama.KafkaVersion, error) { + if !o.IsAssignedVersion { + return detectedVersion, nil + } + assignedVersion, err := sarama.ParseKafkaVersion(o.Version) + if err != nil { + return assignedVersion, errors.WrapError(errors.ErrKafkaInvalidConfig, err) + } + if !assignedVersion.IsAtLeast(maxKafkaVersion) && + assignedVersion.String() != detectedVersion.String() { + log.Warn("configured kafka version differs from detected version", + zap.String("assignedVersion", assignedVersion.String()), + zap.String("desiredVersion", detectedVersion.String())) } - return targetVersion, nil + return assignedVersion, nil } func getKafkaVersionFromBroker(config *sarama.Config, requestVersion int16, addr string) (sarama.KafkaVersion, error) { @@ -237,12 +235,10 @@ func getKafkaVersionFromBroker(config *sarama.Config, requestVersion int16, addr _ = broker.Close() }() if err != nil { - log.Warn("Kafka fail to open broker", zap.String("addr", addr), zap.Error(err)) return KafkaVersion, err } apiResponse, err := broker.ApiVersions(&sarama.ApiVersionsRequest{Version: requestVersion}) if err != nil { - log.Warn("Kafka fail to get ApiVersions", zap.String("addr", addr), zap.Error(err)) return KafkaVersion, err } // ApiKey method diff --git a/pkg/sink/kafka/sarama_config_test.go b/pkg/sink/kafka/sarama_config_test.go index ca3adbbdcb..f8970aa659 100644 --- a/pkg/sink/kafka/sarama_config_test.go +++ b/pkg/sink/kafka/sarama_config_test.go @@ -85,6 +85,63 @@ func TestNewSaramaConfig(t *testing.T) { require.Equal(t, sarama.SASLMechanism("SCRAM-SHA-256"), cfg.Net.SASL.Mechanism) } +func TestSelectKafkaVersion(t *testing.T) { + tests := []struct { + name string + detectedVersion sarama.KafkaVersion + assignedVersion string + expectedVersion sarama.KafkaVersion + expectedErr error + }{ + { + name: "use detected version", + detectedVersion: sarama.V2_4_0_0, + expectedVersion: sarama.V2_4_0_0, + }, + { + name: "use fallback version", + detectedVersion: defaultKafkaVersion, + expectedVersion: defaultKafkaVersion, + }, + { + name: "assigned version overrides detected version", + detectedVersion: sarama.V2_4_0_0, + assignedVersion: "2.6.0", + expectedVersion: sarama.V2_6_0_0, + }, + { + name: "assigned version overrides fallback version", + detectedVersion: defaultKafkaVersion, + assignedVersion: "2.6.0", + expectedVersion: sarama.V2_6_0_0, + }, + { + name: "reject invalid assigned version", + detectedVersion: sarama.V2_4_0_0, + assignedVersion: "invalid", + expectedErr: errors.ErrKafkaInvalidConfig, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + options := NewOptions() + if test.assignedVersion != "" { + options.IsAssignedVersion = true + options.Version = test.assignedVersion + } + + version, err := selectKafkaVersion(test.detectedVersion, options) + if test.expectedErr != nil { + require.ErrorIs(t, err, test.expectedErr) + return + } + require.NoError(t, err) + require.Equal(t, test.expectedVersion, version) + }) + } +} + func TestNewSaramaConfigInvalidOAuthTokenURL(t *testing.T) { options := NewOptions() options.SASL = &security.SASL{ diff --git a/pkg/sink/kafka/sarama_factory.go b/pkg/sink/kafka/sarama_factory.go index 57bf5ef27c..8f73ca70b5 100644 --- a/pkg/sink/kafka/sarama_factory.go +++ b/pkg/sink/kafka/sarama_factory.go @@ -40,10 +40,12 @@ func NewSaramaFactory( ) (Factory, error) { start := time.Now() config, err := newSaramaConfig(ctx, o) - duration := time.Since(start).Seconds() - if duration > 2 { - log.Warn("new sarama config cost too much time", - zap.Stringer("changefeedID", changefeedID), zap.Any("duration", duration)) + duration := time.Since(start) + if duration > 2*time.Second { + log.Warn("kafka configuration initialization is slow", + zap.String("keyspace", changefeedID.Keyspace()), + zap.String("changefeed", changefeedID.Name()), + zap.Duration("duration", duration)) } if err != nil { return nil, err @@ -57,9 +59,22 @@ func NewSaramaFactory( admin.Close() }() - if err = adjustOptions(ctx, admin, o, o.Topic); err != nil { + if err = adjustOptions(changefeedID, admin, o, o.Topic); err != nil { return nil, err } + log.Info("kafka sink configuration resolved", + zap.String("namespace", changefeedID.Keyspace()), + zap.String("changefeed", changefeedID.Name()), + zap.String("topic", o.Topic), + zap.Int32("partitionNum", o.PartitionNum), + zap.Int("maxMessageBytes", o.MaxMessageBytes), + zap.Int("maxBatchedBytes", o.MaxBatchedBytes), + zap.String("compression", config.Producer.Compression.String()), + zap.Int16("requiredAcks", int16(o.RequiredAcks)), + zap.Int("maxRetry", o.MaxRetry), + zap.Duration("dialTimeout", o.DialTimeout), + zap.Duration("readTimeout", o.ReadTimeout), + zap.Duration("writeTimeout", o.WriteTimeout)) return &saramaFactory{ changefeedID: changefeedID, @@ -71,10 +86,12 @@ func NewSaramaFactory( func newAdminClient(changefeedID common.ChangeFeedID, endpoints []string, config *sarama.Config) (ClusterAdminClient, error) { start := time.Now() client, err := sarama.NewClient(endpoints, config) - duration := time.Since(start).Seconds() - if duration > 2 { - log.Warn("new sarama client cost too much time", - zap.Any("duration", duration), zap.Stringer("changefeedID", changefeedID)) + duration := time.Since(start) + if duration > 2*time.Second { + log.Warn("kafka client initialization is slow", + zap.String("keyspace", changefeedID.Keyspace()), + zap.String("changefeed", changefeedID.Name()), + zap.Duration("duration", duration)) } if err != nil { return nil, errors.WrapError(errors.ErrNewKafkaSink, err) @@ -82,10 +99,12 @@ func newAdminClient(changefeedID common.ChangeFeedID, endpoints []string, config start = time.Now() admin, err := sarama.NewClusterAdminFromClient(client) - duration = time.Since(start).Seconds() - if duration > 2 { - log.Warn("new sarama cluster admin cost too much time", - zap.Any("duration", duration), zap.Stringer("changefeedID", changefeedID)) + duration = time.Since(start) + if duration > 2*time.Second { + log.Warn("kafka admin client initialization is slow", + zap.String("keyspace", changefeedID.Keyspace()), + zap.String("changefeed", changefeedID.Name()), + zap.Duration("duration", duration)) } if err != nil { // `sarama.NewClusterAdminFromClient` does not take ownership of the client, diff --git a/pkg/sink/kafka/sarama_sync_producer.go b/pkg/sink/kafka/sarama_sync_producer.go index 754d1db9e1..fcf1c9c258 100644 --- a/pkg/sink/kafka/sarama_sync_producer.go +++ b/pkg/sink/kafka/sarama_sync_producer.go @@ -58,7 +58,7 @@ func (p *saramaSyncProducer) SendMessage(topic string, partitionNum int32, messa if err == nil { return nil } - log.Error("send message to kafka failed", + log.Error("kafka message send failed", zap.String("keyspace", p.id.Keyspace()), zap.String("changefeed", p.id.Name()), zap.String("eventContext", BuildEventLogContext(p.id.Keyspace(), p.id.Name(), message.LogInfo)), @@ -84,7 +84,7 @@ func (p *saramaSyncProducer) SendMessages(topic string, partitionNum int32, mess if err == nil { return nil } - log.Error("send message to kafka failed", + log.Error("kafka message send failed", zap.String("keyspace", p.id.Keyspace()), zap.String("changefeed", p.id.Name()), zap.String("eventContext", BuildEventLogContext(p.id.Keyspace(), p.id.Name(), message.LogInfo)), @@ -94,7 +94,7 @@ func (p *saramaSyncProducer) SendMessages(topic string, partitionNum int32, mess func (p *saramaSyncProducer) Close() { if p.closed.Load() { - log.Warn("kafka DDL producer already closed", + log.Warn("kafka ddl producer already closed", zap.String("keyspace", p.id.Keyspace()), zap.String("changefeed", p.id.Name())) return @@ -106,7 +106,7 @@ func (p *saramaSyncProducer) Close() { // so producer.Close() alone won't release the underlying client resources. if p.client != nil { if err := p.client.Close(); err != nil { - log.Warn("Close Kafka DDL producer client with error", + log.Warn("kafka ddl producer client close failed", zap.String("keyspace", p.id.Keyspace()), zap.String("changefeed", p.id.Name()), zap.Duration("duration", time.Since(start)), @@ -115,7 +115,7 @@ func (p *saramaSyncProducer) Close() { } if p.producer != nil { if err := p.producer.Close(); err != nil { - log.Error("Close Kafka DDL producer with error", + log.Error("kafka ddl producer close failed", zap.String("keyspace", p.id.Keyspace()), zap.String("changefeed", p.id.Name()), zap.Duration("duration", time.Since(start)), @@ -123,7 +123,7 @@ func (p *saramaSyncProducer) Close() { return } } - log.Info("Kafka DDL producer closed", + log.Info("kafka ddl producer closed", zap.String("keyspace", p.id.Keyspace()), zap.String("changefeed", p.id.Name()), zap.Duration("duration", time.Since(start))) diff --git a/tests/integration_tests/kafka_compression/run.sh b/tests/integration_tests/kafka_compression/run.sh index e0f85df648..a57cd937b1 100755 --- a/tests/integration_tests/kafka_compression/run.sh +++ b/tests/integration_tests/kafka_compression/run.sh @@ -18,11 +18,6 @@ function test_compression() { run_kafka_consumer $WORK_DIR "kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=canal-json&version=${KAFKA_VERSION}&enable-tidb-extension=true" run_sql_file $CUR/data/$1_data.sql ${UP_TIDB_HOST} ${UP_TIDB_PORT} - compression_algorithm=$(grep "Kafka producer uses $1 compression algorithm" "$WORK_DIR/cdc.log") - if [[ "$compression_algorithm" -ne 1 ]]; then - echo "can't found producer compression algorithm" - exit 1 - fi check_table_exists test.$1_finish_mark ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 200 check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml cdc_cli_changefeed pause -c $1 diff --git a/tests/integration_tests/log_redaction/run.sh b/tests/integration_tests/log_redaction/run.sh index b99fe5da90..71b7e1e656 100755 --- a/tests/integration_tests/log_redaction/run.sh +++ b/tests/integration_tests/log_redaction/run.sh @@ -372,97 +372,6 @@ function run() { echo "[$(date)] ✓ MySQL sink: All redaction modes validated" fi - # ========================================================================== - # Test 4b: Kafka sink validation (tests Kafka-specific redaction) - # ========================================================================== - if [ "$SINK_TYPE" = "kafka" ]; then - echo "" - echo "=== Test 4b: Kafka sink redaction validation ===" - - # Kafka sink logs message key/value at DEBUG level - # Log message: "send message to kafka" with messageKey and messageValue fields - - # Test ON mode with Kafka sink (most important - full redaction) - echo " [4b-1] ON mode with Kafka sink:" - run_sql "DROP DATABASE IF EXISTS log_redaction_test;" - run_sql "CREATE DATABASE log_redaction_test;" - - KAFKA_TOPIC="log-redaction-test-$RANDOM" - KAFKA_SINK_URI="kafka://127.0.0.1:9092/$KAFKA_TOPIC?protocol=open-protocol" - - run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY --redact-info-log on --logsuffix "_on_kafka" - - cdc_cli_changefeed create --sink-uri="$KAFKA_SINK_URI" --changefeed-id="kafka-on-test" --config=$CUR/conf/changefeed.toml - - run_sql_file $CUR/data/test.sql ${UP_TIDB_HOST} ${UP_TIDB_PORT} - - echo " Waiting for Kafka sink to process events..." - wait_for_log_content "$WORK_DIR/cdc_on_kafka.log" "send message to kafka" "Kafka message logs" 30 - - echo " [Validation] ON mode with Kafka sink:" - echo "" - - # Capture Kafka logs once for all validations - captured_logs=$(grep "send message to kafka" "$WORK_DIR/cdc_on_kafka.log" 2>/dev/null || echo "") - log_raw_content "Kafka message logs (ON mode)" "$captured_logs" - - # STRICT POSITIVE VALIDATION: messageKey and messageValue must show redacted format - echo " [1/2] Verifying Kafka logs show redacted '?' placeholder:" - require_log_pattern "$WORK_DIR/cdc_on_kafka.log" \ - "send message to kafka.*messageKey.*\?.*messageValue.*\?" \ - "Kafka messageKey and messageValue redacted to '?'" \ - "ON mode should redact both messageKey and messageValue" - - # STRICT NEGATIVE VALIDATION: No sensitive data should leak in Kafka logs - echo " [2/2] Verifying NO sensitive data leaks in Kafka logs:" - sensitive_patterns=("Password1!" "SecretPass1!" "user1@example.com" "4532-1000-1000") - for pattern in "${sensitive_patterns[@]}"; do - require_no_log_pattern "$WORK_DIR/cdc_on_kafka.log" \ - "$pattern" \ - "No leak of sensitive value in Kafka logs: $pattern" - done - - captured_logs="" - cleanup_process $CDC_BINARY - - # Test MARKER mode with Kafka sink - echo "" - echo " [4b-2] MARKER mode with Kafka sink:" - run_sql "DROP DATABASE IF EXISTS log_redaction_test;" - run_sql "CREATE DATABASE log_redaction_test;" - - KAFKA_TOPIC="log-redaction-marker-$RANDOM" - KAFKA_SINK_URI="kafka://127.0.0.1:9092/$KAFKA_TOPIC?protocol=open-protocol" - - run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY --redact-info-log marker --logsuffix "_marker_kafka" - - cdc_cli_changefeed create --sink-uri="$KAFKA_SINK_URI" --changefeed-id="kafka-marker-test" --config=$CUR/conf/changefeed.toml - - run_sql_file $CUR/data/test.sql ${UP_TIDB_HOST} ${UP_TIDB_PORT} - - echo " Waiting for Kafka sink to process events..." - wait_for_log_content "$WORK_DIR/cdc_marker_kafka.log" "send message to kafka" "Kafka message logs" 30 - - echo " [Validation] MARKER mode with Kafka sink:" - echo "" - - # Capture Kafka logs once for all validations - captured_logs=$(grep "send message to kafka" "$WORK_DIR/cdc_marker_kafka.log" 2>/dev/null || echo "") - log_raw_content "Kafka message logs (MARKER mode)" "$captured_logs" - - # STRICT POSITIVE VALIDATION: messageKey and messageValue must have markers - echo " [1/1] Verifying Kafka logs have ‹› markers:" - require_log_pattern "$WORK_DIR/cdc_marker_kafka.log" \ - "send message to kafka.*‹" \ - "Kafka message values wrapped with ‹› markers" \ - "MARKER mode should wrap Kafka message data with ‹› markers" - - captured_logs="" - cleanup_process $CDC_BINARY - - echo "[$(date)] ✓ Kafka sink: Redaction modes validated" - fi - # ========================================================================== # Test 5: API mode switching # ========================================================================== diff --git a/tests/integration_tests/run_light_it_in_ci.sh b/tests/integration_tests/run_light_it_in_ci.sh index 647fd77f04..22a85bb36a 100755 --- a/tests/integration_tests/run_light_it_in_ci.sh +++ b/tests/integration_tests/run_light_it_in_ci.sh @@ -93,7 +93,7 @@ kafka_groups=( # G08 'capture_session_done_during_task fail_over_ddl_I table_route' # G09 - 'cdc_server_tips ddl_sequence log_redaction fail_over_ddl_J' + 'cdc_server_tips ddl_sequence fail_over_ddl_J' # G10 'changefeed_error batch_add_table fail_over_ddl_K split_table_check' # G11 From 10d76ffa3a7feae641fc8e096e5c3ff34f0c52e8 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Mon, 17 Aug 2026 14:25:17 +0800 Subject: [PATCH 5/6] revert some more changes from 5818 --- pkg/sink/codec/common/config.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pkg/sink/codec/common/config.go b/pkg/sink/codec/common/config.go index 0033f4acc3..5ee241e690 100644 --- a/pkg/sink/codec/common/config.go +++ b/pkg/sink/codec/common/config.go @@ -411,15 +411,11 @@ func (c *Config) Validate() error { } if c.MaxMessageBytes <= 0 { - return errors.ErrCodecInvalidConfig.Wrap( - errors.Errorf("invalid max-message-bytes %d", c.MaxMessageBytes), - ) + return errors.ErrCodecInvalidConfig.GenWithStack("invalid max-message-bytes %d", c.MaxMessageBytes) } if c.MaxBatchSize <= 0 { - return errors.ErrCodecInvalidConfig.Wrap( - errors.Errorf("invalid max-batch-size %d", c.MaxBatchSize), - ) + return errors.ErrCodecInvalidConfig.GenWithStack("invalid max-batch-size %d", c.MaxBatchSize) } if c.LargeMessageHandle != nil { From 62d1d0206f9b7f5763338e430f960b31d1f6b201 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Mon, 17 Aug 2026 14:39:04 +0800 Subject: [PATCH 6/6] revert more changes --- pkg/sink/codec/common/config.go | 2 +- pkg/sink/codec/common/config_test.go | 70 ++++++++++++++++++++++++++++ pkg/sink/kafka/sarama_factory.go | 1 - 3 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 pkg/sink/codec/common/config_test.go diff --git a/pkg/sink/codec/common/config.go b/pkg/sink/codec/common/config.go index 5ee241e690..c55c30fac0 100644 --- a/pkg/sink/codec/common/config.go +++ b/pkg/sink/codec/common/config.go @@ -193,7 +193,7 @@ func (c *Config) Apply(sinkURI *url.URL, sinkConfig *config.SinkConfig) error { var err error urlParameter := &urlConfig{} if err = binding.Query.Bind(req, urlParameter); err != nil { - return errors.WrapError(errors.ErrMySQLInvalidConfig, err) + return errors.WrapError(errors.ErrSinkInvalidConfig, err) } if urlParameter, err = mergeConfig(sinkConfig, urlParameter); err != nil { return err diff --git a/pkg/sink/codec/common/config_test.go b/pkg/sink/codec/common/config_test.go new file mode 100644 index 0000000000..4824bbfaec --- /dev/null +++ b/pkg/sink/codec/common/config_test.go @@ -0,0 +1,70 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "net/url" + "testing" + + "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/stretchr/testify/require" +) + +func TestApplyReturnsSinkInvalidConfigForQueryBindingError(t *testing.T) { + cfg := NewConfig(config.ProtocolOpen) + sinkURI, err := url.Parse("kafka://127.0.0.1:9092/topic?max-batch-size=invalid") + require.NoError(t, err) + + err = cfg.Apply(sinkURI, config.GetDefaultReplicaConfig().Sink) + errCode, ok := errors.RFCCode(err) + require.True(t, ok, err) + require.Equal(t, errors.ErrSinkInvalidConfig.RFCCode(), errCode) +} + +func TestValidateMessageLimits(t *testing.T) { + tests := []struct { + name string + adjust func(*Config) + expected string + }{ + { + name: "non-positive max message bytes", + adjust: func(cfg *Config) { + cfg.MaxMessageBytes = 0 + }, + expected: "invalid max-message-bytes 0", + }, + { + name: "non-positive max batch size", + adjust: func(cfg *Config) { + cfg.MaxBatchSize = 0 + }, + expected: "invalid max-batch-size 0", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cfg := NewConfig(config.ProtocolOpen) + test.adjust(cfg) + + err := cfg.Validate() + require.ErrorContains(t, err, test.expected) + errCode, ok := errors.RFCCode(err) + require.True(t, ok, err) + require.Equal(t, errors.ErrCodecInvalidConfig.RFCCode(), errCode) + }) + } +} diff --git a/pkg/sink/kafka/sarama_factory.go b/pkg/sink/kafka/sarama_factory.go index 8f73ca70b5..5d141f3d1f 100644 --- a/pkg/sink/kafka/sarama_factory.go +++ b/pkg/sink/kafka/sarama_factory.go @@ -68,7 +68,6 @@ func NewSaramaFactory( zap.String("topic", o.Topic), zap.Int32("partitionNum", o.PartitionNum), zap.Int("maxMessageBytes", o.MaxMessageBytes), - zap.Int("maxBatchedBytes", o.MaxBatchedBytes), zap.String("compression", config.Producer.Compression.String()), zap.Int16("requiredAcks", int16(o.RequiredAcks)), zap.Int("maxRetry", o.MaxRetry),