From 43ac52acf6a3f6595a5a9dd674a1aaad256cdfcc Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 31 Mar 2026 17:14:11 +0800 Subject: [PATCH 01/20] Add configuration code --- api/v2/model.go | 19 ++++ pkg/config/changefeed.go | 13 ++- pkg/config/sink.go | 67 +++++++++++-- pkg/config/sink_test.go | 119 ++++++++++++++++++++++++ pkg/errors/error.go | 4 + tests/integration_tests/api_v2/model.go | 2 + 6 files changed, 214 insertions(+), 10 deletions(-) diff --git a/api/v2/model.go b/api/v2/model.go index 3c222c7166..e7fb540e89 100644 --- a/api/v2/model.go +++ b/api/v2/model.go @@ -342,6 +342,8 @@ func (c *ReplicaConfig) toInternalReplicaConfigWithOriginConfig( IndexName: rule.IndexName, Columns: rule.Columns, TopicRule: rule.TopicRule, + TargetSchema: rule.TargetSchema, + TargetTable: rule.TargetTable, }) } var columnSelectors []*config.ColumnSelector @@ -700,6 +702,8 @@ func ToAPIReplicaConfig(c *config.ReplicaConfig) *ReplicaConfig { IndexName: rule.IndexName, Columns: rule.Columns, TopicRule: rule.TopicRule, + TargetSchema: rule.TargetSchema, + TargetTable: rule.TargetTable, }) } var columnSelectors []*ColumnSelector @@ -1190,6 +1194,21 @@ type DispatchRule struct { IndexName string `json:"index,omitempty"` Columns []string `json:"columns,omitempty"` TopicRule string `json:"topic,omitempty"` + + // TargetSchema sets the routed downstream schema name. + // Leave it empty to keep the source schema name. + // For example, if the source table is `sales`.`orders`, `target-schema = "sales_bak"` + // writes to `sales_bak`.`orders`. + // You can also use placeholders. For example, `target-schema = "{schema}_bak"` + // the target schema becomes `sales_bak`. + TargetSchema string `json:"target-schema,omitempty"` + // TargetTable sets the routed downstream table name. + // Leave it empty to keep the source table name. + // For example, if the source table is `sales`.`orders`, `target-table = "orders_bak"` + // writes to `sales`.`orders_bak`. + // You can also use placeholders. For example, `target-table = "{schema}_{table}"` + // becomes `sales_orders`. + TargetTable string `json:"target-table,omitempty"` } // ColumnSelector represents a column selector for a table. diff --git a/pkg/config/changefeed.go b/pkg/config/changefeed.go index 4334096bdc..1080aea6c0 100644 --- a/pkg/config/changefeed.go +++ b/pkg/config/changefeed.go @@ -497,10 +497,15 @@ func (info *ChangeFeedInfo) RmUnusedFields() { } func (info *ChangeFeedInfo) rmMQOnlyFields() { - log.Info("since the downstream is not a MQ, remove MQ only fields", - zap.String("keyspace", info.ChangefeedID.Keyspace()), - zap.String("changefeed", info.ChangefeedID.Name())) - info.Config.Sink.DispatchRules = nil + // Don't nil out DispatchRules entirely - it may contain routing rules (TargetSchema/TargetTable) + // Remove only MQ-specific fields from each rule. + for _, rule := range info.Config.Sink.DispatchRules { + rule.DispatcherRule = "" + rule.PartitionRule = "" + rule.IndexName = "" + rule.Columns = nil + rule.TopicRule = "" + } info.Config.Sink.SchemaRegistry = nil info.Config.Sink.EncoderConcurrency = nil info.Config.Sink.OnlyOutputUpdatedColumns = nil diff --git a/pkg/config/sink.go b/pkg/config/sink.go index a118692037..9d0116cc00 100644 --- a/pkg/config/sink.go +++ b/pkg/config/sink.go @@ -16,6 +16,7 @@ package config import ( "fmt" "net/url" + "regexp" "strconv" "strings" "time" @@ -140,7 +141,6 @@ type SinkConfig struct { // Protocol is NOT available when the downstream is DB. Protocol *string `toml:"protocol" json:"protocol,omitempty"` - // DispatchRules is only available when the downstream is MQ. DispatchRules []*DispatchRule `toml:"dispatchers" json:"dispatchers,omitempty"` ColumnSelectors []*ColumnSelector `toml:"column-selectors" json:"column-selectors,omitempty"` @@ -386,7 +386,9 @@ func (d DateSeparator) String() string { } } -// DispatchRule represents partition rule for a table. +// DispatchRules configures event routing. +// For MQ sinks, rules control topic / partition dispatching. +// TargetSchema and TargetTable configure table routing. type DispatchRule struct { Matcher []string `toml:"matcher" json:"matcher"` // Deprecated, please use PartitionRule. @@ -402,6 +404,22 @@ type DispatchRule struct { Columns []string `toml:"columns" json:"columns"` TopicRule string `toml:"topic" json:"topic"` + + // TargetSchema sets the routed downstream schema name. + // Leave it empty to keep the source schema name. + // For example, if the source table is `sales`.`orders`, `target-schema = "sales_bak"` + // writes to `sales_bak`.`orders`. + // You can also use placeholders. For example, `target-schema = "{schema}_bak"` + // becomes `sales_bak`. + TargetSchema string `toml:"target-schema" json:"target-schema"` + + // TargetTable sets the routed downstream table name. + // Leave it empty to keep the source table name. + // For example, if the source table is `sales`.`orders`, `target-table = "orders_bak"` + // writes to `sales`.`orders_bak`. + // You can also use placeholders. For example, `target-table = "{schema}_{table}"` + // becomes `sales_orders`. + TargetTable string `toml:"target-table" json:"target-table"` } // ColumnSelector represents a column selector for a table. @@ -743,10 +761,6 @@ func (s *SinkConfig) validateAndAdjust(sinkURI *url.URL) error { return err } - if IsMySQLCompatibleScheme(sinkURI.Scheme) { - return nil - } - if util.GetOrZero(s.EnableKafkaSinkV2) { log.Warn("enable-kafka-sink-v2 is deprecated, still use the default kafka sink") } @@ -813,6 +827,14 @@ func (s *SinkConfig) validateAndAdjust(sinkURI *url.URL) error { } } + if err := s.validateTableRoute(); err != nil { + return err + } + + if IsMySQLCompatibleScheme(sinkURI.Scheme) { + return nil + } + if util.GetOrZero(s.EncoderConcurrency) < 0 { return cerror.ErrSinkInvalidConfig.GenWithStack( "encoder-concurrency should greater than 0, but got %d", s.EncoderConcurrency) @@ -861,6 +883,21 @@ func (s *SinkConfig) validateAndAdjust(sinkURI *url.URL) error { return nil } +func (s *SinkConfig) validateTableRoute() error { + for _, rule := range s.DispatchRules { + if rule.TargetSchema == "" && rule.TargetTable == "" { + continue + } + if err := validateRoutingExpression("target-schema", rule.TargetSchema); err != nil { + return err + } + if err := validateRoutingExpression("target-table", rule.TargetTable); err != nil { + return err + } + } + return nil +} + // validateAndAdjustSinkURI validate and adjust `Protocol` and `TxnAtomicity` by sinkURI. func (s *SinkConfig) validateAndAdjustSinkURI(sinkURI *url.URL) error { if sinkURI == nil { @@ -1113,3 +1150,21 @@ type OpenProtocolConfig struct { type DebeziumConfig struct { OutputOldValue bool `toml:"output-old-value" json:"output-old-value"` } + +// validRoutingExpressionRegexp accepts routing expressions made of literal text +// and the {schema}/{table} placeholders, such as "archive", "{table}_bak", or +// "{schema}_{table}". +var validRoutingExpressionRegexp = regexp.MustCompile(`^(?:[^{}]|\{schema\}|\{table\})*$`) + +// validateRoutingExpression validates a routing expression for a single routing field. +// Valid expressions can contain literal text and {schema} or {table} placeholders. +func validateRoutingExpression(fieldName, expr string) error { + if expr == "" || validRoutingExpressionRegexp.MatchString(expr) { + return nil + } + return cerror.ErrInvalidTableRoutingRule.GenWithStack( + "%s %q must contain only literal text, {schema}, and {table}", + fieldName, + expr, + ) +} diff --git a/pkg/config/sink_test.go b/pkg/config/sink_test.go index 20c0d9075c..507bb250ef 100644 --- a/pkg/config/sink_test.go +++ b/pkg/config/sink_test.go @@ -17,6 +17,7 @@ import ( "net/url" "testing" + "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/util" "github.com/stretchr/testify/require" ) @@ -294,6 +295,124 @@ func TestCheckCompatibilityWithSinkURI(t *testing.T) { } } +func TestValidateTableRoute(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + cfg *SinkConfig + wantErr string + }{ + { + name: "valid routing rule", + cfg: &SinkConfig{ + DispatchRules: []*DispatchRule{ + { + Matcher: []string{"db1.*"}, + TargetSchema: "archive", + TargetTable: "{table}_bak", + }, + }, + }, + }, + { + name: "invalid target schema expression", + cfg: &SinkConfig{ + DispatchRules: []*DispatchRule{ + { + Matcher: []string{"db1.*"}, + TargetSchema: "{bad}", + TargetTable: "{table}_bak", + }, + }, + }, + wantErr: "target-schema", + }, + { + name: "mq dispatch rule ignored", + cfg: &SinkConfig{ + DispatchRules: []*DispatchRule{ + { + Matcher: []string{"db1.*"}, + PartitionRule: "columns", + Columns: []string{"id"}, + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := tc.cfg.validateTableRoute() + if tc.wantErr == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, tc.wantErr) + } + }) + } +} + +func TestValidateRoutingExpression(t *testing.T) { + t.Parallel() + + validExpressions := []string{ + "", + "archive", + "orders_bak", + "archive_v2", + "db-01.table_02", + "{schema}", + "{table}", + "{schema}_{table}", + "{table}_bak", + "bak_{table}_v2", + "{schema}_backup", + "archive_{schema}_{table}_v2", + "prefix_{schema}_middle_{table}_suffix", + "{schema}_{schema}_{table}", + } + + for _, expr := range validExpressions { + name := expr + if name == "" { + name = "empty" + } + t.Run(name, func(t *testing.T) { + require.NoError(t, validateRoutingExpression("target-table", expr)) + }) + } +} + +func TestValidateRoutingExpressionRejectsInvalidExpressions(t *testing.T) { + t.Parallel() + + invalidExpressions := []string{ + "{invalid}", + "{Schema}", + "{TABLE}", + "{schema", + "schema}", + "{table", + "{", + "}", + "{{schema}}", + "{schema}{bad}", + "prefix_{schema}_{bad}", + } + + for _, expr := range invalidExpressions { + t.Run(expr, func(t *testing.T) { + err := validateRoutingExpression("target-table", expr) + require.Error(t, err) + code, ok := errors.RFCCode(err) + require.True(t, ok) + require.Equal(t, errors.ErrInvalidTableRoutingRule.RFCCode(), code) + }) + } +} + func TestValidateAndAdjustCSVConfig(t *testing.T) { t.Parallel() tests := []struct { diff --git a/pkg/errors/error.go b/pkg/errors/error.go index 3d0ce1c0cb..b6af6903a2 100644 --- a/pkg/errors/error.go +++ b/pkg/errors/error.go @@ -261,6 +261,10 @@ var ( "sink config invalid", errors.RFCCodeText("CDC:ErrSinkInvalidConfig"), ) + ErrInvalidTableRoutingRule = errors.Normalize( + "invalid table routing rule", + errors.RFCCodeText("CDC:ErrInvalidTableRoutingRule"), + ) ErrMessageTooLarge = errors.Normalize( "message is too large. table:%s, length:%d, maxMessageBytes:%d", errors.RFCCodeText("CDC:ErrMessageTooLarge"), diff --git a/tests/integration_tests/api_v2/model.go b/tests/integration_tests/api_v2/model.go index 59d2b9f3ad..14e2c2e90c 100644 --- a/tests/integration_tests/api_v2/model.go +++ b/tests/integration_tests/api_v2/model.go @@ -256,6 +256,8 @@ type DispatchRule struct { TopicRule string `json:"topic"` IndexName string `json:"index,omitempty"` Columns []string `json:"columns,omitempty"` + TargetSchema string `json:"target-schema,omitempty"` + TargetTable string `json:"target-table,omitempty"` } // ColumnSelector represents a column selector for a table. From e552c927a81eb7c92c64fe8f2b6e2c3cf84bcc54 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 31 Mar 2026 17:50:51 +0800 Subject: [PATCH 02/20] fix by review suggestions --- pkg/config/changefeed.go | 3 +++ pkg/config/sink.go | 19 +++++++++---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/pkg/config/changefeed.go b/pkg/config/changefeed.go index 1080aea6c0..251c8cfa39 100644 --- a/pkg/config/changefeed.go +++ b/pkg/config/changefeed.go @@ -500,6 +500,9 @@ func (info *ChangeFeedInfo) rmMQOnlyFields() { // Don't nil out DispatchRules entirely - it may contain routing rules (TargetSchema/TargetTable) // Remove only MQ-specific fields from each rule. for _, rule := range info.Config.Sink.DispatchRules { + if rule == nil { + continue + } rule.DispatcherRule = "" rule.PartitionRule = "" rule.IndexName = "" diff --git a/pkg/config/sink.go b/pkg/config/sink.go index 9d0116cc00..51052c85ad 100644 --- a/pkg/config/sink.go +++ b/pkg/config/sink.go @@ -761,6 +761,14 @@ func (s *SinkConfig) validateAndAdjust(sinkURI *url.URL) error { return err } + if err := s.validateTableRoute(); err != nil { + return err + } + + if IsMySQLCompatibleScheme(sinkURI.Scheme) { + return nil + } + if util.GetOrZero(s.EnableKafkaSinkV2) { log.Warn("enable-kafka-sink-v2 is deprecated, still use the default kafka sink") } @@ -827,14 +835,6 @@ func (s *SinkConfig) validateAndAdjust(sinkURI *url.URL) error { } } - if err := s.validateTableRoute(); err != nil { - return err - } - - if IsMySQLCompatibleScheme(sinkURI.Scheme) { - return nil - } - if util.GetOrZero(s.EncoderConcurrency) < 0 { return cerror.ErrSinkInvalidConfig.GenWithStack( "encoder-concurrency should greater than 0, but got %d", s.EncoderConcurrency) @@ -1152,8 +1152,7 @@ type DebeziumConfig struct { } // validRoutingExpressionRegexp accepts routing expressions made of literal text -// and the {schema}/{table} placeholders, such as "archive", "{table}_bak", or -// "{schema}_{table}". +// and the {schema}/{table} placeholders, such as "archive", "{table}_bak", or "{schema}_{table}". var validRoutingExpressionRegexp = regexp.MustCompile(`^(?:[^{}]|\{schema\}|\{table\})*$`) // validateRoutingExpression validates a routing expression for a single routing field. From f76378df0dc631beb71b9d8941040f4fe86e1bf0 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 31 Mar 2026 18:14:53 +0800 Subject: [PATCH 03/20] pkg/common,event: add table route naming model --- pkg/common/event/ddl_event.go | 146 +++++++++++++++++++--- pkg/common/event/ddl_event_test.go | 191 +++++++++++++++++++++++++++++ pkg/common/event/dml_event.go | 32 +++-- pkg/common/event/redo.go | 88 +++++++------ pkg/common/event/redo_test.go | 94 ++++++++++++++ pkg/common/table_info.go | 93 ++++++++++++-- pkg/common/table_info_test.go | 76 ++++++++++++ pkg/common/table_name.go | 58 ++++++++- pkg/common/table_name_gen.go | 60 ++++++++- 9 files changed, 763 insertions(+), 75 deletions(-) create mode 100644 pkg/common/event/redo_test.go diff --git a/pkg/common/event/ddl_event.go b/pkg/common/event/ddl_event.go index ef8dcd6732..0cd046461a 100644 --- a/pkg/common/event/ddl_event.go +++ b/pkg/common/event/ddl_event.go @@ -22,6 +22,8 @@ import ( "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/tidb/pkg/meta/model" + "github.com/pingcap/tidb/pkg/parser" + "github.com/pingcap/tidb/pkg/parser/ast" "go.uber.org/zap" ) @@ -41,14 +43,22 @@ type DDLEvent struct { SchemaID int64 `json:"schema_id"` SchemaName string `json:"schema_name"` TableName string `json:"table_name"` + // TargetSchemaName and TargetTableName carry routed names for sink output paths. + // They are runtime-only fields and are not serialized. + TargetSchemaName string `json:"-"` + TargetTableName string `json:"-"` // the following two fields are just used for RenameTable, // they are the old schema/table name of the table - ExtraSchemaName string `json:"extra_schema_name"` - ExtraTableName string `json:"extra_table_name"` - Query string `json:"query"` - TableInfo *common.TableInfo `json:"-"` - StartTs uint64 `json:"start_ts"` - FinishedTs uint64 `json:"finished_ts"` + ExtraSchemaName string `json:"extra_schema_name"` + ExtraTableName string `json:"extra_table_name"` + // TargetExtraSchemaName and TargetExtraTableName carry routed old names for rename DDLs. + // They are runtime-only fields and are not serialized. + TargetExtraSchemaName string `json:"-"` + TargetExtraTableName string `json:"-"` + Query string `json:"query"` + TableInfo *common.TableInfo `json:"-"` + StartTs uint64 `json:"start_ts"` + FinishedTs uint64 `json:"finished_ts"` // The seq of the event. It is set by event service. Seq uint64 `json:"seq"` // The epoch of the event. It is set by event service. @@ -189,18 +199,62 @@ func (d *DDLEvent) GetSchemaName() string { return d.SchemaName } +func (d *DDLEvent) GetSourceSchemaName() string { + return d.SchemaName +} + func (d *DDLEvent) GetTableName() string { return d.TableName } +func (d *DDLEvent) GetSourceTableName() string { + return d.TableName +} + func (d *DDLEvent) GetExtraSchemaName() string { return d.ExtraSchemaName } +func (d *DDLEvent) GetSourceExtraSchemaName() string { + return d.ExtraSchemaName +} + func (d *DDLEvent) GetExtraTableName() string { return d.ExtraTableName } +func (d *DDLEvent) GetSourceExtraTableName() string { + return d.ExtraTableName +} + +func (d *DDLEvent) GetTargetSchemaName() string { + if d.TargetSchemaName != "" { + return d.TargetSchemaName + } + return d.SchemaName +} + +func (d *DDLEvent) GetTargetTableName() string { + if d.TargetTableName != "" { + return d.TargetTableName + } + return d.TableName +} + +func (d *DDLEvent) GetTargetExtraSchemaName() string { + if d.TargetExtraSchemaName != "" { + return d.TargetExtraSchemaName + } + return d.ExtraSchemaName +} + +func (d *DDLEvent) GetTargetExtraTableName() string { + if d.TargetExtraTableName != "" { + return d.TargetExtraTableName + } + return d.ExtraTableName +} + // GetTableID returns the logic table ID of the event. // it returns 0 when there is no tableinfo func (d *DDLEvent) GetTableID() int64 { @@ -230,18 +284,23 @@ func (d *DDLEvent) GetEvents() []*DDLEvent { } for i, info := range d.MultipleTableInfos { event := &DDLEvent{ - Version: d.Version, - Type: byte(t), - SchemaName: info.GetSchemaName(), - TableName: info.GetTableName(), - TableInfo: info, - Query: queries[i], - StartTs: d.StartTs, - FinishedTs: d.FinishedTs, + Version: d.Version, + Type: byte(t), + SchemaName: info.GetSchemaName(), + TableName: info.GetTableName(), + TargetSchemaName: info.GetTargetSchemaName(), + TargetTableName: info.GetTargetTableName(), + TableInfo: info, + Query: queries[i], + StartTs: d.StartTs, + FinishedTs: d.FinishedTs, } if model.ActionType(d.Type) == model.ActionRenameTables { event.ExtraSchemaName = d.TableNameChange.DropName[i].SchemaName event.ExtraTableName = d.TableNameChange.DropName[i].TableName + targetExtraSchemaName, targetExtraTableName := extractRenameTargetExtraFromQuery(queries[i]) + event.TargetExtraSchemaName = targetExtraSchemaName + event.TargetExtraTableName = targetExtraTableName } events = append(events, event) } @@ -251,6 +310,19 @@ func (d *DDLEvent) GetEvents() []*DDLEvent { return []*DDLEvent{d} } +func extractRenameTargetExtraFromQuery(query string) (string, string) { + stmt, err := parser.New().ParseOneStmt(query, "", "") + if err != nil { + log.Panic("parse split rename query failed", zap.String("query", query), zap.Error(err)) + } + renameStmt, ok := stmt.(*ast.RenameTableStmt) + if !ok || len(renameStmt.TableToTables) == 0 { + log.Panic("unexpected split rename query", zap.String("query", query), zap.Any("stmt", stmt)) + } + oldTable := renameStmt.TableToTables[0].OldTable + return oldTable.Schema.O, oldTable.Name.O +} + func (d *DDLEvent) GetSeq() uint64 { return d.Seq } @@ -299,6 +371,13 @@ func (e *DDLEvent) GetDDLQuery() string { return e.Query } +func (e *DDLEvent) GetDDLSchemaName() string { + if e == nil { + return "" + } + return e.GetTargetSchemaName() +} + func (e *DDLEvent) GetDDLType() model.ActionType { return model.ActionType(e.Type) } @@ -479,6 +558,45 @@ func (t *DDLEvent) IsPaused() bool { return false } +// CloneForRouting creates a shallow copy of the DDLEvent that can safely be mutated +// for table-route purposes without affecting the original event. +// +// The clone shares most read-only fields with the original. Slice fields that can be +// replaced independently downstream are copied so routing can update them without +// mutating shared state. +func (d *DDLEvent) CloneForRouting() *DDLEvent { + if d == nil { + return nil + } + + // Create shallow copy + clone := *d + + // PostTxnFlushed needs its own backing array to prevent potential races. + // Currently, DDL events arrive with nil PostTxnFlushed (callbacks are added + // downstream by basic_dispatcher.go), so append(nil, f) naturally creates a + // fresh slice. However, we make an explicit copy here for future-proofing: + // if any code path later adds callbacks before cloning, sharing the backing + // array could cause nondeterministic callback visibility or data races. + if d.PostTxnFlushed != nil { + clone.PostTxnFlushed = make([]func(), len(d.PostTxnFlushed)) + copy(clone.PostTxnFlushed, d.PostTxnFlushed) + } + + // MultipleTableInfos needs a new slice so each dispatcher can independently + // apply routing to its elements without affecting others + if d.MultipleTableInfos != nil { + clone.MultipleTableInfos = make([]*common.TableInfo, len(d.MultipleTableInfos)) + copy(clone.MultipleTableInfos, d.MultipleTableInfos) + } + + if d.BlockedTableNames != nil { + clone.BlockedTableNames = append([]SchemaTableName(nil), d.BlockedTableNames...) + } + + return &clone +} + func (t *DDLEvent) Len() int32 { return 1 } diff --git a/pkg/common/event/ddl_event_test.go b/pkg/common/event/ddl_event_test.go index 6cb58852e8..2921b33096 100644 --- a/pkg/common/event/ddl_event_test.go +++ b/pkg/common/event/ddl_event_test.go @@ -21,6 +21,8 @@ import ( "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/tidb/pkg/meta/model" + "github.com/pingcap/tidb/pkg/parser/ast" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -503,3 +505,192 @@ INSERT INTO test VALUES (1); }) } } + +// TestDDLEventCloneForRouting tests the CloneForRouting method to ensure it properly +// clones DDL events to avoid race conditions between multiple dispatchers +func TestDDLEventCloneForRouting(t *testing.T) { + helper := NewEventTestHelper(t) + defer helper.Close() + + helper.tk.MustExec("use test") + ddlJob := helper.DDL2Job(createTableSQL) + require.NotNil(t, ddlJob) + + // Create original DDL event with all fields populated + originalTableInfo := common.WrapTableInfo(ddlJob.SchemaName, ddlJob.BinlogInfo.TableInfo) + originalTableInfo.InitPrivateFields() + + multipleTableInfo1 := common.WrapTableInfo("schema1", ddlJob.BinlogInfo.TableInfo) + multipleTableInfo1.InitPrivateFields() + multipleTableInfo2 := common.WrapTableInfo("schema2", ddlJob.BinlogInfo.TableInfo) + multipleTableInfo2.InitPrivateFields() + + postFlushFunc1 := func() {} + postFlushFunc2 := func() {} + + original := &DDLEvent{ + Version: DDLEventVersion1, + DispatcherID: common.NewDispatcherID(), + Type: byte(ddlJob.Type), + SchemaID: ddlJob.SchemaID, + SchemaName: ddlJob.SchemaName, + TableName: ddlJob.TableName, + Query: ddlJob.Query, + TableInfo: originalTableInfo, + FinishedTs: ddlJob.BinlogInfo.FinishedTS, + Seq: 1, + Epoch: 2, + MultipleTableInfos: []*common.TableInfo{multipleTableInfo1, multipleTableInfo2}, + PostTxnFlushed: []func(){postFlushFunc1, postFlushFunc2}, + TiDBOnly: true, + BDRMode: "test-mode", + } + + // Clone the event + cloned := original.CloneForRouting() + require.NotNil(t, cloned) + + // Verify that cloned is a separate object + require.False(t, original == cloned, "cloned event should be a different object") + + // Verify that immutable fields are shared (shallow copy) + require.Equal(t, original.Version, cloned.Version) + require.Equal(t, original.DispatcherID, cloned.DispatcherID) + require.Equal(t, original.Type, cloned.Type) + require.Equal(t, original.SchemaID, cloned.SchemaID) + require.Equal(t, original.SchemaName, cloned.SchemaName) + require.Equal(t, original.TableName, cloned.TableName) + require.Equal(t, original.Query, cloned.Query) + require.Equal(t, original.FinishedTs, cloned.FinishedTs) + require.Equal(t, original.Seq, cloned.Seq) + require.Equal(t, original.Epoch, cloned.Epoch) + require.Equal(t, original.TiDBOnly, cloned.TiDBOnly) + require.Equal(t, original.BDRMode, cloned.BDRMode) + + // Verify that TableInfo pointer is shared initially + require.True(t, original.TableInfo == cloned.TableInfo, "TableInfo should be shared initially") + + // Verify that MultipleTableInfos is a new slice (but points to same TableInfo objects initially) + require.False(t, &original.MultipleTableInfos[0] == &cloned.MultipleTableInfos[0], "MultipleTableInfos should be a new slice") + require.True(t, original.MultipleTableInfos[0] == cloned.MultipleTableInfos[0], "MultipleTableInfos elements should be shared initially") + require.True(t, original.MultipleTableInfos[1] == cloned.MultipleTableInfos[1], "MultipleTableInfos elements should be shared initially") + + // Verify that PostTxnFlushed is an independent copy (not shared) + // This is defensive: currently DDL events arrive with nil PostTxnFlushed, + // but we copy it to prevent races if callbacks are ever added before cloning. + require.NotNil(t, cloned.PostTxnFlushed) + require.Equal(t, 2, len(cloned.PostTxnFlushed), "PostTxnFlushed should have same length as original") + require.Equal(t, 2, len(original.PostTxnFlushed), "Original PostTxnFlushed should remain unchanged") + // Verify independent backing arrays - appending to clone should not affect original + require.NotEqual(t, &original.PostTxnFlushed[0], &cloned.PostTxnFlushed[0], "PostTxnFlushed should have independent backing arrays") + + // Verify that appending to cloned PostTxnFlushed doesn't affect original + cloned.AddPostFlushFunc(func() {}) + require.Equal(t, 3, len(cloned.PostTxnFlushed), "Clone should have appended callback") + require.Equal(t, 2, len(original.PostTxnFlushed), "Original should be unaffected by clone's append") + + // Now simulate what happens during routing: mutate the cloned event + cloned.SchemaName = "routed_schema" + cloned.Query = "CREATE TABLE routed_schema.test ..." + newRoutedTableInfo := originalTableInfo.CloneWithRouting("routed_schema", "test") + cloned.TableInfo = newRoutedTableInfo + cloned.MultipleTableInfos[0] = multipleTableInfo1.CloneWithRouting("routed_schema1", "table1") + cloned.MultipleTableInfos[1] = multipleTableInfo2.CloneWithRouting("routed_schema2", "table2") + + // Verify that mutations to cloned event don't affect the original + require.Equal(t, ddlJob.SchemaName, original.SchemaName, "Original SchemaName should be unchanged") + require.Equal(t, ddlJob.Query, original.Query, "Original Query should be unchanged") + require.True(t, original.TableInfo == originalTableInfo, "Original TableInfo should be unchanged") + require.True(t, original.MultipleTableInfos[0] == multipleTableInfo1, "Original MultipleTableInfos[0] should be unchanged") + require.True(t, original.MultipleTableInfos[1] == multipleTableInfo2, "Original MultipleTableInfos[1] should be unchanged") + + // Verify that cloned event has the mutations + require.Equal(t, "routed_schema", cloned.TargetSchemaName) + require.Equal(t, "CREATE TABLE routed_schema.test ...", cloned.Query) + require.True(t, cloned.TableInfo == newRoutedTableInfo) + require.Equal(t, "routed_schema", cloned.TableInfo.TableName.TargetSchema) + require.Equal(t, original.SchemaName, cloned.GetSourceSchemaName()) + require.Equal(t, original.TableName, cloned.GetSourceTableName()) + + // Test cloning nil event + var nilEvent *DDLEvent + clonedNil := nilEvent.CloneForRouting() + require.Nil(t, clonedNil) +} + +func TestCloneForRoutingPreservesSourceFields(t *testing.T) { + original := &DDLEvent{ + SchemaName: "source_db", + TableName: "new_orders", + ExtraSchemaName: "source_db", + ExtraTableName: "old_orders", + TargetSchemaName: "target_db", + TargetTableName: "new_orders_routed", + TargetExtraSchemaName: "target_db", + TargetExtraTableName: "old_orders_routed", + } + + cloned := original.CloneForRouting() + cloned.TargetSchemaName = "target_db_v2" + cloned.TargetTableName = "new_orders_routed_v2" + cloned.TargetExtraSchemaName = "target_db_v2" + cloned.TargetExtraTableName = "old_orders_routed_v2" + + require.Equal(t, "source_db", cloned.GetSourceSchemaName()) + require.Equal(t, "new_orders", cloned.GetSourceTableName()) + require.Equal(t, "source_db", cloned.GetSourceExtraSchemaName()) + require.Equal(t, "old_orders", cloned.GetSourceExtraTableName()) + require.Equal(t, "target_db_v2", cloned.GetTargetSchemaName()) + require.Equal(t, "new_orders_routed_v2", cloned.GetTargetTableName()) + require.Equal(t, "target_db_v2", cloned.GetTargetExtraSchemaName()) + require.Equal(t, "old_orders_routed_v2", cloned.GetTargetExtraTableName()) +} + +func TestGetEventsForRenameTablesPreservesSourceAndTargetNames(t *testing.T) { + sourceTable1 := common.WrapTableInfo("new_db1", &model.TableInfo{ + ID: 100, + Name: ast.NewCIStr("new_table1"), + UpdateTS: 10, + }) + sourceTable2 := common.WrapTableInfo("new_db2", &model.TableInfo{ + ID: 101, + Name: ast.NewCIStr("new_table2"), + UpdateTS: 11, + }) + + ddl := &DDLEvent{ + Type: byte(model.ActionRenameTables), + Query: "RENAME TABLE `old_target_db1`.`old_target_table1` TO `new_target_db1`.`new_target_table1`; RENAME TABLE `old_target_db2`.`old_target_table2` TO `new_target_db2`.`new_target_table2`", + MultipleTableInfos: []*common.TableInfo{ + sourceTable1.CloneWithRouting("new_target_db1", "new_target_table1"), + sourceTable2.CloneWithRouting("new_target_db2", "new_target_table2"), + }, + TableNameChange: &TableNameChange{ + DropName: []SchemaTableName{ + {SchemaName: "old_db1", TableName: "old_table1"}, + {SchemaName: "old_db2", TableName: "old_table2"}, + }, + }, + } + + events := ddl.GetEvents() + require.Len(t, events, 2) + + require.Equal(t, "new_db1", events[0].SchemaName) + require.Equal(t, "new_table1", events[0].TableName) + require.Equal(t, "new_target_db1", events[0].TargetSchemaName) + require.Equal(t, "new_target_table1", events[0].TargetTableName) + require.Equal(t, "old_db1", events[0].ExtraSchemaName) + require.Equal(t, "old_table1", events[0].ExtraTableName) + require.Equal(t, "old_target_db1", events[0].TargetExtraSchemaName) + require.Equal(t, "old_target_table1", events[0].TargetExtraTableName) + + require.Equal(t, "new_db2", events[1].SchemaName) + require.Equal(t, "new_table2", events[1].TableName) + require.Equal(t, "new_target_db2", events[1].TargetSchemaName) + require.Equal(t, "new_target_table2", events[1].TargetTableName) + require.Equal(t, "old_db2", events[1].ExtraSchemaName) + require.Equal(t, "old_table2", events[1].ExtraTableName) + require.Equal(t, "old_target_db2", events[1].TargetExtraSchemaName) + require.Equal(t, "old_target_table2", events[1].TargetExtraTableName) +} diff --git a/pkg/common/event/dml_event.go b/pkg/common/event/dml_event.go index 8868da40dd..48f75bfcff 100644 --- a/pkg/common/event/dml_event.go +++ b/pkg/common/event/dml_event.go @@ -278,17 +278,37 @@ func (b *BatchDMLEvent) encodeV1() ([]byte, error) { // AssembleRows assembles the Rows from the RawRows. // It also sets the TableInfo and clears the RawRows. +// For local events (same node, b.Rows already set), it only applies routing +// without replacing the TableInfo to preserve schema version compatibility. func (b *BatchDMLEvent) AssembleRows(tableInfo *common.TableInfo) { + if tableInfo == nil { + log.Panic("DMLEvent: TableInfo is nil") + } + defer func() { b.TableInfo.InitPrivateFields() }() - // rows is already set, no need to assemble again - // When the event is passed from the same node, the Rows is already set. + + // For local events (same node), rows are already set. + // If routing is configured, reassign the TableInfo pointer to the passed tableInfo + // (which already has TargetSchema/TargetTable set via CloneWithRouting). + // IMPORTANT: We modify the POINTER, not the object it points to, because the + // original TableInfo is shared from the schema store across all dispatchers. if b.Rows != nil { + if tableInfo.TableName.TargetSchema != "" || tableInfo.TableName.TargetTable != "" { + b.TableInfo = tableInfo + for _, dml := range b.DMLEvents { + dml.TableInfo = tableInfo + } + } return } - if tableInfo == nil { - log.Panic("DMLEvent: TableInfo is nil") + + // For remote events, verify schema version compatibility before replacing TableInfo + if b.TableInfo != nil && b.TableInfo.GetUpdateTS() != tableInfo.GetUpdateTS() { + log.Panic("DMLEvent: TableInfoVersion mismatch", + zap.Uint64("dmlEventTableInfoVersion", b.TableInfo.GetUpdateTS()), + zap.Uint64("tableInfoVersion", tableInfo.GetUpdateTS())) return } @@ -297,10 +317,6 @@ func (b *BatchDMLEvent) AssembleRows(tableInfo *common.TableInfo) { return } - if b.TableInfo != nil && b.TableInfo.GetUpdateTS() != tableInfo.GetUpdateTS() { - log.Panic("DMLEvent: TableInfoVersion mismatch", zap.Uint64("dmlEventTableInfoVersion", b.TableInfo.GetUpdateTS()), zap.Uint64("tableInfoVersion", tableInfo.GetUpdateTS())) - return - } decoder := chunk.NewCodec(tableInfo.GetFieldSlice()) b.Rows, _ = decoder.Decode(b.RawRows) b.TableInfo = tableInfo diff --git a/pkg/common/event/redo.go b/pkg/common/event/redo.go index 5f3b48ad9e..de051cc5a5 100644 --- a/pkg/common/event/redo.go +++ b/pkg/common/event/redo.go @@ -17,7 +17,7 @@ import ( "fmt" "github.com/pingcap/log" - commonType "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/util" timodel "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/parser/ast" @@ -51,10 +51,10 @@ type RedoDMLEvent struct { // RedoDDLEvent represents DDL event used in redo log persistent type RedoDDLEvent struct { - DDL *DDLEventInRedoLog `msg:"ddl"` - Type byte `msg:"type"` - TableName commonType.TableName `msg:"table-name"` - TableSchemaStore *TableSchemaStore `msg:"table-schema-store"` + DDL *DDLEventInRedoLog `msg:"ddl"` + Type byte `msg:"type"` + TableName common.TableName `msg:"table-name"` + TableSchemaStore *TableSchemaStore `msg:"table-schema-store"` } // DMLEventInRedoLog is used to store DMLEvent in redo log v2 format @@ -64,7 +64,7 @@ type DMLEventInRedoLog struct { // Table contains the table name and table ID. // NOTICE: We store the physical table ID here, not the logical table ID. - Table *commonType.TableName `msg:"table"` + Table *common.TableName `msg:"table"` Columns []*RedoColumn `msg:"columns"` PreColumns []*RedoColumn `msg:"pre-columns"` @@ -105,7 +105,7 @@ type RedoRowEvent struct { StartTs uint64 CommitTs uint64 PhysicalTableID int64 - TableInfo *commonType.TableInfo + TableInfo *common.TableInfo Event RowChange Callback func() } @@ -142,29 +142,31 @@ func (r *RedoRowEvent) ToRedoLog() *RedoLog { Type: RedoLogTypeRow, } if r.TableInfo != nil { - redoLog.RedoRow.Row.Table = &commonType.TableName{ - Schema: r.TableInfo.TableName.Schema, - Table: r.TableInfo.TableName.Table, - TableID: r.PhysicalTableID, - IsPartition: r.TableInfo.TableName.IsPartition, + redoLog.RedoRow.Row.Table = &common.TableName{ + Schema: r.TableInfo.TableName.Schema, + Table: r.TableInfo.TableName.Table, + TableID: r.PhysicalTableID, + IsPartition: r.TableInfo.TableName.IsPartition, + TargetSchema: r.TableInfo.TableName.TargetSchema, + TargetTable: r.TableInfo.TableName.TargetTable, } redoLog.RedoRow.Row.IndexColumns = getIndexColumns(r.TableInfo) columnCount := len(r.TableInfo.GetColumns()) columns := make([]*RedoColumn, 0, columnCount) switch r.Event.RowType { - case commonType.RowTypeInsert: + case common.RowTypeInsert: redoLog.RedoRow.Columns = make([]RedoColumnValue, 0, columnCount) - case commonType.RowTypeDelete: + case common.RowTypeDelete: redoLog.RedoRow.PreColumns = make([]RedoColumnValue, 0, columnCount) - case commonType.RowTypeUpdate: + case common.RowTypeUpdate: redoLog.RedoRow.Columns = make([]RedoColumnValue, 0, columnCount) redoLog.RedoRow.PreColumns = make([]RedoColumnValue, 0, columnCount) default: } for i, column := range r.TableInfo.GetColumns() { - if commonType.IsColCDCVisible(column) { + if common.IsColCDCVisible(column) { columns = append(columns, &RedoColumn{ Name: column.Name.String(), Type: column.GetType(), @@ -173,13 +175,13 @@ func (r *RedoRowEvent) ToRedoLog() *RedoLog { }) isHandleKey := r.TableInfo.IsHandleKey(column.ID) switch r.Event.RowType { - case commonType.RowTypeInsert: + case common.RowTypeInsert: v := parseColumnValue(&r.Event.Row, column, i, isHandleKey) redoLog.RedoRow.Columns = append(redoLog.RedoRow.Columns, v) - case commonType.RowTypeDelete: + case common.RowTypeDelete: v := parseColumnValue(&r.Event.PreRow, column, i, isHandleKey) redoLog.RedoRow.PreColumns = append(redoLog.RedoRow.PreColumns, v) - case commonType.RowTypeUpdate: + case common.RowTypeUpdate: v := parseColumnValue(&r.Event.Row, column, i, isHandleKey) redoLog.RedoRow.Columns = append(redoLog.RedoRow.Columns, v) v = parseColumnValue(&r.Event.PreRow, column, i, isHandleKey) @@ -189,11 +191,11 @@ func (r *RedoRowEvent) ToRedoLog() *RedoLog { } } switch r.Event.RowType { - case commonType.RowTypeInsert: + case common.RowTypeInsert: redoLog.RedoRow.Row.Columns = columns - case commonType.RowTypeDelete: + case common.RowTypeDelete: redoLog.RedoRow.Row.PreColumns = columns - case commonType.RowTypeUpdate: + case common.RowTypeUpdate: redoLog.RedoRow.Row.Columns = columns redoLog.RedoRow.Row.PreColumns = columns } @@ -226,7 +228,7 @@ func (d *DDLEvent) ToRedoLog() *RedoLog { } // GetCommitTs returns commit timestamp of the log event. -func (r *RedoLog) GetCommitTs() commonType.Ts { +func (r *RedoLog) GetCommitTs() common.Ts { switch r.Type { case RedoLogTypeRow: return r.RedoRow.Row.CommitTs @@ -279,7 +281,7 @@ func (r *RedoDMLEvent) ToDMLEvent() *DMLEvent { colInfo.SetType(col.Type) colInfo.SetCharset(col.Charset) colInfo.SetCollate(col.Collation) - flag := commonType.ColumnFlagType(rawColsValue[idx].Flag) + flag := common.ColumnFlagType(rawColsValue[idx].Flag) // if flag.IsHandleKey() { // } // if flag.IsBinary(){ @@ -327,8 +329,18 @@ func (r *RedoDMLEvent) ToDMLEvent() *DMLEvent { indexInfo.Primary = isPrimary tidbTableInfo.Indices = append(tidbTableInfo.Indices, indexInfo) } + tableInfo := common.NewTableInfo4Decoder(r.Row.Table.Schema, tidbTableInfo) + // Restore routing info from redo log (TargetSchema/TargetTable for table routing). + // We must use CloneWithRouting because NewTableInfo4Decoder already called InitPrivateFields() + // which pre-computed SQL statements using the source schema/table. CloneWithRouting creates + // a new TableInfo with routing applied and uninitialized preSQLs that will be computed + // correctly when InitPrivateFields() is called. + if r.Row.Table.TargetSchema != "" || r.Row.Table.TargetTable != "" { + tableInfo = tableInfo.CloneWithRouting(r.Row.Table.TargetSchema, r.Row.Table.TargetTable) + tableInfo.InitPrivateFields() + } event := &DMLEvent{ - TableInfo: commonType.NewTableInfo4Decoder(r.Row.Table.Schema, tidbTableInfo), + TableInfo: tableInfo, CommitTs: r.Row.CommitTs, StartTs: r.Row.StartTs, Length: 1, @@ -342,15 +354,15 @@ func (r *RedoDMLEvent) ToDMLEvent() *DMLEvent { columns := event.TableInfo.GetColumns() if r.IsDelete() { collectAllColumnsValue(r.PreColumns, columns, chk) - event.RowTypes = append(event.RowTypes, commonType.RowTypeDelete) + event.RowTypes = append(event.RowTypes, common.RowTypeDelete) } else if r.IsUpdate() { collectAllColumnsValue(r.PreColumns, columns, chk) collectAllColumnsValue(r.Columns, columns, chk) // FIXME: exclude columns with same value - event.RowTypes = append(event.RowTypes, commonType.RowTypeUpdate, commonType.RowTypeUpdate) + event.RowTypes = append(event.RowTypes, common.RowTypeUpdate, common.RowTypeUpdate) } else if r.IsInsert() { collectAllColumnsValue(r.Columns, columns, chk) - event.RowTypes = append(event.RowTypes, commonType.RowTypeInsert) + event.RowTypes = append(event.RowTypes, common.RowTypeInsert) } else { log.Panic("unknown event type for the DML event") } @@ -361,18 +373,24 @@ func (r *RedoDMLEvent) ToDMLEvent() *DMLEvent { func (r *RedoDDLEvent) ToDDLEvent() *DDLEvent { blockedTables := r.DDL.BlockedTables blockedTableNames := r.DDL.BlockedTableNames + sourceSchemaName := r.TableName.GetOriginSchema() + sourceTableName := r.TableName.GetOriginTable() + targetSchemaName := r.TableName.GetTargetSchema() + targetTableName := r.TableName.GetTargetTable() if blockedTables == nil { blockedTables = &InfluencedTables{InfluenceType: InfluenceTypeNormal} - blockedTableNames = []SchemaTableName{{SchemaName: r.TableName.Schema, TableName: r.TableName.Table}} + blockedTableNames = []SchemaTableName{{SchemaName: targetSchemaName, TableName: targetTableName}} } return &DDLEvent{ - TableInfo: &commonType.TableInfo{ + TableInfo: &common.TableInfo{ TableName: r.TableName, }, Query: r.DDL.Query, Type: r.Type, - SchemaName: r.TableName.Schema, - TableName: r.TableName.Table, + SchemaName: sourceSchemaName, + TableName: sourceTableName, + TargetSchemaName: targetSchemaName, + TargetTableName: targetTableName, FinishedTs: r.DDL.CommitTs, StartTs: r.DDL.StartTs, BlockedTables: blockedTables, @@ -389,7 +407,7 @@ func (r *RedoDDLEvent) SetTableSchemaStore(tableSchemaStore *TableSchemaStore) { } func parseColumnValue(row *chunk.Row, colInfo *timodel.ColumnInfo, i int, isHandleKey bool) RedoColumnValue { - v := commonType.ExtractColVal(row, colInfo, i) + v := common.ExtractColVal(row, colInfo, i) switch colInfo.GetType() { case mysql.TypeString, mysql.TypeVarString, mysql.TypeVarchar, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob, mysql.TypeBlob: @@ -409,7 +427,7 @@ func parseColumnValue(row *chunk.Row, colInfo *timodel.ColumnInfo, i int, isHand // For compatibility func convertFlag(colInfo *timodel.ColumnInfo, isHandleKey bool) uint64 { - var flag commonType.ColumnFlagType + var flag common.ColumnFlagType if isHandleKey { flag.SetIsHandleKey() } @@ -438,7 +456,7 @@ func convertFlag(colInfo *timodel.ColumnInfo, isHandleKey bool) uint64 { } // For compatibility -func getIndexColumns(tableInfo *commonType.TableInfo) [][]int { +func getIndexColumns(tableInfo *common.TableInfo) [][]int { indexColumns := make([][]int, 0, len(tableInfo.GetIndexColumns())) rowColumnsOffset := tableInfo.GetRowColumnsOffset() for _, index := range tableInfo.GetIndexColumns() { diff --git a/pkg/common/event/redo_test.go b/pkg/common/event/redo_test.go new file mode 100644 index 0000000000..ce9408f68c --- /dev/null +++ b/pkg/common/event/redo_test.go @@ -0,0 +1,94 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package event + +import ( + "testing" + + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/tidb/pkg/meta/model" + "github.com/stretchr/testify/require" +) + +func TestRedoDMLEventToDMLEventPreservesSourceAndTargetNames(t *testing.T) { + t.Parallel() + + helper := NewEventTestHelper(t) + defer helper.Close() + + helper.Tk().MustExec("use test") + job := helper.DDL2Job(`create table test.t(id int primary key, name varchar(32))`) + require.NotNil(t, job) + + sourceTableInfo := helper.GetTableInfo(job) + routedTableInfo := sourceTableInfo.CloneWithRouting("target_db", "target_table") + + dmlEvent := helper.DML2Event("test", "t", `insert into test.t values (1, 'alice')`) + dmlEvent.TableInfo = routedTableInfo + + row, ok := dmlEvent.GetNextRow() + require.True(t, ok) + + redoRow := (&RedoRowEvent{ + StartTs: dmlEvent.StartTs, + CommitTs: dmlEvent.CommitTs, + PhysicalTableID: dmlEvent.PhysicalTableID, + TableInfo: routedTableInfo, + Event: row, + }).ToRedoLog().RedoRow + + decoded := redoRow.ToDMLEvent() + require.Equal(t, "test", decoded.TableInfo.GetSchemaName()) + require.Equal(t, "t", decoded.TableInfo.GetTableName()) + require.Equal(t, "target_db", decoded.TableInfo.GetTargetSchemaName()) + require.Equal(t, "target_table", decoded.TableInfo.GetTargetTableName()) + require.Equal(t, "test", decoded.TableInfo.GetSourceSchemaName()) + require.Equal(t, "t", decoded.TableInfo.GetSourceTableName()) +} + +func TestRedoDDLEventToDDLEventPreservesSourceAndTargetNames(t *testing.T) { + t.Parallel() + + redoDDLEvent := &RedoDDLEvent{ + DDL: &DDLEventInRedoLog{ + StartTs: 100, + CommitTs: 200, + Query: "ALTER TABLE `target_db`.`target_table` ADD COLUMN age INT", + }, + Type: byte(model.ActionAddColumn), + TableName: common.TableName{ + Schema: "source_db", + Table: "source_table", + TargetSchema: "target_db", + TargetTable: "target_table", + }, + } + + ddlEvent := redoDDLEvent.ToDDLEvent() + require.Equal(t, "source_db", ddlEvent.SchemaName) + require.Equal(t, "source_table", ddlEvent.TableName) + require.Equal(t, "target_db", ddlEvent.TargetSchemaName) + require.Equal(t, "target_table", ddlEvent.TargetTableName) + require.Equal(t, "target_db", ddlEvent.GetDDLSchemaName()) + require.Equal(t, "source_db", ddlEvent.TableInfo.GetSchemaName()) + require.Equal(t, "source_table", ddlEvent.TableInfo.GetTableName()) + require.Equal(t, "target_db", ddlEvent.TableInfo.GetTargetSchemaName()) + require.Equal(t, "target_table", ddlEvent.TableInfo.GetTargetTableName()) + require.Equal(t, "source_db", ddlEvent.TableInfo.GetSourceSchemaName()) + require.Equal(t, "source_table", ddlEvent.TableInfo.GetSourceTableName()) + require.Equal(t, []SchemaTableName{{ + SchemaName: "target_db", + TableName: "target_table", + }}, ddlEvent.BlockedTableNames) +} diff --git a/pkg/common/table_info.go b/pkg/common/table_info.go index 8af5ebdc77..a43d2f79fe 100644 --- a/pkg/common/table_info.go +++ b/pkg/common/table_info.go @@ -125,6 +125,14 @@ func (ti *TableInfo) InitPrivateFields() { return } + // columnSchema may be nil for minimal TableInfo instances (e.g., in tests). + // In production, columnSchema is always set via WrapTableInfo or similar. + // Early return here without marking as initialized, so if columnSchema is + // set later, InitPrivateFields can be called again to properly initialize. + if ti.columnSchema == nil { + return + } + ti.preSQLs.mutex.Lock() defer ti.preSQLs.mutex.Unlock() @@ -133,13 +141,56 @@ func (ti *TableInfo) InitPrivateFields() { return } - ti.preSQLs.m[preSQLInsert] = fmt.Sprintf(ti.columnSchema.PreSQLs[preSQLInsert], ti.TableName.QuoteString()) - ti.preSQLs.m[preSQLReplace] = fmt.Sprintf(ti.columnSchema.PreSQLs[preSQLReplace], ti.TableName.QuoteString()) - ti.preSQLs.m[preSQLUpdate] = fmt.Sprintf(ti.columnSchema.PreSQLs[preSQLUpdate], ti.TableName.QuoteString()) + ti.preSQLs.m[preSQLInsert] = fmt.Sprintf(ti.columnSchema.PreSQLs[preSQLInsert], ti.TableName.QuoteTargetString()) + ti.preSQLs.m[preSQLReplace] = fmt.Sprintf(ti.columnSchema.PreSQLs[preSQLReplace], ti.TableName.QuoteTargetString()) + ti.preSQLs.m[preSQLUpdate] = fmt.Sprintf(ti.columnSchema.PreSQLs[preSQLUpdate], ti.TableName.QuoteTargetString()) ti.preSQLs.isInitialized.Store(true) } +// CloneWithRouting creates a shallow copy of TableInfo with routing applied. +// The new TableInfo shares the same columnSchema, View, Sequence pointers +// but has its own TableName (with TargetSchema/TargetTable set) and uninitialized preSQLs. +// This is safe because: +// - columnSchema, View, Sequence are read-only after creation +// - preSQLs will be initialized later via InitPrivateFields() using the new TableName +// - TableName is a value type that gets copied +func (ti *TableInfo) CloneWithRouting(targetSchema, targetTable string) *TableInfo { + if ti == nil { + return nil + } + // Create a new TableInfo with copied basic fields + cloned := &TableInfo{ + TableName: ti.TableName, // Value copy of TableName struct + Charset: ti.Charset, + Collate: ti.Collate, + Comment: ti.Comment, + columnSchema: ti.columnSchema, // Share the pointer (read-only) + HasPKOrNotNullUK: ti.HasPKOrNotNullUK, + View: ti.View, // Share the pointer (read-only) + Sequence: ti.Sequence, // Share the pointer (read-only) + UpdateTS: ti.UpdateTS, + ActiveActiveTable: ti.ActiveActiveTable, + SoftDeleteTable: ti.SoftDeleteTable, + // preSQLs is zero-initialized (uninitialized mutex/atomic, empty strings) + } + // Apply routing to the cloned TableName while keeping Schema/Table as source names. + cloned.TableName.TargetSchema = targetSchema + cloned.TableName.TargetTable = targetTable + + // Increment refcount for the shared columnSchema and set finalizer to decrement + // when the clone is garbage collected. This prevents use-after-free if the + // original TableInfo is GC'd before the clone. + if ti.columnSchema != nil { + GetSharedColumnSchemaStorage().incColumnSchemaCount(ti.columnSchema) + runtime.SetFinalizer(cloned, func(ti *TableInfo) { + GetSharedColumnSchemaStorage().tryReleaseColumnSchema(ti.columnSchema) + }) + } + + return cloned +} + func (ti *TableInfo) Marshal() ([]byte, error) { // otherField | columnSchemaData | columnSchemaDataSize data, err := json.Marshal(ti) @@ -341,26 +392,26 @@ func (ti *TableInfo) MustGetColumnOffsetByID(id int64) int { return offset } -// GetSchemaName returns the schema name of the table +// GetSchemaName returns the source schema name carried by this TableInfo. func (ti *TableInfo) GetSchemaName() string { - return ti.TableName.Schema + return ti.TableName.GetOriginSchema() } -// GetTableName returns the table name of the table +// GetTableName returns the source table name carried by this TableInfo. func (ti *TableInfo) GetTableName() string { - return ti.TableName.Table + return ti.TableName.GetOriginTable() } func (ti *TableInfo) GetTableNameCIStr() ast.CIStr { - return ast.NewCIStr(ti.TableName.Table) + return ast.NewCIStr(ti.GetTableName()) } -// GetSchemaNamePtr returns the pointer to the schema name of the table +// GetSchemaNamePtr returns the pointer to the source schema name. func (ti *TableInfo) GetSchemaNamePtr() *string { return &ti.TableName.Schema } -// GetTableNamePtr returns the pointer to the table name of the table +// GetTableNamePtr returns the pointer to the source table name. func (ti *TableInfo) GetTableNamePtr() *string { return &ti.TableName.Table } @@ -370,6 +421,28 @@ func (ti *TableInfo) IsPartitionTable() bool { return ti.TableName.IsPartition } +// GetTargetSchemaName returns the target schema name for routing. +// If TargetSchema is empty, returns Schema. +func (ti *TableInfo) GetTargetSchemaName() string { + return ti.TableName.GetTargetSchema() +} + +// GetTargetTableName returns the target table name for routing. +// If TargetTable is empty, returns Table. +func (ti *TableInfo) GetTargetTableName() string { + return ti.TableName.GetTargetTable() +} + +// GetSourceSchemaName returns the source schema name before routing. +func (ti *TableInfo) GetSourceSchemaName() string { + return ti.TableName.GetOriginSchema() +} + +// GetSourceTableName returns the source table name before routing. +func (ti *TableInfo) GetSourceTableName() string { + return ti.TableName.GetOriginTable() +} + // IsView checks if TableInfo is a view. func (t *TableInfo) IsView() bool { return t.View != nil diff --git a/pkg/common/table_info_test.go b/pkg/common/table_info_test.go index 55c7e3fb22..b3d6607e0c 100644 --- a/pkg/common/table_info_test.go +++ b/pkg/common/table_info_test.go @@ -24,6 +24,82 @@ import ( "github.com/stretchr/testify/require" ) +func TestCloneWithRouting(t *testing.T) { + t.Parallel() + + t.Run("nil TableInfo", func(t *testing.T) { + var ti *TableInfo + cloned := ti.CloneWithRouting("target_schema", "target_table") + require.Nil(t, cloned) + }) + + t.Run("basic cloning with routing", func(t *testing.T) { + original := &TableInfo{ + TableName: TableName{ + Schema: "source_db", + Table: "source_table", + TableID: 123, + }, + Charset: "utf8mb4", + Collate: "utf8mb4_bin", + Comment: "test table", + HasPKOrNotNullUK: true, + UpdateTS: 1000, + } + + cloned := original.CloneWithRouting("target_db", "target_table") + + // Verify cloned has routing applied + require.Equal(t, "source_db", cloned.TableName.Schema) + require.Equal(t, "source_table", cloned.TableName.Table) + require.Equal(t, "target_db", cloned.TableName.TargetSchema) + require.Equal(t, "target_table", cloned.TableName.TargetTable) + require.Equal(t, int64(123), cloned.TableName.TableID) + require.Equal(t, "source_db", cloned.GetSchemaName()) + require.Equal(t, "source_table", cloned.GetTableName()) + require.Equal(t, "source_db", cloned.GetSourceSchemaName()) + require.Equal(t, "source_table", cloned.GetSourceTableName()) + require.Equal(t, "target_db", cloned.GetTargetSchemaName()) + require.Equal(t, "target_table", cloned.GetTargetTableName()) + require.Equal(t, "source_db.source_table", cloned.TableName.String()) + require.Equal(t, "`source_db`.`source_table`", cloned.TableName.QuoteString()) + require.Equal(t, "source_db.source_table", cloned.TableName.OriginString()) + require.Equal(t, "`source_db`.`source_table`", cloned.TableName.QuoteOriginString()) + require.Same(t, &cloned.TableName.Schema, cloned.GetSchemaNamePtr()) + require.Same(t, &cloned.TableName.Table, cloned.GetTableNamePtr()) + + // Verify other fields are copied + require.Equal(t, "utf8mb4", cloned.Charset) + require.Equal(t, "utf8mb4_bin", cloned.Collate) + require.Equal(t, "test table", cloned.Comment) + require.Equal(t, true, cloned.HasPKOrNotNullUK) + require.Equal(t, uint64(1000), cloned.UpdateTS) + + // Verify original is NOT modified + require.Equal(t, "", original.TableName.TargetSchema) + require.Equal(t, "", original.TableName.TargetTable) + }) + + t.Run("target getters remain available without changing source fields", func(t *testing.T) { + original := &TableInfo{ + TableName: TableName{ + Schema: "source_db", + Table: "source_table", + TableID: 123, + }, + } + + cloned := original.CloneWithRouting("target_db", "target_table") + + require.Equal(t, "source_db", cloned.GetSchemaName()) + require.Equal(t, "source_table", cloned.GetTableName()) + require.Equal(t, "source_db", cloned.GetSourceSchemaName()) + require.Equal(t, "source_table", cloned.GetSourceTableName()) + require.Equal(t, "target_db", cloned.GetTargetSchemaName()) + require.Equal(t, "target_table", cloned.GetTargetTableName()) + }) +} + func TestUnmarshalJSONToTableInfoInvalidData(t *testing.T) { t.Parallel() diff --git a/pkg/common/table_name.go b/pkg/common/table_name.go index 646a18c6a7..728d80b3cb 100644 --- a/pkg/common/table_name.go +++ b/pkg/common/table_name.go @@ -26,14 +26,66 @@ type TableName struct { // TableID is the logic table ID TableID int64 `toml:"tbl-id" msg:"tbl-id"` IsPartition bool `toml:"is-partition" msg:"is-partition"` + + // TargetSchema and TargetTable is not empty if table routing enabled + TargetSchema string `toml:"target-db-name" msg:"target-db-name"` + TargetTable string `toml:"target-tbl-name" msg:"target-tbl-name"` } // String implements fmt.Stringer interface. func (t TableName) String() string { - return fmt.Sprintf("%s.%s", t.Schema, t.Table) + return t.OriginString() +} + +// OriginString returns the source schema.table string. +func (t TableName) OriginString() string { + return fmt.Sprintf("%s.%s", t.GetOriginSchema(), t.GetOriginTable()) } -// QuoteString returns quoted full table name +// QuoteString returns quoted full canonical table name. func (t TableName) QuoteString() string { - return QuoteSchema(t.Schema, t.Table) + return t.QuoteOriginString() +} + +// QuoteOriginString returns quoted full source table name. +func (t TableName) QuoteOriginString() string { + return QuoteSchema(t.GetOriginSchema(), t.GetOriginTable()) +} + +// GetOriginSchema returns the source schema name. +func (t *TableName) GetOriginSchema() string { + return t.Schema +} + +// GetOriginTable returns the source table name. +func (t *TableName) GetOriginTable() string { + return t.Table +} + +// GetTargetSchema returns the target schema name for routing. +// If TargetSchema is empty, returns Schema. +func (t *TableName) GetTargetSchema() string { + if t.TargetSchema != "" { + return t.TargetSchema + } + return t.Schema +} + +// GetTargetTable returns the target table name for routing. +// If TargetTable is empty, returns Table. +func (t *TableName) GetTargetTable() string { + if t.TargetTable != "" { + return t.TargetTable + } + return t.Table +} + +// TargetString returns the target schema.table string for routing. +func (t TableName) TargetString() string { + return fmt.Sprintf("%s.%s", t.GetTargetSchema(), t.GetTargetTable()) +} + +// QuoteTargetString returns quoted full target table name for routing. +func (t TableName) QuoteTargetString() string { + return QuoteSchema(t.GetTargetSchema(), t.GetTargetTable()) } diff --git a/pkg/common/table_name_gen.go b/pkg/common/table_name_gen.go index 2f0df4ba92..31cb8c036e 100644 --- a/pkg/common/table_name_gen.go +++ b/pkg/common/table_name_gen.go @@ -48,6 +48,18 @@ func (z *TableName) DecodeMsg(dc *msgp.Reader) (err error) { err = msgp.WrapError(err, "IsPartition") return } + case "target-db-name": + z.TargetSchema, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "TargetSchema") + return + } + case "target-tbl-name": + z.TargetTable, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "TargetTable") + return + } default: err = dc.Skip() if err != nil { @@ -61,9 +73,9 @@ func (z *TableName) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *TableName) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 4 + // map header, size 6 // write "db-name" - err = en.Append(0x84, 0xa7, 0x64, 0x62, 0x2d, 0x6e, 0x61, 0x6d, 0x65) + err = en.Append(0x86, 0xa7, 0x64, 0x62, 0x2d, 0x6e, 0x61, 0x6d, 0x65) if err != nil { return } @@ -102,15 +114,35 @@ func (z *TableName) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "IsPartition") return } + // write "target-db-name" + err = en.Append(0xae, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2d, 0x64, 0x62, 0x2d, 0x6e, 0x61, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteString(z.TargetSchema) + if err != nil { + err = msgp.WrapError(err, "TargetSchema") + return + } + // write "target-tbl-name" + err = en.Append(0xaf, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2d, 0x74, 0x62, 0x6c, 0x2d, 0x6e, 0x61, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteString(z.TargetTable) + if err != nil { + err = msgp.WrapError(err, "TargetTable") + return + } return } // MarshalMsg implements msgp.Marshaler func (z *TableName) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // map header, size 4 + // map header, size 6 // string "db-name" - o = append(o, 0x84, 0xa7, 0x64, 0x62, 0x2d, 0x6e, 0x61, 0x6d, 0x65) + o = append(o, 0x86, 0xa7, 0x64, 0x62, 0x2d, 0x6e, 0x61, 0x6d, 0x65) o = msgp.AppendString(o, z.Schema) // string "tbl-name" o = append(o, 0xa8, 0x74, 0x62, 0x6c, 0x2d, 0x6e, 0x61, 0x6d, 0x65) @@ -121,6 +153,12 @@ func (z *TableName) MarshalMsg(b []byte) (o []byte, err error) { // string "is-partition" o = append(o, 0xac, 0x69, 0x73, 0x2d, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e) o = msgp.AppendBool(o, z.IsPartition) + // string "target-db-name" + o = append(o, 0xae, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2d, 0x64, 0x62, 0x2d, 0x6e, 0x61, 0x6d, 0x65) + o = msgp.AppendString(o, z.TargetSchema) + // string "target-tbl-name" + o = append(o, 0xaf, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2d, 0x74, 0x62, 0x6c, 0x2d, 0x6e, 0x61, 0x6d, 0x65) + o = msgp.AppendString(o, z.TargetTable) return } @@ -166,6 +204,18 @@ func (z *TableName) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "IsPartition") return } + case "target-db-name": + z.TargetSchema, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "TargetSchema") + return + } + case "target-tbl-name": + z.TargetTable, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "TargetTable") + return + } default: bts, err = msgp.Skip(bts) if err != nil { @@ -180,6 +230,6 @@ func (z *TableName) UnmarshalMsg(bts []byte) (o []byte, err error) { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *TableName) Msgsize() (s int) { - s = 1 + 8 + msgp.StringPrefixSize + len(z.Schema) + 9 + msgp.StringPrefixSize + len(z.Table) + 7 + msgp.Int64Size + 13 + msgp.BoolSize + s = 1 + 8 + msgp.StringPrefixSize + len(z.Schema) + 9 + msgp.StringPrefixSize + len(z.Table) + 7 + msgp.Int64Size + 13 + msgp.BoolSize + 15 + msgp.StringPrefixSize + len(z.TargetSchema) + 16 + msgp.StringPrefixSize + len(z.TargetTable) return } From f4a5e2b4db4e8f8a44e870f033826434db839830 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Wed, 1 Apr 2026 10:55:34 +0800 Subject: [PATCH 04/20] add more code --- pkg/common/event/redo_gen.go | 6 +++--- pkg/redo/writer/file/file_mock.go | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pkg/common/event/redo_gen.go b/pkg/common/event/redo_gen.go index 1e0226c78c..95d197304d 100644 --- a/pkg/common/event/redo_gen.go +++ b/pkg/common/event/redo_gen.go @@ -3,7 +3,7 @@ package event import ( - commonType "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/common" "github.com/tinylib/msgp/msgp" ) @@ -475,7 +475,7 @@ func (z *DMLEventInRedoLog) DecodeMsg(dc *msgp.Reader) (err error) { z.Table = nil } else { if z.Table == nil { - z.Table = new(commonType.TableName) + z.Table = new(common.TableName) } err = z.Table.DecodeMsg(dc) if err != nil { @@ -803,7 +803,7 @@ func (z *DMLEventInRedoLog) UnmarshalMsg(bts []byte) (o []byte, err error) { z.Table = nil } else { if z.Table == nil { - z.Table = new(commonType.TableName) + z.Table = new(common.TableName) } bts, err = z.Table.UnmarshalMsg(bts) if err != nil { diff --git a/pkg/redo/writer/file/file_mock.go b/pkg/redo/writer/file/file_mock.go index 40679247ce..b4305d52cd 100644 --- a/pkg/redo/writer/file/file_mock.go +++ b/pkg/redo/writer/file/file_mock.go @@ -6,9 +6,8 @@ import ( context "context" event "github.com/pingcap/ticdc/pkg/common/event" - mock "github.com/stretchr/testify/mock" - writer "github.com/pingcap/ticdc/pkg/redo/writer" + mock "github.com/stretchr/testify/mock" ) // mockFileWriter is an autogenerated mock type for the fileWriter type From 0988fbc283fce790e29af2d7ac480816ab0976a6 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Wed, 1 Apr 2026 11:46:55 +0800 Subject: [PATCH 05/20] pkg/common,event: rename source getters to origin --- pkg/common/event/ddl_event.go | 8 ++++---- pkg/common/event/ddl_event_test.go | 12 ++++++------ pkg/common/event/redo_test.go | 8 ++++---- pkg/common/table_info.go | 8 ++++---- pkg/common/table_info_test.go | 8 ++++---- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/pkg/common/event/ddl_event.go b/pkg/common/event/ddl_event.go index 0cd046461a..96cc908909 100644 --- a/pkg/common/event/ddl_event.go +++ b/pkg/common/event/ddl_event.go @@ -199,7 +199,7 @@ func (d *DDLEvent) GetSchemaName() string { return d.SchemaName } -func (d *DDLEvent) GetSourceSchemaName() string { +func (d *DDLEvent) GetOriginSchemaName() string { return d.SchemaName } @@ -207,7 +207,7 @@ func (d *DDLEvent) GetTableName() string { return d.TableName } -func (d *DDLEvent) GetSourceTableName() string { +func (d *DDLEvent) GetOriginTableName() string { return d.TableName } @@ -215,7 +215,7 @@ func (d *DDLEvent) GetExtraSchemaName() string { return d.ExtraSchemaName } -func (d *DDLEvent) GetSourceExtraSchemaName() string { +func (d *DDLEvent) GetOriginExtraSchemaName() string { return d.ExtraSchemaName } @@ -223,7 +223,7 @@ func (d *DDLEvent) GetExtraTableName() string { return d.ExtraTableName } -func (d *DDLEvent) GetSourceExtraTableName() string { +func (d *DDLEvent) GetOriginExtraTableName() string { return d.ExtraTableName } diff --git a/pkg/common/event/ddl_event_test.go b/pkg/common/event/ddl_event_test.go index 2921b33096..01ec7ef853 100644 --- a/pkg/common/event/ddl_event_test.go +++ b/pkg/common/event/ddl_event_test.go @@ -609,8 +609,8 @@ func TestDDLEventCloneForRouting(t *testing.T) { require.Equal(t, "CREATE TABLE routed_schema.test ...", cloned.Query) require.True(t, cloned.TableInfo == newRoutedTableInfo) require.Equal(t, "routed_schema", cloned.TableInfo.TableName.TargetSchema) - require.Equal(t, original.SchemaName, cloned.GetSourceSchemaName()) - require.Equal(t, original.TableName, cloned.GetSourceTableName()) + require.Equal(t, original.SchemaName, cloned.GetOriginSchemaName()) + require.Equal(t, original.TableName, cloned.GetOriginTableName()) // Test cloning nil event var nilEvent *DDLEvent @@ -636,10 +636,10 @@ func TestCloneForRoutingPreservesSourceFields(t *testing.T) { cloned.TargetExtraSchemaName = "target_db_v2" cloned.TargetExtraTableName = "old_orders_routed_v2" - require.Equal(t, "source_db", cloned.GetSourceSchemaName()) - require.Equal(t, "new_orders", cloned.GetSourceTableName()) - require.Equal(t, "source_db", cloned.GetSourceExtraSchemaName()) - require.Equal(t, "old_orders", cloned.GetSourceExtraTableName()) + require.Equal(t, "source_db", cloned.GetOriginSchemaName()) + require.Equal(t, "new_orders", cloned.GetOriginTableName()) + require.Equal(t, "source_db", cloned.GetOriginExtraSchemaName()) + require.Equal(t, "old_orders", cloned.GetOriginExtraTableName()) require.Equal(t, "target_db_v2", cloned.GetTargetSchemaName()) require.Equal(t, "new_orders_routed_v2", cloned.GetTargetTableName()) require.Equal(t, "target_db_v2", cloned.GetTargetExtraSchemaName()) diff --git a/pkg/common/event/redo_test.go b/pkg/common/event/redo_test.go index ce9408f68c..3834b9f4b9 100644 --- a/pkg/common/event/redo_test.go +++ b/pkg/common/event/redo_test.go @@ -53,8 +53,8 @@ func TestRedoDMLEventToDMLEventPreservesSourceAndTargetNames(t *testing.T) { require.Equal(t, "t", decoded.TableInfo.GetTableName()) require.Equal(t, "target_db", decoded.TableInfo.GetTargetSchemaName()) require.Equal(t, "target_table", decoded.TableInfo.GetTargetTableName()) - require.Equal(t, "test", decoded.TableInfo.GetSourceSchemaName()) - require.Equal(t, "t", decoded.TableInfo.GetSourceTableName()) + require.Equal(t, "test", decoded.TableInfo.GetOriginSchemaName()) + require.Equal(t, "t", decoded.TableInfo.GetOriginTableName()) } func TestRedoDDLEventToDDLEventPreservesSourceAndTargetNames(t *testing.T) { @@ -85,8 +85,8 @@ func TestRedoDDLEventToDDLEventPreservesSourceAndTargetNames(t *testing.T) { require.Equal(t, "source_table", ddlEvent.TableInfo.GetTableName()) require.Equal(t, "target_db", ddlEvent.TableInfo.GetTargetSchemaName()) require.Equal(t, "target_table", ddlEvent.TableInfo.GetTargetTableName()) - require.Equal(t, "source_db", ddlEvent.TableInfo.GetSourceSchemaName()) - require.Equal(t, "source_table", ddlEvent.TableInfo.GetSourceTableName()) + require.Equal(t, "source_db", ddlEvent.TableInfo.GetOriginSchemaName()) + require.Equal(t, "source_table", ddlEvent.TableInfo.GetOriginTableName()) require.Equal(t, []SchemaTableName{{ SchemaName: "target_db", TableName: "target_table", diff --git a/pkg/common/table_info.go b/pkg/common/table_info.go index a43d2f79fe..0bdd7874a2 100644 --- a/pkg/common/table_info.go +++ b/pkg/common/table_info.go @@ -433,13 +433,13 @@ func (ti *TableInfo) GetTargetTableName() string { return ti.TableName.GetTargetTable() } -// GetSourceSchemaName returns the source schema name before routing. -func (ti *TableInfo) GetSourceSchemaName() string { +// GetOriginSchemaName returns the origin schema name before routing. +func (ti *TableInfo) GetOriginSchemaName() string { return ti.TableName.GetOriginSchema() } -// GetSourceTableName returns the source table name before routing. -func (ti *TableInfo) GetSourceTableName() string { +// GetOriginTableName returns the origin table name before routing. +func (ti *TableInfo) GetOriginTableName() string { return ti.TableName.GetOriginTable() } diff --git a/pkg/common/table_info_test.go b/pkg/common/table_info_test.go index b3d6607e0c..1f16b5a24e 100644 --- a/pkg/common/table_info_test.go +++ b/pkg/common/table_info_test.go @@ -57,8 +57,8 @@ func TestCloneWithRouting(t *testing.T) { require.Equal(t, int64(123), cloned.TableName.TableID) require.Equal(t, "source_db", cloned.GetSchemaName()) require.Equal(t, "source_table", cloned.GetTableName()) - require.Equal(t, "source_db", cloned.GetSourceSchemaName()) - require.Equal(t, "source_table", cloned.GetSourceTableName()) + require.Equal(t, "source_db", cloned.GetOriginSchemaName()) + require.Equal(t, "source_table", cloned.GetOriginTableName()) require.Equal(t, "target_db", cloned.GetTargetSchemaName()) require.Equal(t, "target_table", cloned.GetTargetTableName()) require.Equal(t, "source_db.source_table", cloned.TableName.String()) @@ -93,8 +93,8 @@ func TestCloneWithRouting(t *testing.T) { require.Equal(t, "source_db", cloned.GetSchemaName()) require.Equal(t, "source_table", cloned.GetTableName()) - require.Equal(t, "source_db", cloned.GetSourceSchemaName()) - require.Equal(t, "source_table", cloned.GetSourceTableName()) + require.Equal(t, "source_db", cloned.GetOriginSchemaName()) + require.Equal(t, "source_table", cloned.GetOriginTableName()) require.Equal(t, "target_db", cloned.GetTargetSchemaName()) require.Equal(t, "target_table", cloned.GetTargetTableName()) }) From ece624627c0b6e6f6a85d71492f4e96b44b5b133 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Wed, 1 Apr 2026 13:46:18 +0800 Subject: [PATCH 06/20] remove useless --- pkg/common/event/ddl_event.go | 16 ---------------- pkg/common/event/ddl_event_test.go | 14 +++++++------- pkg/common/event/redo_test.go | 8 ++------ pkg/common/table_info.go | 10 ---------- pkg/common/table_info_test.go | 4 ---- 5 files changed, 9 insertions(+), 43 deletions(-) diff --git a/pkg/common/event/ddl_event.go b/pkg/common/event/ddl_event.go index 96cc908909..42e377dd3d 100644 --- a/pkg/common/event/ddl_event.go +++ b/pkg/common/event/ddl_event.go @@ -199,34 +199,18 @@ func (d *DDLEvent) GetSchemaName() string { return d.SchemaName } -func (d *DDLEvent) GetOriginSchemaName() string { - return d.SchemaName -} - func (d *DDLEvent) GetTableName() string { return d.TableName } -func (d *DDLEvent) GetOriginTableName() string { - return d.TableName -} - func (d *DDLEvent) GetExtraSchemaName() string { return d.ExtraSchemaName } -func (d *DDLEvent) GetOriginExtraSchemaName() string { - return d.ExtraSchemaName -} - func (d *DDLEvent) GetExtraTableName() string { return d.ExtraTableName } -func (d *DDLEvent) GetOriginExtraTableName() string { - return d.ExtraTableName -} - func (d *DDLEvent) GetTargetSchemaName() string { if d.TargetSchemaName != "" { return d.TargetSchemaName diff --git a/pkg/common/event/ddl_event_test.go b/pkg/common/event/ddl_event_test.go index 01ec7ef853..f521715e5e 100644 --- a/pkg/common/event/ddl_event_test.go +++ b/pkg/common/event/ddl_event_test.go @@ -609,8 +609,8 @@ func TestDDLEventCloneForRouting(t *testing.T) { require.Equal(t, "CREATE TABLE routed_schema.test ...", cloned.Query) require.True(t, cloned.TableInfo == newRoutedTableInfo) require.Equal(t, "routed_schema", cloned.TableInfo.TableName.TargetSchema) - require.Equal(t, original.SchemaName, cloned.GetOriginSchemaName()) - require.Equal(t, original.TableName, cloned.GetOriginTableName()) + require.Equal(t, original.SchemaName, cloned.GetSchemaName()) + require.Equal(t, original.TableName, cloned.GetTableName()) // Test cloning nil event var nilEvent *DDLEvent @@ -618,7 +618,7 @@ func TestDDLEventCloneForRouting(t *testing.T) { require.Nil(t, clonedNil) } -func TestCloneForRoutingPreservesSourceFields(t *testing.T) { +func TestCloneForRoutingPreservesOriginFields(t *testing.T) { original := &DDLEvent{ SchemaName: "source_db", TableName: "new_orders", @@ -636,10 +636,10 @@ func TestCloneForRoutingPreservesSourceFields(t *testing.T) { cloned.TargetExtraSchemaName = "target_db_v2" cloned.TargetExtraTableName = "old_orders_routed_v2" - require.Equal(t, "source_db", cloned.GetOriginSchemaName()) - require.Equal(t, "new_orders", cloned.GetOriginTableName()) - require.Equal(t, "source_db", cloned.GetOriginExtraSchemaName()) - require.Equal(t, "old_orders", cloned.GetOriginExtraTableName()) + require.Equal(t, "source_db", cloned.GetSchemaName()) + require.Equal(t, "new_orders", cloned.GetTableName()) + require.Equal(t, "source_db", cloned.GetExtraSchemaName()) + require.Equal(t, "old_orders", cloned.GetExtraTableName()) require.Equal(t, "target_db_v2", cloned.GetTargetSchemaName()) require.Equal(t, "new_orders_routed_v2", cloned.GetTargetTableName()) require.Equal(t, "target_db_v2", cloned.GetTargetExtraSchemaName()) diff --git a/pkg/common/event/redo_test.go b/pkg/common/event/redo_test.go index 3834b9f4b9..5840214d6c 100644 --- a/pkg/common/event/redo_test.go +++ b/pkg/common/event/redo_test.go @@ -21,7 +21,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestRedoDMLEventToDMLEventPreservesSourceAndTargetNames(t *testing.T) { +func TestRedoDMLEventToDMLEventPreservesOriginAndTargetNames(t *testing.T) { t.Parallel() helper := NewEventTestHelper(t) @@ -53,11 +53,9 @@ func TestRedoDMLEventToDMLEventPreservesSourceAndTargetNames(t *testing.T) { require.Equal(t, "t", decoded.TableInfo.GetTableName()) require.Equal(t, "target_db", decoded.TableInfo.GetTargetSchemaName()) require.Equal(t, "target_table", decoded.TableInfo.GetTargetTableName()) - require.Equal(t, "test", decoded.TableInfo.GetOriginSchemaName()) - require.Equal(t, "t", decoded.TableInfo.GetOriginTableName()) } -func TestRedoDDLEventToDDLEventPreservesSourceAndTargetNames(t *testing.T) { +func TestRedoDDLEventToDDLEventPreservesOriginAndTargetNames(t *testing.T) { t.Parallel() redoDDLEvent := &RedoDDLEvent{ @@ -85,8 +83,6 @@ func TestRedoDDLEventToDDLEventPreservesSourceAndTargetNames(t *testing.T) { require.Equal(t, "source_table", ddlEvent.TableInfo.GetTableName()) require.Equal(t, "target_db", ddlEvent.TableInfo.GetTargetSchemaName()) require.Equal(t, "target_table", ddlEvent.TableInfo.GetTargetTableName()) - require.Equal(t, "source_db", ddlEvent.TableInfo.GetOriginSchemaName()) - require.Equal(t, "source_table", ddlEvent.TableInfo.GetOriginTableName()) require.Equal(t, []SchemaTableName{{ SchemaName: "target_db", TableName: "target_table", diff --git a/pkg/common/table_info.go b/pkg/common/table_info.go index 0bdd7874a2..33f2680cc4 100644 --- a/pkg/common/table_info.go +++ b/pkg/common/table_info.go @@ -433,16 +433,6 @@ func (ti *TableInfo) GetTargetTableName() string { return ti.TableName.GetTargetTable() } -// GetOriginSchemaName returns the origin schema name before routing. -func (ti *TableInfo) GetOriginSchemaName() string { - return ti.TableName.GetOriginSchema() -} - -// GetOriginTableName returns the origin table name before routing. -func (ti *TableInfo) GetOriginTableName() string { - return ti.TableName.GetOriginTable() -} - // IsView checks if TableInfo is a view. func (t *TableInfo) IsView() bool { return t.View != nil diff --git a/pkg/common/table_info_test.go b/pkg/common/table_info_test.go index 1f16b5a24e..d97739fa11 100644 --- a/pkg/common/table_info_test.go +++ b/pkg/common/table_info_test.go @@ -57,8 +57,6 @@ func TestCloneWithRouting(t *testing.T) { require.Equal(t, int64(123), cloned.TableName.TableID) require.Equal(t, "source_db", cloned.GetSchemaName()) require.Equal(t, "source_table", cloned.GetTableName()) - require.Equal(t, "source_db", cloned.GetOriginSchemaName()) - require.Equal(t, "source_table", cloned.GetOriginTableName()) require.Equal(t, "target_db", cloned.GetTargetSchemaName()) require.Equal(t, "target_table", cloned.GetTargetTableName()) require.Equal(t, "source_db.source_table", cloned.TableName.String()) @@ -93,8 +91,6 @@ func TestCloneWithRouting(t *testing.T) { require.Equal(t, "source_db", cloned.GetSchemaName()) require.Equal(t, "source_table", cloned.GetTableName()) - require.Equal(t, "source_db", cloned.GetOriginSchemaName()) - require.Equal(t, "source_table", cloned.GetOriginTableName()) require.Equal(t, "target_db", cloned.GetTargetSchemaName()) require.Equal(t, "target_table", cloned.GetTargetTableName()) }) From 6e92df418332e104540c01159e949ef66e1dcd14 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Wed, 1 Apr 2026 17:56:08 +0800 Subject: [PATCH 07/20] add more code --- pkg/common/event/ddl_event.go | 135 ++++++++++++--------- pkg/common/event/ddl_event_test.go | 183 +++++++++++++++-------------- pkg/common/event/redo.go | 4 +- pkg/common/event/redo_test.go | 4 +- pkg/redo/writer/file/file_mock.go | 3 +- 5 files changed, 180 insertions(+), 149 deletions(-) diff --git a/pkg/common/event/ddl_event.go b/pkg/common/event/ddl_event.go index 42e377dd3d..1f013f02c1 100644 --- a/pkg/common/event/ddl_event.go +++ b/pkg/common/event/ddl_event.go @@ -40,25 +40,25 @@ type DDLEvent struct { // Type is the type of the DDL. Type byte `json:"type"` // SchemaID is from upstream job.SchemaID - SchemaID int64 `json:"schema_id"` + SchemaID int64 `json:"schema_id"` + // SchemaName and TableName carry the origin upstream names. SchemaName string `json:"schema_name"` TableName string `json:"table_name"` - // TargetSchemaName and TargetTableName carry routed names for sink output paths. - // They are runtime-only fields and are not serialized. - TargetSchemaName string `json:"-"` - TargetTableName string `json:"-"` - // the following two fields are just used for RenameTable, - // they are the old schema/table name of the table + // ExtraSchemaName and ExtraTableName carry the origin old names for RenameTable. ExtraSchemaName string `json:"extra_schema_name"` ExtraTableName string `json:"extra_table_name"` - // TargetExtraSchemaName and TargetExtraTableName carry routed old names for rename DDLs. + + // target related fields carry routed names for sink output paths. // They are runtime-only fields and are not serialized. - TargetExtraSchemaName string `json:"-"` - TargetExtraTableName string `json:"-"` - Query string `json:"query"` - TableInfo *common.TableInfo `json:"-"` - StartTs uint64 `json:"start_ts"` - FinishedTs uint64 `json:"finished_ts"` + targetSchemaName string `json:"-"` + targetTableName string `json:"-"` + targetExtraSchemaName string `json:"-"` + targetExtraTableName string `json:"-"` + + Query string `json:"query"` + TableInfo *common.TableInfo `json:"-"` + StartTs uint64 `json:"start_ts"` + FinishedTs uint64 `json:"finished_ts"` // The seq of the event. It is set by event service. Seq uint64 `json:"seq"` // The epoch of the event. It is set by event service. @@ -212,29 +212,29 @@ func (d *DDLEvent) GetExtraTableName() string { } func (d *DDLEvent) GetTargetSchemaName() string { - if d.TargetSchemaName != "" { - return d.TargetSchemaName + if d.targetSchemaName != "" { + return d.targetSchemaName } return d.SchemaName } func (d *DDLEvent) GetTargetTableName() string { - if d.TargetTableName != "" { - return d.TargetTableName + if d.targetTableName != "" { + return d.targetTableName } return d.TableName } func (d *DDLEvent) GetTargetExtraSchemaName() string { - if d.TargetExtraSchemaName != "" { - return d.TargetExtraSchemaName + if d.targetExtraSchemaName != "" { + return d.targetExtraSchemaName } return d.ExtraSchemaName } func (d *DDLEvent) GetTargetExtraTableName() string { - if d.TargetExtraTableName != "" { - return d.TargetExtraTableName + if d.targetExtraTableName != "" { + return d.targetExtraTableName } return d.ExtraTableName } @@ -272,8 +272,8 @@ func (d *DDLEvent) GetEvents() []*DDLEvent { Type: byte(t), SchemaName: info.GetSchemaName(), TableName: info.GetTableName(), - TargetSchemaName: info.GetTargetSchemaName(), - TargetTableName: info.GetTargetTableName(), + targetSchemaName: info.GetTargetSchemaName(), + targetTableName: info.GetTargetTableName(), TableInfo: info, Query: queries[i], StartTs: d.StartTs, @@ -283,8 +283,8 @@ func (d *DDLEvent) GetEvents() []*DDLEvent { event.ExtraSchemaName = d.TableNameChange.DropName[i].SchemaName event.ExtraTableName = d.TableNameChange.DropName[i].TableName targetExtraSchemaName, targetExtraTableName := extractRenameTargetExtraFromQuery(queries[i]) - event.TargetExtraSchemaName = targetExtraSchemaName - event.TargetExtraTableName = targetExtraTableName + event.targetExtraSchemaName = targetExtraSchemaName + event.targetExtraTableName = targetExtraTableName } events = append(events, event) } @@ -542,43 +542,64 @@ func (t *DDLEvent) IsPaused() bool { return false } -// CloneForRouting creates a shallow copy of the DDLEvent that can safely be mutated -// for table-route purposes without affecting the original event. -// -// The clone shares most read-only fields with the original. Slice fields that can be -// replaced independently downstream are copied so routing can update them without -// mutating shared state. -func (d *DDLEvent) CloneForRouting() *DDLEvent { +// NewRoutedDDLEvent builds a routed DDL event from the origin event and final routed fields. +func NewRoutedDDLEvent( + d *DDLEvent, + query string, + targetSchemaName, targetTableName string, + targetExtraSchemaName, targetExtraTableName string, + tableInfo *common.TableInfo, + multipleTableInfos []*common.TableInfo, + blockedTableNames []SchemaTableName, +) *DDLEvent { if d == nil { return nil } - // Create shallow copy - clone := *d - - // PostTxnFlushed needs its own backing array to prevent potential races. - // Currently, DDL events arrive with nil PostTxnFlushed (callbacks are added - // downstream by basic_dispatcher.go), so append(nil, f) naturally creates a - // fresh slice. However, we make an explicit copy here for future-proofing: - // if any code path later adds callbacks before cloning, sharing the backing - // array could cause nondeterministic callback visibility or data races. - if d.PostTxnFlushed != nil { - clone.PostTxnFlushed = make([]func(), len(d.PostTxnFlushed)) - copy(clone.PostTxnFlushed, d.PostTxnFlushed) - } - - // MultipleTableInfos needs a new slice so each dispatcher can independently - // apply routing to its elements without affecting others - if d.MultipleTableInfos != nil { - clone.MultipleTableInfos = make([]*common.TableInfo, len(d.MultipleTableInfos)) - copy(clone.MultipleTableInfos, d.MultipleTableInfos) - } - - if d.BlockedTableNames != nil { - clone.BlockedTableNames = append([]SchemaTableName(nil), d.BlockedTableNames...) + return &DDLEvent{ + Version: d.Version, + DispatcherID: d.DispatcherID, + Type: d.Type, + SchemaID: d.SchemaID, + SchemaName: d.SchemaName, + TableName: d.TableName, + ExtraSchemaName: d.ExtraSchemaName, + ExtraTableName: d.ExtraTableName, + targetSchemaName: targetSchemaName, + targetTableName: targetTableName, + targetExtraSchemaName: targetExtraSchemaName, + targetExtraTableName: targetExtraTableName, + Query: query, + TableInfo: tableInfo, + StartTs: d.StartTs, + FinishedTs: d.FinishedTs, + Seq: d.Seq, + Epoch: d.Epoch, + MultipleTableInfos: multipleTableInfos, + BlockedTables: d.BlockedTables, + BlockedTableNames: blockedTableNames, + NeedDroppedTables: d.NeedDroppedTables, + NeedAddedTables: d.NeedAddedTables, + UpdatedSchemas: d.UpdatedSchemas, + TableNameChange: d.TableNameChange, + TiDBOnly: d.TiDBOnly, + BDRMode: d.BDRMode, + Err: d.Err, + PostTxnFlushed: clonePostTxnFlushed(d.PostTxnFlushed), + eventSize: d.eventSize, + IsBootstrap: d.IsBootstrap, + NotSync: d.NotSync, + } +} + +func clonePostTxnFlushed(postTxnFlushed []func()) []func() { + if postTxnFlushed == nil { + return nil } - return &clone + cloned := make([]func(), len(postTxnFlushed)) + copy(cloned, postTxnFlushed) + return cloned } func (t *DDLEvent) Len() int32 { diff --git a/pkg/common/event/ddl_event_test.go b/pkg/common/event/ddl_event_test.go index f521715e5e..9447acefe6 100644 --- a/pkg/common/event/ddl_event_test.go +++ b/pkg/common/event/ddl_event_test.go @@ -506,9 +506,9 @@ INSERT INTO test VALUES (1); } } -// TestDDLEventCloneForRouting tests the CloneForRouting method to ensure it properly -// clones DDL events to avoid race conditions between multiple dispatchers -func TestDDLEventCloneForRouting(t *testing.T) { +// TestNewRoutedDDLEvent ensures routed DDL construction preserves the origin event +// while producing an independent routed event for downstream use. +func TestNewRoutedDDLEvent(t *testing.T) { helper := NewEventTestHelper(t) defer helper.Close() @@ -546,104 +546,113 @@ func TestDDLEventCloneForRouting(t *testing.T) { BDRMode: "test-mode", } - // Clone the event - cloned := original.CloneForRouting() - require.NotNil(t, cloned) - - // Verify that cloned is a separate object - require.False(t, original == cloned, "cloned event should be a different object") - - // Verify that immutable fields are shared (shallow copy) - require.Equal(t, original.Version, cloned.Version) - require.Equal(t, original.DispatcherID, cloned.DispatcherID) - require.Equal(t, original.Type, cloned.Type) - require.Equal(t, original.SchemaID, cloned.SchemaID) - require.Equal(t, original.SchemaName, cloned.SchemaName) - require.Equal(t, original.TableName, cloned.TableName) - require.Equal(t, original.Query, cloned.Query) - require.Equal(t, original.FinishedTs, cloned.FinishedTs) - require.Equal(t, original.Seq, cloned.Seq) - require.Equal(t, original.Epoch, cloned.Epoch) - require.Equal(t, original.TiDBOnly, cloned.TiDBOnly) - require.Equal(t, original.BDRMode, cloned.BDRMode) - - // Verify that TableInfo pointer is shared initially - require.True(t, original.TableInfo == cloned.TableInfo, "TableInfo should be shared initially") - - // Verify that MultipleTableInfos is a new slice (but points to same TableInfo objects initially) - require.False(t, &original.MultipleTableInfos[0] == &cloned.MultipleTableInfos[0], "MultipleTableInfos should be a new slice") - require.True(t, original.MultipleTableInfos[0] == cloned.MultipleTableInfos[0], "MultipleTableInfos elements should be shared initially") - require.True(t, original.MultipleTableInfos[1] == cloned.MultipleTableInfos[1], "MultipleTableInfos elements should be shared initially") + newRoutedTableInfo := originalTableInfo.CloneWithRouting("routed_schema", "test") + routedMultipleTableInfos := []*common.TableInfo{ + multipleTableInfo1.CloneWithRouting("routed_schema1", "table1"), + multipleTableInfo2.CloneWithRouting("routed_schema2", "table2"), + } + + routed := NewRoutedDDLEvent( + original, + "CREATE TABLE routed_schema.test ...", + "routed_schema", + "", + "", + "", + newRoutedTableInfo, + routedMultipleTableInfos, + original.BlockedTableNames, + ) + require.NotNil(t, routed) + + // Verify that the routed event is a separate object. + require.False(t, original == routed, "routed event should be a different object") + + // Verify that non-routing fields are copied as-is. + require.Equal(t, original.Version, routed.Version) + require.Equal(t, original.DispatcherID, routed.DispatcherID) + require.Equal(t, original.Type, routed.Type) + require.Equal(t, original.SchemaID, routed.SchemaID) + require.Equal(t, original.SchemaName, routed.SchemaName) + require.Equal(t, original.TableName, routed.TableName) + require.Equal(t, original.FinishedTs, routed.FinishedTs) + require.Equal(t, original.Seq, routed.Seq) + require.Equal(t, original.Epoch, routed.Epoch) + require.Equal(t, original.TiDBOnly, routed.TiDBOnly) + require.Equal(t, original.BDRMode, routed.BDRMode) + + // Verify that MultipleTableInfos is a new slice so later mutations remain isolated. + require.False(t, &original.MultipleTableInfos[0] == &routed.MultipleTableInfos[0], "MultipleTableInfos should be a new slice") // Verify that PostTxnFlushed is an independent copy (not shared) // This is defensive: currently DDL events arrive with nil PostTxnFlushed, - // but we copy it to prevent races if callbacks are ever added before cloning. - require.NotNil(t, cloned.PostTxnFlushed) - require.Equal(t, 2, len(cloned.PostTxnFlushed), "PostTxnFlushed should have same length as original") + // but we copy it to prevent races if callbacks are ever added before building the routed event. + require.NotNil(t, routed.PostTxnFlushed) + require.Equal(t, 2, len(routed.PostTxnFlushed), "PostTxnFlushed should have same length as original") require.Equal(t, 2, len(original.PostTxnFlushed), "Original PostTxnFlushed should remain unchanged") - // Verify independent backing arrays - appending to clone should not affect original - require.NotEqual(t, &original.PostTxnFlushed[0], &cloned.PostTxnFlushed[0], "PostTxnFlushed should have independent backing arrays") + // Verify independent backing arrays. + require.NotEqual(t, &original.PostTxnFlushed[0], &routed.PostTxnFlushed[0], "PostTxnFlushed should have independent backing arrays") - // Verify that appending to cloned PostTxnFlushed doesn't affect original - cloned.AddPostFlushFunc(func() {}) - require.Equal(t, 3, len(cloned.PostTxnFlushed), "Clone should have appended callback") - require.Equal(t, 2, len(original.PostTxnFlushed), "Original should be unaffected by clone's append") + // Verify that appending to the routed event doesn't affect the original. + routed.AddPostFlushFunc(func() {}) + require.Equal(t, 3, len(routed.PostTxnFlushed), "Routed event should have appended callback") + require.Equal(t, 2, len(original.PostTxnFlushed), "Original should be unaffected by routed event append") - // Now simulate what happens during routing: mutate the cloned event - cloned.SchemaName = "routed_schema" - cloned.Query = "CREATE TABLE routed_schema.test ..." - newRoutedTableInfo := originalTableInfo.CloneWithRouting("routed_schema", "test") - cloned.TableInfo = newRoutedTableInfo - cloned.MultipleTableInfos[0] = multipleTableInfo1.CloneWithRouting("routed_schema1", "table1") - cloned.MultipleTableInfos[1] = multipleTableInfo2.CloneWithRouting("routed_schema2", "table2") - - // Verify that mutations to cloned event don't affect the original + // Verify that routed state doesn't affect the original. require.Equal(t, ddlJob.SchemaName, original.SchemaName, "Original SchemaName should be unchanged") require.Equal(t, ddlJob.Query, original.Query, "Original Query should be unchanged") require.True(t, original.TableInfo == originalTableInfo, "Original TableInfo should be unchanged") require.True(t, original.MultipleTableInfos[0] == multipleTableInfo1, "Original MultipleTableInfos[0] should be unchanged") require.True(t, original.MultipleTableInfos[1] == multipleTableInfo2, "Original MultipleTableInfos[1] should be unchanged") - // Verify that cloned event has the mutations - require.Equal(t, "routed_schema", cloned.TargetSchemaName) - require.Equal(t, "CREATE TABLE routed_schema.test ...", cloned.Query) - require.True(t, cloned.TableInfo == newRoutedTableInfo) - require.Equal(t, "routed_schema", cloned.TableInfo.TableName.TargetSchema) - require.Equal(t, original.SchemaName, cloned.GetSchemaName()) - require.Equal(t, original.TableName, cloned.GetTableName()) - - // Test cloning nil event + // Verify that the routed event has the routed state. + require.Equal(t, "routed_schema", routed.GetTargetSchemaName()) + require.Equal(t, "CREATE TABLE routed_schema.test ...", routed.Query) + require.True(t, routed.TableInfo == newRoutedTableInfo) + require.Equal(t, "routed_schema", routed.TableInfo.TableName.TargetSchema) + require.Equal(t, original.SchemaName, routed.GetSchemaName()) + require.Equal(t, original.TableName, routed.GetTableName()) + require.True(t, routed.MultipleTableInfos[0] == routedMultipleTableInfos[0]) + require.True(t, routed.MultipleTableInfos[1] == routedMultipleTableInfos[1]) + + // Test nil origin event. var nilEvent *DDLEvent - clonedNil := nilEvent.CloneForRouting() - require.Nil(t, clonedNil) + routedNil := NewRoutedDDLEvent(nilEvent, "", "", "", "", "", nil, nil, nil) + require.Nil(t, routedNil) } -func TestCloneForRoutingPreservesOriginFields(t *testing.T) { +func TestNewRoutedDDLEventPreservesSourceFields(t *testing.T) { original := &DDLEvent{ SchemaName: "source_db", TableName: "new_orders", ExtraSchemaName: "source_db", ExtraTableName: "old_orders", - TargetSchemaName: "target_db", - TargetTableName: "new_orders_routed", - TargetExtraSchemaName: "target_db", - TargetExtraTableName: "old_orders_routed", + targetSchemaName: "target_db", + targetTableName: "new_orders_routed", + targetExtraSchemaName: "target_db", + targetExtraTableName: "old_orders_routed", } - cloned := original.CloneForRouting() - cloned.TargetSchemaName = "target_db_v2" - cloned.TargetTableName = "new_orders_routed_v2" - cloned.TargetExtraSchemaName = "target_db_v2" - cloned.TargetExtraTableName = "old_orders_routed_v2" - - require.Equal(t, "source_db", cloned.GetSchemaName()) - require.Equal(t, "new_orders", cloned.GetTableName()) - require.Equal(t, "source_db", cloned.GetExtraSchemaName()) - require.Equal(t, "old_orders", cloned.GetExtraTableName()) - require.Equal(t, "target_db_v2", cloned.GetTargetSchemaName()) - require.Equal(t, "new_orders_routed_v2", cloned.GetTargetTableName()) - require.Equal(t, "target_db_v2", cloned.GetTargetExtraSchemaName()) - require.Equal(t, "old_orders_routed_v2", cloned.GetTargetExtraTableName()) + routed := NewRoutedDDLEvent( + original, + original.Query, + "target_db_v2", + "new_orders_routed_v2", + "target_db_v2", + "old_orders_routed_v2", + original.TableInfo, + original.MultipleTableInfos, + original.BlockedTableNames, + ) + + require.Equal(t, "source_db", routed.GetSchemaName()) + require.Equal(t, "new_orders", routed.GetTableName()) + require.Equal(t, "source_db", routed.GetExtraSchemaName()) + require.Equal(t, "old_orders", routed.GetExtraTableName()) + require.Equal(t, "target_db_v2", routed.GetTargetSchemaName()) + require.Equal(t, "new_orders_routed_v2", routed.GetTargetTableName()) + require.Equal(t, "target_db_v2", routed.GetTargetExtraSchemaName()) + require.Equal(t, "old_orders_routed_v2", routed.GetTargetExtraTableName()) } func TestGetEventsForRenameTablesPreservesSourceAndTargetNames(t *testing.T) { @@ -678,19 +687,19 @@ func TestGetEventsForRenameTablesPreservesSourceAndTargetNames(t *testing.T) { require.Equal(t, "new_db1", events[0].SchemaName) require.Equal(t, "new_table1", events[0].TableName) - require.Equal(t, "new_target_db1", events[0].TargetSchemaName) - require.Equal(t, "new_target_table1", events[0].TargetTableName) + require.Equal(t, "new_target_db1", events[0].GetTargetSchemaName()) + require.Equal(t, "new_target_table1", events[0].GetTargetTableName()) require.Equal(t, "old_db1", events[0].ExtraSchemaName) require.Equal(t, "old_table1", events[0].ExtraTableName) - require.Equal(t, "old_target_db1", events[0].TargetExtraSchemaName) - require.Equal(t, "old_target_table1", events[0].TargetExtraTableName) + require.Equal(t, "old_target_db1", events[0].GetTargetExtraSchemaName()) + require.Equal(t, "old_target_table1", events[0].GetTargetExtraTableName()) require.Equal(t, "new_db2", events[1].SchemaName) require.Equal(t, "new_table2", events[1].TableName) - require.Equal(t, "new_target_db2", events[1].TargetSchemaName) - require.Equal(t, "new_target_table2", events[1].TargetTableName) + require.Equal(t, "new_target_db2", events[1].GetTargetSchemaName()) + require.Equal(t, "new_target_table2", events[1].GetTargetTableName()) require.Equal(t, "old_db2", events[1].ExtraSchemaName) require.Equal(t, "old_table2", events[1].ExtraTableName) - require.Equal(t, "old_target_db2", events[1].TargetExtraSchemaName) - require.Equal(t, "old_target_table2", events[1].TargetExtraTableName) + require.Equal(t, "old_target_db2", events[1].GetTargetExtraSchemaName()) + require.Equal(t, "old_target_table2", events[1].GetTargetExtraTableName()) } diff --git a/pkg/common/event/redo.go b/pkg/common/event/redo.go index de051cc5a5..31509c7e54 100644 --- a/pkg/common/event/redo.go +++ b/pkg/common/event/redo.go @@ -389,8 +389,8 @@ func (r *RedoDDLEvent) ToDDLEvent() *DDLEvent { Type: r.Type, SchemaName: sourceSchemaName, TableName: sourceTableName, - TargetSchemaName: targetSchemaName, - TargetTableName: targetTableName, + targetSchemaName: targetSchemaName, + targetTableName: targetTableName, FinishedTs: r.DDL.CommitTs, StartTs: r.DDL.StartTs, BlockedTables: blockedTables, diff --git a/pkg/common/event/redo_test.go b/pkg/common/event/redo_test.go index 5840214d6c..f9a56a75f7 100644 --- a/pkg/common/event/redo_test.go +++ b/pkg/common/event/redo_test.go @@ -76,8 +76,8 @@ func TestRedoDDLEventToDDLEventPreservesOriginAndTargetNames(t *testing.T) { ddlEvent := redoDDLEvent.ToDDLEvent() require.Equal(t, "source_db", ddlEvent.SchemaName) require.Equal(t, "source_table", ddlEvent.TableName) - require.Equal(t, "target_db", ddlEvent.TargetSchemaName) - require.Equal(t, "target_table", ddlEvent.TargetTableName) + require.Equal(t, "target_db", ddlEvent.GetTargetSchemaName()) + require.Equal(t, "target_table", ddlEvent.GetTargetTableName()) require.Equal(t, "target_db", ddlEvent.GetDDLSchemaName()) require.Equal(t, "source_db", ddlEvent.TableInfo.GetSchemaName()) require.Equal(t, "source_table", ddlEvent.TableInfo.GetTableName()) diff --git a/pkg/redo/writer/file/file_mock.go b/pkg/redo/writer/file/file_mock.go index b4305d52cd..40679247ce 100644 --- a/pkg/redo/writer/file/file_mock.go +++ b/pkg/redo/writer/file/file_mock.go @@ -6,8 +6,9 @@ import ( context "context" event "github.com/pingcap/ticdc/pkg/common/event" - writer "github.com/pingcap/ticdc/pkg/redo/writer" mock "github.com/stretchr/testify/mock" + + writer "github.com/pingcap/ticdc/pkg/redo/writer" ) // mockFileWriter is an autogenerated mock type for the fileWriter type From ae0f4eb2e5ecdac9b87814c43b189024c2b14882 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Wed, 1 Apr 2026 18:22:22 +0800 Subject: [PATCH 08/20] add more code --- pkg/common/event/redo_gen.go | 6 +++--- pkg/common/table_info.go | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/common/event/redo_gen.go b/pkg/common/event/redo_gen.go index 95d197304d..1e0226c78c 100644 --- a/pkg/common/event/redo_gen.go +++ b/pkg/common/event/redo_gen.go @@ -3,7 +3,7 @@ package event import ( - "github.com/pingcap/ticdc/pkg/common" + commonType "github.com/pingcap/ticdc/pkg/common" "github.com/tinylib/msgp/msgp" ) @@ -475,7 +475,7 @@ func (z *DMLEventInRedoLog) DecodeMsg(dc *msgp.Reader) (err error) { z.Table = nil } else { if z.Table == nil { - z.Table = new(common.TableName) + z.Table = new(commonType.TableName) } err = z.Table.DecodeMsg(dc) if err != nil { @@ -803,7 +803,7 @@ func (z *DMLEventInRedoLog) UnmarshalMsg(bts []byte) (o []byte, err error) { z.Table = nil } else { if z.Table == nil { - z.Table = new(common.TableName) + z.Table = new(commonType.TableName) } bts, err = z.Table.UnmarshalMsg(bts) if err != nil { diff --git a/pkg/common/table_info.go b/pkg/common/table_info.go index 33f2680cc4..e224b6c1e8 100644 --- a/pkg/common/table_info.go +++ b/pkg/common/table_info.go @@ -392,12 +392,12 @@ func (ti *TableInfo) MustGetColumnOffsetByID(id int64) int { return offset } -// GetSchemaName returns the source schema name carried by this TableInfo. +// GetSchemaName returns the origin schema name carried by this TableInfo. func (ti *TableInfo) GetSchemaName() string { return ti.TableName.GetOriginSchema() } -// GetTableName returns the source table name carried by this TableInfo. +// GetTableName returns the origin table name carried by this TableInfo. func (ti *TableInfo) GetTableName() string { return ti.TableName.GetOriginTable() } @@ -406,12 +406,12 @@ func (ti *TableInfo) GetTableNameCIStr() ast.CIStr { return ast.NewCIStr(ti.GetTableName()) } -// GetSchemaNamePtr returns the pointer to the source schema name. +// GetSchemaNamePtr returns the pointer to the origin schema name. func (ti *TableInfo) GetSchemaNamePtr() *string { return &ti.TableName.Schema } -// GetTableNamePtr returns the pointer to the source table name. +// GetTableNamePtr returns the pointer to the origin table name. func (ti *TableInfo) GetTableNamePtr() *string { return &ti.TableName.Table } From 249a7760c90737bfb3bbacedeb5b398f8762cf28 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Wed, 1 Apr 2026 22:50:08 +0800 Subject: [PATCH 09/20] fix some code --- pkg/common/event/redo.go | 4 ++-- pkg/common/event/redo_gen.go | 6 +++--- pkg/common/table_info.go | 4 ++-- pkg/common/table_info_test.go | 4 ++-- pkg/common/table_name.go | 29 +++++++---------------------- 5 files changed, 16 insertions(+), 31 deletions(-) diff --git a/pkg/common/event/redo.go b/pkg/common/event/redo.go index 31509c7e54..1658cf5625 100644 --- a/pkg/common/event/redo.go +++ b/pkg/common/event/redo.go @@ -373,8 +373,8 @@ func (r *RedoDMLEvent) ToDMLEvent() *DMLEvent { func (r *RedoDDLEvent) ToDDLEvent() *DDLEvent { blockedTables := r.DDL.BlockedTables blockedTableNames := r.DDL.BlockedTableNames - sourceSchemaName := r.TableName.GetOriginSchema() - sourceTableName := r.TableName.GetOriginTable() + sourceSchemaName := r.TableName.GetSchema() + sourceTableName := r.TableName.GetTable() targetSchemaName := r.TableName.GetTargetSchema() targetTableName := r.TableName.GetTargetTable() if blockedTables == nil { diff --git a/pkg/common/event/redo_gen.go b/pkg/common/event/redo_gen.go index 1e0226c78c..95d197304d 100644 --- a/pkg/common/event/redo_gen.go +++ b/pkg/common/event/redo_gen.go @@ -3,7 +3,7 @@ package event import ( - commonType "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/common" "github.com/tinylib/msgp/msgp" ) @@ -475,7 +475,7 @@ func (z *DMLEventInRedoLog) DecodeMsg(dc *msgp.Reader) (err error) { z.Table = nil } else { if z.Table == nil { - z.Table = new(commonType.TableName) + z.Table = new(common.TableName) } err = z.Table.DecodeMsg(dc) if err != nil { @@ -803,7 +803,7 @@ func (z *DMLEventInRedoLog) UnmarshalMsg(bts []byte) (o []byte, err error) { z.Table = nil } else { if z.Table == nil { - z.Table = new(commonType.TableName) + z.Table = new(common.TableName) } bts, err = z.Table.UnmarshalMsg(bts) if err != nil { diff --git a/pkg/common/table_info.go b/pkg/common/table_info.go index e224b6c1e8..e2c9143f4e 100644 --- a/pkg/common/table_info.go +++ b/pkg/common/table_info.go @@ -394,12 +394,12 @@ func (ti *TableInfo) MustGetColumnOffsetByID(id int64) int { // GetSchemaName returns the origin schema name carried by this TableInfo. func (ti *TableInfo) GetSchemaName() string { - return ti.TableName.GetOriginSchema() + return ti.TableName.GetSchema() } // GetTableName returns the origin table name carried by this TableInfo. func (ti *TableInfo) GetTableName() string { - return ti.TableName.GetOriginTable() + return ti.TableName.GetTable() } func (ti *TableInfo) GetTableNameCIStr() ast.CIStr { diff --git a/pkg/common/table_info_test.go b/pkg/common/table_info_test.go index d97739fa11..2a1054d7e6 100644 --- a/pkg/common/table_info_test.go +++ b/pkg/common/table_info_test.go @@ -61,8 +61,8 @@ func TestCloneWithRouting(t *testing.T) { require.Equal(t, "target_table", cloned.GetTargetTableName()) require.Equal(t, "source_db.source_table", cloned.TableName.String()) require.Equal(t, "`source_db`.`source_table`", cloned.TableName.QuoteString()) - require.Equal(t, "source_db.source_table", cloned.TableName.OriginString()) - require.Equal(t, "`source_db`.`source_table`", cloned.TableName.QuoteOriginString()) + require.Equal(t, "source_db.source_table", cloned.TableName.String()) + require.Equal(t, "`source_db`.`source_table`", cloned.TableName.QuoteString()) require.Same(t, &cloned.TableName.Schema, cloned.GetSchemaNamePtr()) require.Same(t, &cloned.TableName.Table, cloned.GetTableNamePtr()) diff --git a/pkg/common/table_name.go b/pkg/common/table_name.go index 728d80b3cb..976ba38160 100644 --- a/pkg/common/table_name.go +++ b/pkg/common/table_name.go @@ -34,31 +34,21 @@ type TableName struct { // String implements fmt.Stringer interface. func (t TableName) String() string { - return t.OriginString() + return fmt.Sprintf("%s.%s", t.GetSchema(), t.GetTable()) } -// OriginString returns the source schema.table string. -func (t TableName) OriginString() string { - return fmt.Sprintf("%s.%s", t.GetOriginSchema(), t.GetOriginTable()) -} - -// QuoteString returns quoted full canonical table name. +// QuoteString returns quoted full original table name. func (t TableName) QuoteString() string { - return t.QuoteOriginString() -} - -// QuoteOriginString returns quoted full source table name. -func (t TableName) QuoteOriginString() string { - return QuoteSchema(t.GetOriginSchema(), t.GetOriginTable()) + return QuoteSchema(t.GetSchema(), t.GetTable()) } -// GetOriginSchema returns the source schema name. -func (t *TableName) GetOriginSchema() string { +// GetSchema returns the original schema name. +func (t *TableName) GetSchema() string { return t.Schema } -// GetOriginTable returns the source table name. -func (t *TableName) GetOriginTable() string { +// GetTable returns the original table name. +func (t *TableName) GetTable() string { return t.Table } @@ -80,11 +70,6 @@ func (t *TableName) GetTargetTable() string { return t.Table } -// TargetString returns the target schema.table string for routing. -func (t TableName) TargetString() string { - return fmt.Sprintf("%s.%s", t.GetTargetSchema(), t.GetTargetTable()) -} - // QuoteTargetString returns quoted full target table name for routing. func (t TableName) QuoteTargetString() string { return QuoteSchema(t.GetTargetSchema(), t.GetTargetTable()) From 2dc72bf6748ac54ff7ecf7ff12a3a4fb35fd1c71 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Wed, 1 Apr 2026 23:07:21 +0800 Subject: [PATCH 10/20] update comment about the ddl event --- pkg/common/event/ddl_event.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/event/ddl_event.go b/pkg/common/event/ddl_event.go index 1f013f02c1..12a2e6ae5b 100644 --- a/pkg/common/event/ddl_event.go +++ b/pkg/common/event/ddl_event.go @@ -49,7 +49,7 @@ type DDLEvent struct { ExtraTableName string `json:"extra_table_name"` // target related fields carry routed names for sink output paths. - // They are runtime-only fields and are not serialized. + // They are set after the unmarshal, so no need to be serialized. targetSchemaName string `json:"-"` targetTableName string `json:"-"` targetExtraSchemaName string `json:"-"` From a411da26cb8bd48ac4865da780e3c10c000ad8b9 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Thu, 2 Apr 2026 11:38:13 +0800 Subject: [PATCH 11/20] fix check --- pkg/redo/writer/file/file_mock.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/redo/writer/file/file_mock.go b/pkg/redo/writer/file/file_mock.go index 40679247ce..b4305d52cd 100644 --- a/pkg/redo/writer/file/file_mock.go +++ b/pkg/redo/writer/file/file_mock.go @@ -6,9 +6,8 @@ import ( context "context" event "github.com/pingcap/ticdc/pkg/common/event" - mock "github.com/stretchr/testify/mock" - writer "github.com/pingcap/ticdc/pkg/redo/writer" + mock "github.com/stretchr/testify/mock" ) // mockFileWriter is an autogenerated mock type for the fileWriter type From 0a7ea8c1a313b867b2e53151b7105ed572c8ae06 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Thu, 2 Apr 2026 12:08:10 +0800 Subject: [PATCH 12/20] tiny adjust code --- pkg/common/event/ddl_event.go | 16 ++++++---------- pkg/common/event/redo_test.go | 1 - pkg/common/table_name.go | 4 ++-- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/pkg/common/event/ddl_event.go b/pkg/common/event/ddl_event.go index 12a2e6ae5b..1d3390fbb2 100644 --- a/pkg/common/event/ddl_event.go +++ b/pkg/common/event/ddl_event.go @@ -41,14 +41,17 @@ type DDLEvent struct { Type byte `json:"type"` // SchemaID is from upstream job.SchemaID SchemaID int64 `json:"schema_id"` - // SchemaName and TableName carry the origin upstream names. + + // SchemaName and TableName carry the origin names. SchemaName string `json:"schema_name"` TableName string `json:"table_name"` - // ExtraSchemaName and ExtraTableName carry the origin old names for RenameTable. + + // the following two fields are just used for RenameTable, + // they are the old schema/table name of the table ExtraSchemaName string `json:"extra_schema_name"` ExtraTableName string `json:"extra_table_name"` - // target related fields carry routed names for sink output paths. + // target related fields carry routed names. // They are set after the unmarshal, so no need to be serialized. targetSchemaName string `json:"-"` targetTableName string `json:"-"` @@ -355,13 +358,6 @@ func (e *DDLEvent) GetDDLQuery() string { return e.Query } -func (e *DDLEvent) GetDDLSchemaName() string { - if e == nil { - return "" - } - return e.GetTargetSchemaName() -} - func (e *DDLEvent) GetDDLType() model.ActionType { return model.ActionType(e.Type) } diff --git a/pkg/common/event/redo_test.go b/pkg/common/event/redo_test.go index f9a56a75f7..4a45f2e1e0 100644 --- a/pkg/common/event/redo_test.go +++ b/pkg/common/event/redo_test.go @@ -78,7 +78,6 @@ func TestRedoDDLEventToDDLEventPreservesOriginAndTargetNames(t *testing.T) { require.Equal(t, "source_table", ddlEvent.TableName) require.Equal(t, "target_db", ddlEvent.GetTargetSchemaName()) require.Equal(t, "target_table", ddlEvent.GetTargetTableName()) - require.Equal(t, "target_db", ddlEvent.GetDDLSchemaName()) require.Equal(t, "source_db", ddlEvent.TableInfo.GetSchemaName()) require.Equal(t, "source_table", ddlEvent.TableInfo.GetTableName()) require.Equal(t, "target_db", ddlEvent.TableInfo.GetTargetSchemaName()) diff --git a/pkg/common/table_name.go b/pkg/common/table_name.go index 976ba38160..01b0b06efc 100644 --- a/pkg/common/table_name.go +++ b/pkg/common/table_name.go @@ -34,12 +34,12 @@ type TableName struct { // String implements fmt.Stringer interface. func (t TableName) String() string { - return fmt.Sprintf("%s.%s", t.GetSchema(), t.GetTable()) + return fmt.Sprintf("%s.%s", t.Schema, t.Table) } // QuoteString returns quoted full original table name. func (t TableName) QuoteString() string { - return QuoteSchema(t.GetSchema(), t.GetTable()) + return QuoteSchema(t.Schema, t.Table) } // GetSchema returns the original schema name. From 73f51e474da1ce4b65ebee1a3c83ec0b287392ae Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Thu, 2 Apr 2026 14:27:50 +0800 Subject: [PATCH 13/20] fix redo --- pkg/common/event/redo.go | 33 ++++++---------- pkg/common/event/redo_test.go | 74 +++++++++++++++++------------------ 2 files changed, 50 insertions(+), 57 deletions(-) diff --git a/pkg/common/event/redo.go b/pkg/common/event/redo.go index 1658cf5625..54bc745467 100644 --- a/pkg/common/event/redo.go +++ b/pkg/common/event/redo.go @@ -94,7 +94,7 @@ type RedoColumn struct { // RedoColumnValue stores Column change type RedoColumnValue struct { // Fields from Column and can't be marshaled directly in Column. - Value interface{} `msg:"column"` + Value any `msg:"column"` // msgp transforms empty byte slice into nil, PTAL msgp#247. ValueIsEmptyBytes bool `msg:"value-is-empty-bytes"` Flag uint64 `msg:"flag"` @@ -143,12 +143,10 @@ func (r *RedoRowEvent) ToRedoLog() *RedoLog { } if r.TableInfo != nil { redoLog.RedoRow.Row.Table = &common.TableName{ - Schema: r.TableInfo.TableName.Schema, - Table: r.TableInfo.TableName.Table, - TableID: r.PhysicalTableID, - IsPartition: r.TableInfo.TableName.IsPartition, - TargetSchema: r.TableInfo.TableName.TargetSchema, - TargetTable: r.TableInfo.TableName.TargetTable, + Schema: r.TableInfo.GetTargetSchemaName(), + Table: r.TableInfo.GetTargetTableName(), + TableID: r.PhysicalTableID, + IsPartition: r.TableInfo.TableName.IsPartition, } redoLog.RedoRow.Row.IndexColumns = getIndexColumns(r.TableInfo) @@ -221,7 +219,12 @@ func (d *DDLEvent) ToRedoLog() *RedoLog { Type: RedoLogTypeDDL, } if d.TableInfo != nil { - redoLog.RedoDDL.TableName = d.TableInfo.TableName + redoLog.RedoDDL.TableName = common.TableName{ + Schema: d.TableInfo.GetTargetSchemaName(), + Table: d.TableInfo.GetTargetTableName(), + TableID: d.TableInfo.TableName.TableID, + IsPartition: d.TableInfo.TableName.IsPartition, + } } return redoLog @@ -329,18 +332,8 @@ func (r *RedoDMLEvent) ToDMLEvent() *DMLEvent { indexInfo.Primary = isPrimary tidbTableInfo.Indices = append(tidbTableInfo.Indices, indexInfo) } - tableInfo := common.NewTableInfo4Decoder(r.Row.Table.Schema, tidbTableInfo) - // Restore routing info from redo log (TargetSchema/TargetTable for table routing). - // We must use CloneWithRouting because NewTableInfo4Decoder already called InitPrivateFields() - // which pre-computed SQL statements using the source schema/table. CloneWithRouting creates - // a new TableInfo with routing applied and uninitialized preSQLs that will be computed - // correctly when InitPrivateFields() is called. - if r.Row.Table.TargetSchema != "" || r.Row.Table.TargetTable != "" { - tableInfo = tableInfo.CloneWithRouting(r.Row.Table.TargetSchema, r.Row.Table.TargetTable) - tableInfo.InitPrivateFields() - } event := &DMLEvent{ - TableInfo: tableInfo, + TableInfo: common.NewTableInfo4Decoder(r.Row.Table.Schema, tidbTableInfo), CommitTs: r.Row.CommitTs, StartTs: r.Row.StartTs, Length: 1, @@ -475,7 +468,7 @@ func collectAllColumnsValue(data []RedoColumnValue, columns []*timodel.ColumnInf } } -func appendCol2Chunk(idx int, raw interface{}, ft tiTypes.FieldType, chk *chunk.Chunk) { +func appendCol2Chunk(idx int, raw any, ft tiTypes.FieldType, chk *chunk.Chunk) { if raw == nil { chk.AppendNull(idx) return diff --git a/pkg/common/event/redo_test.go b/pkg/common/event/redo_test.go index 4a45f2e1e0..52b9ad872f 100644 --- a/pkg/common/event/redo_test.go +++ b/pkg/common/event/redo_test.go @@ -16,12 +16,11 @@ package event import ( "testing" - "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/tidb/pkg/meta/model" "github.com/stretchr/testify/require" ) -func TestRedoDMLEventToDMLEventPreservesOriginAndTargetNames(t *testing.T) { +func TestRedoUsesRoutedTableNames(t *testing.T) { t.Parallel() helper := NewEventTestHelper(t) @@ -34,6 +33,35 @@ func TestRedoDMLEventToDMLEventPreservesOriginAndTargetNames(t *testing.T) { sourceTableInfo := helper.GetTableInfo(job) routedTableInfo := sourceTableInfo.CloneWithRouting("target_db", "target_table") + redoDDLEvent := (&DDLEvent{ + Query: "ALTER TABLE `target_db`.`target_table` ADD COLUMN age INT", + Type: byte(model.ActionAddColumn), + SchemaName: "test", + TableName: "t", + TableInfo: routedTableInfo, + FinishedTs: 200, + StartTs: 100, + }).ToRedoLog().RedoDDL + + require.Equal(t, "target_db", redoDDLEvent.TableName.Schema) + require.Equal(t, "target_table", redoDDLEvent.TableName.Table) + require.Empty(t, redoDDLEvent.TableName.TargetSchema) + require.Empty(t, redoDDLEvent.TableName.TargetTable) + + ddlEvent := redoDDLEvent.ToDDLEvent() + require.Equal(t, "target_db", ddlEvent.SchemaName) + require.Equal(t, "target_table", ddlEvent.TableName) + require.Equal(t, "target_db", ddlEvent.GetTargetSchemaName()) + require.Equal(t, "target_table", ddlEvent.GetTargetTableName()) + require.Equal(t, "target_db", ddlEvent.TableInfo.GetSchemaName()) + require.Equal(t, "target_table", ddlEvent.TableInfo.GetTableName()) + require.Equal(t, "target_db", ddlEvent.TableInfo.GetTargetSchemaName()) + require.Equal(t, "target_table", ddlEvent.TableInfo.GetTargetTableName()) + require.Equal(t, []SchemaTableName{{ + SchemaName: "target_db", + TableName: "target_table", + }}, ddlEvent.BlockedTableNames) + dmlEvent := helper.DML2Event("test", "t", `insert into test.t values (1, 'alice')`) dmlEvent.TableInfo = routedTableInfo @@ -48,42 +76,14 @@ func TestRedoDMLEventToDMLEventPreservesOriginAndTargetNames(t *testing.T) { Event: row, }).ToRedoLog().RedoRow + require.Equal(t, "target_db", redoRow.Row.Table.Schema) + require.Equal(t, "target_table", redoRow.Row.Table.Table) + require.Empty(t, redoRow.Row.Table.TargetSchema) + require.Empty(t, redoRow.Row.Table.TargetTable) + decoded := redoRow.ToDMLEvent() - require.Equal(t, "test", decoded.TableInfo.GetSchemaName()) - require.Equal(t, "t", decoded.TableInfo.GetTableName()) + require.Equal(t, "target_db", decoded.TableInfo.GetSchemaName()) + require.Equal(t, "target_table", decoded.TableInfo.GetTableName()) require.Equal(t, "target_db", decoded.TableInfo.GetTargetSchemaName()) require.Equal(t, "target_table", decoded.TableInfo.GetTargetTableName()) } - -func TestRedoDDLEventToDDLEventPreservesOriginAndTargetNames(t *testing.T) { - t.Parallel() - - redoDDLEvent := &RedoDDLEvent{ - DDL: &DDLEventInRedoLog{ - StartTs: 100, - CommitTs: 200, - Query: "ALTER TABLE `target_db`.`target_table` ADD COLUMN age INT", - }, - Type: byte(model.ActionAddColumn), - TableName: common.TableName{ - Schema: "source_db", - Table: "source_table", - TargetSchema: "target_db", - TargetTable: "target_table", - }, - } - - ddlEvent := redoDDLEvent.ToDDLEvent() - require.Equal(t, "source_db", ddlEvent.SchemaName) - require.Equal(t, "source_table", ddlEvent.TableName) - require.Equal(t, "target_db", ddlEvent.GetTargetSchemaName()) - require.Equal(t, "target_table", ddlEvent.GetTargetTableName()) - require.Equal(t, "source_db", ddlEvent.TableInfo.GetSchemaName()) - require.Equal(t, "source_table", ddlEvent.TableInfo.GetTableName()) - require.Equal(t, "target_db", ddlEvent.TableInfo.GetTargetSchemaName()) - require.Equal(t, "target_table", ddlEvent.TableInfo.GetTargetTableName()) - require.Equal(t, []SchemaTableName{{ - SchemaName: "target_db", - TableName: "target_table", - }}, ddlEvent.BlockedTableNames) -} From 442d6f1b82e602e2aba433ca2c794396f3e0de7f Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Thu, 2 Apr 2026 15:05:38 +0800 Subject: [PATCH 14/20] fix redo --- pkg/common/event/ddl_event.go | 4 +--- pkg/common/event/dml_event.go | 20 ++++++++------------ pkg/common/table_info.go | 16 ++++------------ pkg/common/table_name.go | 6 +++--- 4 files changed, 16 insertions(+), 30 deletions(-) diff --git a/pkg/common/event/ddl_event.go b/pkg/common/event/ddl_event.go index 1d3390fbb2..56b24cacec 100644 --- a/pkg/common/event/ddl_event.go +++ b/pkg/common/event/ddl_event.go @@ -40,9 +40,7 @@ type DDLEvent struct { // Type is the type of the DDL. Type byte `json:"type"` // SchemaID is from upstream job.SchemaID - SchemaID int64 `json:"schema_id"` - - // SchemaName and TableName carry the origin names. + SchemaID int64 `json:"schema_id"` SchemaName string `json:"schema_name"` TableName string `json:"table_name"` diff --git a/pkg/common/event/dml_event.go b/pkg/common/event/dml_event.go index 48f75bfcff..04bf4687c4 100644 --- a/pkg/common/event/dml_event.go +++ b/pkg/common/event/dml_event.go @@ -278,8 +278,6 @@ func (b *BatchDMLEvent) encodeV1() ([]byte, error) { // AssembleRows assembles the Rows from the RawRows. // It also sets the TableInfo and clears the RawRows. -// For local events (same node, b.Rows already set), it only applies routing -// without replacing the TableInfo to preserve schema version compatibility. func (b *BatchDMLEvent) AssembleRows(tableInfo *common.TableInfo) { if tableInfo == nil { log.Panic("DMLEvent: TableInfo is nil") @@ -290,11 +288,10 @@ func (b *BatchDMLEvent) AssembleRows(tableInfo *common.TableInfo) { }() // For local events (same node), rows are already set. - // If routing is configured, reassign the TableInfo pointer to the passed tableInfo - // (which already has TargetSchema/TargetTable set via CloneWithRouting). - // IMPORTANT: We modify the POINTER, not the object it points to, because the - // original TableInfo is shared from the schema store across all dispatchers. if b.Rows != nil { + // table routing is enabled, reassign the TableInfo pointer to the passed tableInfo + // IMPORTANT: We modify the POINTER, not the object it points to, because the + // original TableInfo is shared from the schema store across all dispatchers. if tableInfo.TableName.TargetSchema != "" || tableInfo.TableName.TargetTable != "" { b.TableInfo = tableInfo for _, dml := range b.DMLEvents { @@ -304,7 +301,11 @@ func (b *BatchDMLEvent) AssembleRows(tableInfo *common.TableInfo) { return } - // For remote events, verify schema version compatibility before replacing TableInfo + if len(b.RawRows) == 0 { + log.Panic("DMLEvent: RawRows is empty") + return + } + if b.TableInfo != nil && b.TableInfo.GetUpdateTS() != tableInfo.GetUpdateTS() { log.Panic("DMLEvent: TableInfoVersion mismatch", zap.Uint64("dmlEventTableInfoVersion", b.TableInfo.GetUpdateTS()), @@ -312,11 +313,6 @@ func (b *BatchDMLEvent) AssembleRows(tableInfo *common.TableInfo) { return } - if len(b.RawRows) == 0 { - log.Panic("DMLEvent: RawRows is empty") - return - } - decoder := chunk.NewCodec(tableInfo.GetFieldSlice()) b.Rows, _ = decoder.Decode(b.RawRows) b.TableInfo = tableInfo diff --git a/pkg/common/table_info.go b/pkg/common/table_info.go index e2c9143f4e..bfb25e53ee 100644 --- a/pkg/common/table_info.go +++ b/pkg/common/table_info.go @@ -125,14 +125,6 @@ func (ti *TableInfo) InitPrivateFields() { return } - // columnSchema may be nil for minimal TableInfo instances (e.g., in tests). - // In production, columnSchema is always set via WrapTableInfo or similar. - // Early return here without marking as initialized, so if columnSchema is - // set later, InitPrivateFields can be called again to properly initialize. - if ti.columnSchema == nil { - return - } - ti.preSQLs.mutex.Lock() defer ti.preSQLs.mutex.Unlock() @@ -392,12 +384,12 @@ func (ti *TableInfo) MustGetColumnOffsetByID(id int64) int { return offset } -// GetSchemaName returns the origin schema name carried by this TableInfo. +// GetSchemaName returns the schema name of the table func (ti *TableInfo) GetSchemaName() string { return ti.TableName.GetSchema() } -// GetTableName returns the origin table name carried by this TableInfo. +// GetTableName returns the table name of the table func (ti *TableInfo) GetTableName() string { return ti.TableName.GetTable() } @@ -406,12 +398,12 @@ func (ti *TableInfo) GetTableNameCIStr() ast.CIStr { return ast.NewCIStr(ti.GetTableName()) } -// GetSchemaNamePtr returns the pointer to the origin schema name. +// GetSchemaNamePtr returns the pointer to the schema name of the table func (ti *TableInfo) GetSchemaNamePtr() *string { return &ti.TableName.Schema } -// GetTableNamePtr returns the pointer to the origin table name. +// GetTableNamePtr returns the pointer to the table name of the table func (ti *TableInfo) GetTableNamePtr() *string { return &ti.TableName.Table } diff --git a/pkg/common/table_name.go b/pkg/common/table_name.go index 01b0b06efc..440af53fbc 100644 --- a/pkg/common/table_name.go +++ b/pkg/common/table_name.go @@ -37,17 +37,17 @@ func (t TableName) String() string { return fmt.Sprintf("%s.%s", t.Schema, t.Table) } -// QuoteString returns quoted full original table name. +// QuoteString returns quoted full table name. func (t TableName) QuoteString() string { return QuoteSchema(t.Schema, t.Table) } -// GetSchema returns the original schema name. +// GetSchema returns the schema name. func (t *TableName) GetSchema() string { return t.Schema } -// GetTable returns the original table name. +// GetTable returns the table name. func (t *TableName) GetTable() string { return t.Table } From 1157186505b629a0645227ef4c9d184839c03e9a Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Thu, 2 Apr 2026 15:28:17 +0800 Subject: [PATCH 15/20] fix file mock --- pkg/redo/writer/file/file_mock.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/redo/writer/file/file_mock.go b/pkg/redo/writer/file/file_mock.go index b4305d52cd..40679247ce 100644 --- a/pkg/redo/writer/file/file_mock.go +++ b/pkg/redo/writer/file/file_mock.go @@ -6,8 +6,9 @@ import ( context "context" event "github.com/pingcap/ticdc/pkg/common/event" - writer "github.com/pingcap/ticdc/pkg/redo/writer" mock "github.com/stretchr/testify/mock" + + writer "github.com/pingcap/ticdc/pkg/redo/writer" ) // mockFileWriter is an autogenerated mock type for the fileWriter type From 71de4e107219ed050adead1bf8f5442b485e6b02 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 21 Apr 2026 18:10:45 +0800 Subject: [PATCH 16/20] adjust more code --- pkg/common/event/ddl_event.go | 1 + pkg/common/event/dml_event.go | 26 ++++++++-------- pkg/common/table_name.go | 5 +++ pkg/common/table_name_test.go | 57 +++++++++++++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 12 deletions(-) create mode 100644 pkg/common/table_name_test.go diff --git a/pkg/common/event/ddl_event.go b/pkg/common/event/ddl_event.go index 56b24cacec..38bd3e6a7d 100644 --- a/pkg/common/event/ddl_event.go +++ b/pkg/common/event/ddl_event.go @@ -249,6 +249,7 @@ func (d *DDLEvent) GetTableID() int64 { return 0 } +// GetEvents split the multi tables DDL into single table DDLs. func (d *DDLEvent) GetEvents() []*DDLEvent { // Some ddl event may be multi-events, we need to split it into multiple messages. // Such as rename table test.table1 to test.table10, test.table2 to test.table20 diff --git a/pkg/common/event/dml_event.go b/pkg/common/event/dml_event.go index 04bf4687c4..33b6165280 100644 --- a/pkg/common/event/dml_event.go +++ b/pkg/common/event/dml_event.go @@ -289,14 +289,17 @@ func (b *BatchDMLEvent) AssembleRows(tableInfo *common.TableInfo) { // For local events (same node), rows are already set. if b.Rows != nil { - // table routing is enabled, reassign the TableInfo pointer to the passed tableInfo - // IMPORTANT: We modify the POINTER, not the object it points to, because the - // original TableInfo is shared from the schema store across all dispatchers. - if tableInfo.TableName.TargetSchema != "" || tableInfo.TableName.TargetTable != "" { - b.TableInfo = tableInfo - for _, dml := range b.DMLEvents { - dml.TableInfo = tableInfo - } + if !tableInfo.TableName.IsRouted() { + return + } + if b.TableInfo != nil && b.TableInfo.GetUpdateTS() != tableInfo.GetUpdateTS() { + log.Panic("table version mismatch when set routed table info", + zap.Uint64("originTableVersion", b.TableInfo.GetUpdateTS()), + zap.Uint64("routedTableVersion", tableInfo.GetUpdateTS())) + } + b.TableInfo = tableInfo + for _, dml := range b.DMLEvents { + dml.TableInfo = tableInfo } return } @@ -307,10 +310,9 @@ func (b *BatchDMLEvent) AssembleRows(tableInfo *common.TableInfo) { } if b.TableInfo != nil && b.TableInfo.GetUpdateTS() != tableInfo.GetUpdateTS() { - log.Panic("DMLEvent: TableInfoVersion mismatch", - zap.Uint64("dmlEventTableInfoVersion", b.TableInfo.GetUpdateTS()), - zap.Uint64("tableInfoVersion", tableInfo.GetUpdateTS())) - return + log.Panic("table version mismatch when decode remote raw rows", + zap.Uint64("originTableVersion", b.TableInfo.GetUpdateTS()), + zap.Uint64("routedTableVersion", tableInfo.GetUpdateTS())) } decoder := chunk.NewCodec(tableInfo.GetFieldSlice()) diff --git a/pkg/common/table_name.go b/pkg/common/table_name.go index 440af53fbc..bf437be6a8 100644 --- a/pkg/common/table_name.go +++ b/pkg/common/table_name.go @@ -52,6 +52,11 @@ func (t *TableName) GetTable() string { return t.Table } +// IsRouted returns whether table routing is enabled. +func (t *TableName) IsRouted() bool { + return t.TargetSchema != "" || t.TargetTable != "" +} + // GetTargetSchema returns the target schema name for routing. // If TargetSchema is empty, returns Schema. func (t *TableName) GetTargetSchema() string { diff --git a/pkg/common/table_name_test.go b/pkg/common/table_name_test.go new file mode 100644 index 0000000000..5b1aabf1af --- /dev/null +++ b/pkg/common/table_name_test.go @@ -0,0 +1,57 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import "testing" + +func TestTableNameIsRouted(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + tableName TableName + expected bool + }{ + { + name: "not routed", + tableName: TableName{Schema: "test", Table: "t"}, + expected: false, + }, + { + name: "target schema only", + tableName: TableName{Schema: "test", Table: "t", TargetSchema: "target"}, + expected: true, + }, + { + name: "target table only", + tableName: TableName{Schema: "test", Table: "t", TargetTable: "target_t"}, + expected: true, + }, + { + name: "target schema and table", + tableName: TableName{Schema: "test", Table: "t", TargetSchema: "target", TargetTable: "target_t"}, + expected: true, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + if got := tt.tableName.IsRouted(); got != tt.expected { + t.Fatalf("IsRouted() = %v, expected %v", got, tt.expected) + } + }) + } +} From 85ffa6fee33fb1fd81455a984afab15d3413f0e7 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 21 Apr 2026 18:32:27 +0800 Subject: [PATCH 17/20] adjust more code --- pkg/common/event/ddl_event.go | 32 ++++++++++++++++++-------------- pkg/common/table_name.go | 18 +++++++++--------- pkg/common/table_name_test.go | 2 +- 3 files changed, 28 insertions(+), 24 deletions(-) diff --git a/pkg/common/event/ddl_event.go b/pkg/common/event/ddl_event.go index 38bd3e6a7d..0ef8946228 100644 --- a/pkg/common/event/ddl_event.go +++ b/pkg/common/event/ddl_event.go @@ -570,20 +570,24 @@ func NewRoutedDDLEvent( FinishedTs: d.FinishedTs, Seq: d.Seq, Epoch: d.Epoch, - MultipleTableInfos: multipleTableInfos, - BlockedTables: d.BlockedTables, - BlockedTableNames: blockedTableNames, - NeedDroppedTables: d.NeedDroppedTables, - NeedAddedTables: d.NeedAddedTables, - UpdatedSchemas: d.UpdatedSchemas, - TableNameChange: d.TableNameChange, - TiDBOnly: d.TiDBOnly, - BDRMode: d.BDRMode, - Err: d.Err, - PostTxnFlushed: clonePostTxnFlushed(d.PostTxnFlushed), - eventSize: d.eventSize, - IsBootstrap: d.IsBootstrap, - NotSync: d.NotSync, + // MultipleTableInfos and BlockedTableNames carry table names used by downstream + // execution paths, so the routed versions must be passed in explicitly. + MultipleTableInfos: multipleTableInfos, + BlockedTableNames: blockedTableNames, + // The following fields do not participate in table route name rewriting, + // so the routed event keeps the original values from the source event. + BlockedTables: d.BlockedTables, + NeedDroppedTables: d.NeedDroppedTables, + NeedAddedTables: d.NeedAddedTables, + UpdatedSchemas: d.UpdatedSchemas, + TableNameChange: d.TableNameChange, + TiDBOnly: d.TiDBOnly, + BDRMode: d.BDRMode, + Err: d.Err, + PostTxnFlushed: clonePostTxnFlushed(d.PostTxnFlushed), + eventSize: d.eventSize, + IsBootstrap: d.IsBootstrap, + NotSync: d.NotSync, } } diff --git a/pkg/common/table_name.go b/pkg/common/table_name.go index bf437be6a8..965c4d0de0 100644 --- a/pkg/common/table_name.go +++ b/pkg/common/table_name.go @@ -19,7 +19,7 @@ import ( //go:generate msgp -// TableName represents name of a table, includes table name and schema name. +// TableName represents name of a table, includes table name and schema name type TableName struct { Schema string `toml:"db-name" msg:"db-name"` Table string `toml:"tbl-name" msg:"tbl-name"` @@ -32,33 +32,33 @@ type TableName struct { TargetTable string `toml:"target-tbl-name" msg:"target-tbl-name"` } -// String implements fmt.Stringer interface. +// String implements fmt.Stringer interface func (t TableName) String() string { return fmt.Sprintf("%s.%s", t.Schema, t.Table) } -// QuoteString returns quoted full table name. +// QuoteString returns quoted full table name func (t TableName) QuoteString() string { return QuoteSchema(t.Schema, t.Table) } -// GetSchema returns the schema name. +// GetSchema returns the schema name func (t *TableName) GetSchema() string { return t.Schema } -// GetTable returns the table name. +// GetTable returns the table name func (t *TableName) GetTable() string { return t.Table } -// IsRouted returns whether table routing is enabled. +// IsRouted returns whether table routing is enabled func (t *TableName) IsRouted() bool { return t.TargetSchema != "" || t.TargetTable != "" } // GetTargetSchema returns the target schema name for routing. -// If TargetSchema is empty, returns Schema. +// If TargetSchema is empty, returns Schema func (t *TableName) GetTargetSchema() string { if t.TargetSchema != "" { return t.TargetSchema @@ -66,7 +66,7 @@ func (t *TableName) GetTargetSchema() string { return t.Schema } -// GetTargetTable returns the target table name for routing. +// GetTargetTable returns the target table name for routing // If TargetTable is empty, returns Table. func (t *TableName) GetTargetTable() string { if t.TargetTable != "" { @@ -75,7 +75,7 @@ func (t *TableName) GetTargetTable() string { return t.Table } -// QuoteTargetString returns quoted full target table name for routing. +// QuoteTargetString returns quoted full target table name for routing func (t TableName) QuoteTargetString() string { return QuoteSchema(t.GetTargetSchema(), t.GetTargetTable()) } diff --git a/pkg/common/table_name_test.go b/pkg/common/table_name_test.go index 5b1aabf1af..5881b6e840 100644 --- a/pkg/common/table_name_test.go +++ b/pkg/common/table_name_test.go @@ -1,4 +1,4 @@ -// Copyright 2025 PingCAP, Inc. +// Copyright 2026 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. From ba25d82284221eaed642e870a1c2e1fb5852571f Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 21 Apr 2026 19:04:13 +0800 Subject: [PATCH 18/20] adjust more code --- pkg/common/event/dml_event.go | 24 +++++--- pkg/common/event/dml_event_test.go | 98 ++++++++++++++++++++++++++++++ pkg/common/table_info_test.go | 4 +- pkg/common/table_name_test.go | 56 ++++++++++++++++- 4 files changed, 171 insertions(+), 11 deletions(-) diff --git a/pkg/common/event/dml_event.go b/pkg/common/event/dml_event.go index 33b6165280..b5d3b7d3a0 100644 --- a/pkg/common/event/dml_event.go +++ b/pkg/common/event/dml_event.go @@ -292,10 +292,14 @@ func (b *BatchDMLEvent) AssembleRows(tableInfo *common.TableInfo) { if !tableInfo.TableName.IsRouted() { return } - if b.TableInfo != nil && b.TableInfo.GetUpdateTS() != tableInfo.GetUpdateTS() { - log.Panic("table version mismatch when set routed table info", - zap.Uint64("originTableVersion", b.TableInfo.GetUpdateTS()), - zap.Uint64("routedTableVersion", tableInfo.GetUpdateTS())) + if b.TableInfo != nil { + originVersion := b.TableInfo.GetUpdateTS() + routedVersion := tableInfo.GetUpdateTS() + if originVersion != routedVersion { + log.Panic("table version mismatch when set routed table info", + zap.Uint64("originTableVersion", originVersion), + zap.Uint64("routedTableVersion", routedVersion)) + } } b.TableInfo = tableInfo for _, dml := range b.DMLEvents { @@ -309,10 +313,14 @@ func (b *BatchDMLEvent) AssembleRows(tableInfo *common.TableInfo) { return } - if b.TableInfo != nil && b.TableInfo.GetUpdateTS() != tableInfo.GetUpdateTS() { - log.Panic("table version mismatch when decode remote raw rows", - zap.Uint64("originTableVersion", b.TableInfo.GetUpdateTS()), - zap.Uint64("routedTableVersion", tableInfo.GetUpdateTS())) + if b.TableInfo != nil { + originVersion := b.TableInfo.GetUpdateTS() + routedVersion := tableInfo.GetUpdateTS() + if originVersion != routedVersion { + log.Panic("table version mismatch when decode remote raw rows", + zap.Uint64("originTableVersion", originVersion), + zap.Uint64("routedTableVersion", routedVersion)) + } } decoder := chunk.NewCodec(tableInfo.GetFieldSlice()) diff --git a/pkg/common/event/dml_event_test.go b/pkg/common/event/dml_event_test.go index 39f72f566e..9a0ed27e99 100644 --- a/pkg/common/event/dml_event_test.go +++ b/pkg/common/event/dml_event_test.go @@ -150,6 +150,104 @@ func TestBatchDMLEvent(t *testing.T) { require.Contains(t, err.Error(), "unsupported BatchDMLEvent version") } +func TestBatchDMLEventAssembleRowsRebindsRoutedTableInfoForLocalRows(t *testing.T) { + helper := NewEventTestHelper(t) + defer helper.Close() + + helper.tk.MustExec("use test") + helper.DDL2Job(createTableSQL) + + dmlEvent := helper.DML2Event("test", "t", insertDataSQL) + require.NotNil(t, dmlEvent) + + originTableInfo := dmlEvent.TableInfo + routedTableInfo := originTableInfo.CloneWithRouting("target_schema", "target_table") + require.NotNil(t, routedTableInfo) + + batchDMLEvent := &BatchDMLEvent{ + Version: BatchDMLEventVersion1, + DMLEventCount: 1, + DMLEvents: []*DMLEvent{dmlEvent}, + Rows: dmlEvent.Rows, + TableInfo: originTableInfo, + } + + batchDMLEvent.AssembleRows(routedTableInfo) + + require.Same(t, dmlEvent.Rows, batchDMLEvent.Rows) + require.Same(t, routedTableInfo, batchDMLEvent.TableInfo) + require.Same(t, routedTableInfo, batchDMLEvent.DMLEvents[0].TableInfo) + require.Equal(t, "target_schema", batchDMLEvent.TableInfo.GetTargetSchemaName()) + require.Equal(t, "target_table", batchDMLEvent.TableInfo.GetTargetTableName()) + require.Contains(t, batchDMLEvent.TableInfo.GetPreInsertSQL(), common.QuoteSchema("target_schema", "target_table")) +} + +func TestBatchDMLEventAssembleRowsKeepsOriginalTableInfoForLocalRowsWithoutRouting(t *testing.T) { + helper := NewEventTestHelper(t) + defer helper.Close() + + helper.tk.MustExec("use test") + helper.DDL2Job(createTableSQL) + + dmlEvent := helper.DML2Event("test", "t", insertDataSQL) + require.NotNil(t, dmlEvent) + + originTableInfo := dmlEvent.TableInfo + notRoutedTableInfo := originTableInfo.CloneWithRouting("", "") + require.NotNil(t, notRoutedTableInfo) + require.False(t, notRoutedTableInfo.TableName.IsRouted()) + notRoutedTableInfo.UpdateTS++ + + batchDMLEvent := &BatchDMLEvent{ + Version: BatchDMLEventVersion1, + DMLEventCount: 1, + DMLEvents: []*DMLEvent{dmlEvent}, + Rows: dmlEvent.Rows, + TableInfo: originTableInfo, + } + + batchDMLEvent.AssembleRows(notRoutedTableInfo) + + require.Same(t, dmlEvent.Rows, batchDMLEvent.Rows) + require.Same(t, originTableInfo, batchDMLEvent.TableInfo) + require.Same(t, originTableInfo, batchDMLEvent.DMLEvents[0].TableInfo) + require.Equal(t, "test", batchDMLEvent.TableInfo.GetTargetSchemaName()) + require.Equal(t, "t", batchDMLEvent.TableInfo.GetTargetTableName()) +} + +func TestBatchDMLEventAssembleRowsDecodesRemoteRawRows(t *testing.T) { + helper := NewEventTestHelper(t) + defer helper.Close() + + helper.tk.MustExec("use test") + helper.DDL2Job(createTableSQL) + + dmlEvent := helper.DML2Event("test", "t", insertDataSQL) + require.NotNil(t, dmlEvent) + + batchDMLEvent := &BatchDMLEvent{ + Version: BatchDMLEventVersion1, + DMLEventCount: 1, + DMLEvents: []*DMLEvent{dmlEvent}, + Rows: dmlEvent.Rows, + TableInfo: dmlEvent.TableInfo, + } + data, err := batchDMLEvent.Marshal() + require.NoError(t, err) + + reverseEvents := &BatchDMLEvent{} + err = reverseEvents.Unmarshal(data) + require.NoError(t, err) + require.Nil(t, reverseEvents.Rows) + require.NotEmpty(t, reverseEvents.RawRows) + + reverseEvents.AssembleRows(batchDMLEvent.TableInfo) + + require.Nil(t, reverseEvents.RawRows) + require.Same(t, batchDMLEvent.TableInfo, reverseEvents.TableInfo) + require.Equal(t, batchDMLEvent.Rows.ToString(batchDMLEvent.TableInfo.GetFieldSlice()), reverseEvents.Rows.ToString(batchDMLEvent.TableInfo.GetFieldSlice())) +} + func TestEncodeAnddecodeV1(t *testing.T) { helper := NewEventTestHelper(t) defer helper.Close() diff --git a/pkg/common/table_info_test.go b/pkg/common/table_info_test.go index 2a1054d7e6..5be97d2f37 100644 --- a/pkg/common/table_info_test.go +++ b/pkg/common/table_info_test.go @@ -61,8 +61,8 @@ func TestCloneWithRouting(t *testing.T) { require.Equal(t, "target_table", cloned.GetTargetTableName()) require.Equal(t, "source_db.source_table", cloned.TableName.String()) require.Equal(t, "`source_db`.`source_table`", cloned.TableName.QuoteString()) - require.Equal(t, "source_db.source_table", cloned.TableName.String()) - require.Equal(t, "`source_db`.`source_table`", cloned.TableName.QuoteString()) + require.Equal(t, "`target_db`.`target_table`", cloned.TableName.QuoteTargetString()) + require.True(t, cloned.TableName.IsRouted()) require.Same(t, &cloned.TableName.Schema, cloned.GetSchemaNamePtr()) require.Same(t, &cloned.TableName.Table, cloned.GetTableNamePtr()) diff --git a/pkg/common/table_name_test.go b/pkg/common/table_name_test.go index 5881b6e840..0988e0d4a2 100644 --- a/pkg/common/table_name_test.go +++ b/pkg/common/table_name_test.go @@ -13,7 +13,11 @@ package common -import "testing" +import ( + "testing" + + "github.com/stretchr/testify/require" +) func TestTableNameIsRouted(t *testing.T) { t.Parallel() @@ -55,3 +59,53 @@ func TestTableNameIsRouted(t *testing.T) { }) } } + +func TestTableNameTargetAccessors(t *testing.T) { + t.Parallel() + + t.Run("fallback to source names", func(t *testing.T) { + tableName := TableName{ + Schema: "source_db", + Table: "source_table", + } + + require.Equal(t, "source_db", tableName.GetTargetSchema()) + require.Equal(t, "source_table", tableName.GetTargetTable()) + require.Equal(t, "`source_db`.`source_table`", tableName.QuoteTargetString()) + }) + + t.Run("use routed names when present", func(t *testing.T) { + tableName := TableName{ + Schema: "source_db", + Table: "source_table", + TargetSchema: "target_db", + TargetTable: "target_table", + } + + require.Equal(t, "target_db", tableName.GetTargetSchema()) + require.Equal(t, "target_table", tableName.GetTargetTable()) + require.Equal(t, "`target_db`.`target_table`", tableName.QuoteTargetString()) + }) +} + +func TestTableNameMsgpackRoundTrip(t *testing.T) { + t.Parallel() + + original := TableName{ + Schema: "source_db", + Table: "source_table", + TableID: 42, + IsPartition: true, + TargetSchema: "target_db", + TargetTable: "target_table", + } + + data, err := original.MarshalMsg(nil) + require.NoError(t, err) + + var decoded TableName + rest, err := decoded.UnmarshalMsg(data) + require.NoError(t, err) + require.Empty(t, rest) + require.Equal(t, original, decoded) +} From 815faf63055aefca1b9b06cc54f4bea4ce0aab13 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 21 Apr 2026 19:36:47 +0800 Subject: [PATCH 19/20] adjust more code --- pkg/common/table_name.go | 8 ++-- pkg/common/table_name_gen.go | 64 ++++--------------------------- pkg/common/table_name_gen_test.go | 4 +- pkg/common/table_name_test.go | 11 +++++- 4 files changed, 23 insertions(+), 64 deletions(-) diff --git a/pkg/common/table_name.go b/pkg/common/table_name.go index 965c4d0de0..dbf5121a77 100644 --- a/pkg/common/table_name.go +++ b/pkg/common/table_name.go @@ -27,9 +27,11 @@ type TableName struct { TableID int64 `toml:"tbl-id" msg:"tbl-id"` IsPartition bool `toml:"is-partition" msg:"is-partition"` - // TargetSchema and TargetTable is not empty if table routing enabled - TargetSchema string `toml:"target-db-name" msg:"target-db-name"` - TargetTable string `toml:"target-tbl-name" msg:"target-tbl-name"` + // TargetSchema and TargetTable are used as an in-memory routing overlay. + // They are intentionally excluded from msgpack serialization because redo + // persists routed names canonically in Schema/Table. + TargetSchema string `toml:"target-db-name" msg:"-"` + TargetTable string `toml:"target-tbl-name" msg:"-"` } // String implements fmt.Stringer interface diff --git a/pkg/common/table_name_gen.go b/pkg/common/table_name_gen.go index 31cb8c036e..b18f0125bf 100644 --- a/pkg/common/table_name_gen.go +++ b/pkg/common/table_name_gen.go @@ -1,7 +1,7 @@ -// Code generated by github.com/tinylib/msgp DO NOT EDIT. - package common +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + import ( "github.com/tinylib/msgp/msgp" ) @@ -48,18 +48,6 @@ func (z *TableName) DecodeMsg(dc *msgp.Reader) (err error) { err = msgp.WrapError(err, "IsPartition") return } - case "target-db-name": - z.TargetSchema, err = dc.ReadString() - if err != nil { - err = msgp.WrapError(err, "TargetSchema") - return - } - case "target-tbl-name": - z.TargetTable, err = dc.ReadString() - if err != nil { - err = msgp.WrapError(err, "TargetTable") - return - } default: err = dc.Skip() if err != nil { @@ -73,9 +61,9 @@ func (z *TableName) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *TableName) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 6 + // map header, size 4 // write "db-name" - err = en.Append(0x86, 0xa7, 0x64, 0x62, 0x2d, 0x6e, 0x61, 0x6d, 0x65) + err = en.Append(0x84, 0xa7, 0x64, 0x62, 0x2d, 0x6e, 0x61, 0x6d, 0x65) if err != nil { return } @@ -114,35 +102,15 @@ func (z *TableName) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "IsPartition") return } - // write "target-db-name" - err = en.Append(0xae, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2d, 0x64, 0x62, 0x2d, 0x6e, 0x61, 0x6d, 0x65) - if err != nil { - return - } - err = en.WriteString(z.TargetSchema) - if err != nil { - err = msgp.WrapError(err, "TargetSchema") - return - } - // write "target-tbl-name" - err = en.Append(0xaf, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2d, 0x74, 0x62, 0x6c, 0x2d, 0x6e, 0x61, 0x6d, 0x65) - if err != nil { - return - } - err = en.WriteString(z.TargetTable) - if err != nil { - err = msgp.WrapError(err, "TargetTable") - return - } return } // MarshalMsg implements msgp.Marshaler func (z *TableName) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // map header, size 6 + // map header, size 4 // string "db-name" - o = append(o, 0x86, 0xa7, 0x64, 0x62, 0x2d, 0x6e, 0x61, 0x6d, 0x65) + o = append(o, 0x84, 0xa7, 0x64, 0x62, 0x2d, 0x6e, 0x61, 0x6d, 0x65) o = msgp.AppendString(o, z.Schema) // string "tbl-name" o = append(o, 0xa8, 0x74, 0x62, 0x6c, 0x2d, 0x6e, 0x61, 0x6d, 0x65) @@ -153,12 +121,6 @@ func (z *TableName) MarshalMsg(b []byte) (o []byte, err error) { // string "is-partition" o = append(o, 0xac, 0x69, 0x73, 0x2d, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e) o = msgp.AppendBool(o, z.IsPartition) - // string "target-db-name" - o = append(o, 0xae, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2d, 0x64, 0x62, 0x2d, 0x6e, 0x61, 0x6d, 0x65) - o = msgp.AppendString(o, z.TargetSchema) - // string "target-tbl-name" - o = append(o, 0xaf, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2d, 0x74, 0x62, 0x6c, 0x2d, 0x6e, 0x61, 0x6d, 0x65) - o = msgp.AppendString(o, z.TargetTable) return } @@ -204,18 +166,6 @@ func (z *TableName) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "IsPartition") return } - case "target-db-name": - z.TargetSchema, bts, err = msgp.ReadStringBytes(bts) - if err != nil { - err = msgp.WrapError(err, "TargetSchema") - return - } - case "target-tbl-name": - z.TargetTable, bts, err = msgp.ReadStringBytes(bts) - if err != nil { - err = msgp.WrapError(err, "TargetTable") - return - } default: bts, err = msgp.Skip(bts) if err != nil { @@ -230,6 +180,6 @@ func (z *TableName) UnmarshalMsg(bts []byte) (o []byte, err error) { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *TableName) Msgsize() (s int) { - s = 1 + 8 + msgp.StringPrefixSize + len(z.Schema) + 9 + msgp.StringPrefixSize + len(z.Table) + 7 + msgp.Int64Size + 13 + msgp.BoolSize + 15 + msgp.StringPrefixSize + len(z.TargetSchema) + 16 + msgp.StringPrefixSize + len(z.TargetTable) + s = 1 + 8 + msgp.StringPrefixSize + len(z.Schema) + 9 + msgp.StringPrefixSize + len(z.Table) + 7 + msgp.Int64Size + 13 + msgp.BoolSize return } diff --git a/pkg/common/table_name_gen_test.go b/pkg/common/table_name_gen_test.go index 6473a803f0..632a8452a6 100644 --- a/pkg/common/table_name_gen_test.go +++ b/pkg/common/table_name_gen_test.go @@ -1,7 +1,7 @@ -// Code generated by github.com/tinylib/msgp DO NOT EDIT. - package common +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + import ( "bytes" "testing" diff --git a/pkg/common/table_name_test.go b/pkg/common/table_name_test.go index 0988e0d4a2..54a2f914cb 100644 --- a/pkg/common/table_name_test.go +++ b/pkg/common/table_name_test.go @@ -88,7 +88,7 @@ func TestTableNameTargetAccessors(t *testing.T) { }) } -func TestTableNameMsgpackRoundTrip(t *testing.T) { +func TestTableNameMsgpackRoundTripDropsRoutingOverlay(t *testing.T) { t.Parallel() original := TableName{ @@ -107,5 +107,12 @@ func TestTableNameMsgpackRoundTrip(t *testing.T) { rest, err := decoded.UnmarshalMsg(data) require.NoError(t, err) require.Empty(t, rest) - require.Equal(t, original, decoded) + require.Equal(t, original.Schema, decoded.Schema) + require.Equal(t, original.Table, decoded.Table) + require.Equal(t, original.TableID, decoded.TableID) + require.Equal(t, original.IsPartition, decoded.IsPartition) + require.Empty(t, decoded.TargetSchema) + require.Empty(t, decoded.TargetTable) + require.Equal(t, original.Schema, decoded.GetTargetSchema()) + require.Equal(t, original.Table, decoded.GetTargetTable()) } From 70e04e1eb91731c2f7fcec93a63e0c9fcfae2111 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 21 Apr 2026 19:44:34 +0800 Subject: [PATCH 20/20] fix make --- pkg/common/table_name_gen.go | 4 ++-- pkg/common/table_name_gen_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/common/table_name_gen.go b/pkg/common/table_name_gen.go index b18f0125bf..2f0df4ba92 100644 --- a/pkg/common/table_name_gen.go +++ b/pkg/common/table_name_gen.go @@ -1,7 +1,7 @@ -package common - // Code generated by github.com/tinylib/msgp DO NOT EDIT. +package common + import ( "github.com/tinylib/msgp/msgp" ) diff --git a/pkg/common/table_name_gen_test.go b/pkg/common/table_name_gen_test.go index 632a8452a6..6473a803f0 100644 --- a/pkg/common/table_name_gen_test.go +++ b/pkg/common/table_name_gen_test.go @@ -1,7 +1,7 @@ -package common - // Code generated by github.com/tinylib/msgp DO NOT EDIT. +package common + import ( "bytes" "testing"