diff --git a/api/v2/model.go b/api/v2/model.go index 67a6ca4728..6cb1c9e809 100644 --- a/api/v2/model.go +++ b/api/v2/model.go @@ -352,6 +352,12 @@ func (c *ReplicaConfig) toInternalReplicaConfigWithOriginConfig( if c.Consistent.FlushConcurrency != nil { res.Consistent.FlushConcurrency = c.Consistent.FlushConcurrency } + if c.Consistent.SpoolDiskQuota != nil { + res.Consistent.SpoolDiskQuota = c.Consistent.SpoolDiskQuota + } + if c.Consistent.SpoolBaseDir != nil { + res.Consistent.SpoolBaseDir = c.Consistent.SpoolBaseDir + } if c.Consistent.MemoryUsage != nil { res.Consistent.MemoryUsage = &config.ConsistentMemoryUsage{ @@ -1018,6 +1024,12 @@ func ToAPIReplicaConfig(c *config.ReplicaConfig) *ReplicaConfig { if cloned.Consistent.FlushConcurrency != nil { res.Consistent.FlushConcurrency = cloned.Consistent.FlushConcurrency } + if cloned.Consistent.SpoolDiskQuota != nil { + res.Consistent.SpoolDiskQuota = cloned.Consistent.SpoolDiskQuota + } + if cloned.Consistent.SpoolBaseDir != nil { + res.Consistent.SpoolBaseDir = cloned.Consistent.SpoolBaseDir + } if cloned.Consistent.MemoryUsage != nil { res.Consistent.MemoryUsage = &ConsistentMemoryUsage{ MemoryQuotaPercentage: cloned.Consistent.MemoryUsage.MemoryQuotaPercentage, @@ -1289,6 +1301,8 @@ type ConsistentConfig struct { UseFileBackend *bool `json:"use_file_backend,omitempty" toml:"use-file-backend,omitempty"` Compression *string `json:"compression,omitempty" toml:"compression,omitempty"` FlushConcurrency *int `json:"flush_concurrency,omitempty" toml:"flush-concurrency,omitempty"` + SpoolDiskQuota *int64 `json:"spool_disk_quota,omitempty" toml:"spool-disk-quota,omitempty"` + SpoolBaseDir *string `json:"spool_base_dir,omitempty" toml:"spool-base-dir,omitempty"` MemoryUsage *ConsistentMemoryUsage `json:"memory_usage,omitempty" toml:"memory-usage,omitempty"` EventCollectorBatchCount *int `json:"event_collector_batch_count,omitempty" toml:"event-collector-batch-count,omitempty"` diff --git a/api/v2/model_test.go b/api/v2/model_test.go index 8ed86f8e42..c1b7f7118b 100644 --- a/api/v2/model_test.go +++ b/api/v2/model_test.go @@ -62,6 +62,8 @@ func TestReplicaConfigConversion(t *testing.T) { MaxLogSize: util.AddressOf(int64(128)), FlushIntervalInMs: util.AddressOf(int64(2000)), Storage: util.AddressOf("s3://test"), + SpoolDiskQuota: util.AddressOf(int64(2048)), + SpoolBaseDir: util.AddressOf("/tmp/redo-spool"), }, } @@ -88,6 +90,8 @@ func TestReplicaConfigConversion(t *testing.T) { require.Equal(t, int64(128), util.GetOrZero(internalCfg.Consistent.MaxLogSize)) require.Equal(t, int64(2000), util.GetOrZero(internalCfg.Consistent.FlushIntervalInMs)) require.Equal(t, "s3://test", util.GetOrZero(internalCfg.Consistent.Storage)) + require.Equal(t, int64(2048), util.GetOrZero(internalCfg.Consistent.SpoolDiskQuota)) + require.Equal(t, "/tmp/redo-spool", util.GetOrZero(internalCfg.Consistent.SpoolBaseDir)) // output_old_value is omitted in apiCfg and must keep its default (true). require.True(t, internalCfg.Sink.Debezium.OutputOldValue) @@ -128,6 +132,8 @@ func TestReplicaConfigConversion(t *testing.T) { require.True(t, *apiCfgBack.Scheduler.EnableTableAcrossNodes) require.Equal(t, "correctness", *apiCfgBack.Integrity.IntegrityCheckLevel) require.Equal(t, "eventual", *apiCfgBack.Consistent.Level) + require.Equal(t, int64(2048), *apiCfgBack.Consistent.SpoolDiskQuota) + require.Equal(t, "/tmp/redo-spool", *apiCfgBack.Consistent.SpoolBaseDir) // Test case 4: batch fields round trip and nil preservation apiBatchCfg := &ReplicaConfig{ diff --git a/downstreamadapter/dispatchermanager/dispatcher_manager_redo.go b/downstreamadapter/dispatchermanager/dispatcher_manager_redo.go index 7ae077afb2..0ef14a28b4 100644 --- a/downstreamadapter/dispatchermanager/dispatcher_manager_redo.go +++ b/downstreamadapter/dispatchermanager/dispatcher_manager_redo.go @@ -376,7 +376,7 @@ func (e *DispatcherManager) SetRedoResolvedTs(resolvedTs uint64) bool { } func (e *DispatcherManager) collectRedoMeta(ctx context.Context) error { - ticker := time.NewTicker(time.Duration(*e.config.Consistent.FlushIntervalInMs) * time.Millisecond) + ticker := time.NewTicker(time.Duration(*e.config.Consistent.MetaFlushIntervalInMs) * time.Millisecond) defer ticker.Stop() mc := appcontext.GetService[messaging.MessageCenter](appcontext.MessageCenter) var preResolvedTs uint64 diff --git a/downstreamadapter/sink/cloudstorage/buffer_manager.go b/downstreamadapter/sink/cloudstorage/buffer_manager.go index fa49c404e1..2e4610c3c6 100644 --- a/downstreamadapter/sink/cloudstorage/buffer_manager.go +++ b/downstreamadapter/sink/cloudstorage/buffer_manager.go @@ -18,11 +18,11 @@ import ( "context" "time" - "github.com/pingcap/ticdc/downstreamadapter/sink/cloudstorage/spool" "github.com/pingcap/ticdc/downstreamadapter/sink/metrics" "github.com/pingcap/ticdc/pkg/cloudstorage" "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/sink/spool" ) const ( diff --git a/downstreamadapter/sink/cloudstorage/buffer_manager_test.go b/downstreamadapter/sink/cloudstorage/buffer_manager_test.go index 370ae3a986..2f64dc9a27 100644 --- a/downstreamadapter/sink/cloudstorage/buffer_manager_test.go +++ b/downstreamadapter/sink/cloudstorage/buffer_manager_test.go @@ -18,12 +18,12 @@ import ( "testing" "time" - "github.com/pingcap/ticdc/downstreamadapter/sink/cloudstorage/spool" "github.com/pingcap/ticdc/pkg/cloudstorage" commonType "github.com/pingcap/ticdc/pkg/common" commonEvent "github.com/pingcap/ticdc/pkg/common/event" "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/pingcap/ticdc/pkg/sink/spool" "github.com/stretchr/testify/require" ) diff --git a/downstreamadapter/sink/cloudstorage/dml_writers.go b/downstreamadapter/sink/cloudstorage/dml_writers.go index 6f4d8e9eb2..640ae3c00e 100644 --- a/downstreamadapter/sink/cloudstorage/dml_writers.go +++ b/downstreamadapter/sink/cloudstorage/dml_writers.go @@ -17,7 +17,6 @@ import ( "context" "time" - "github.com/pingcap/ticdc/downstreamadapter/sink/cloudstorage/spool" "github.com/pingcap/ticdc/downstreamadapter/sink/columnselector" sinkmetrics "github.com/pingcap/ticdc/downstreamadapter/sink/metrics" "github.com/pingcap/ticdc/pkg/cloudstorage" @@ -25,6 +24,7 @@ import ( commonEvent "github.com/pingcap/ticdc/pkg/common/event" "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/pingcap/ticdc/pkg/sink/spool" "github.com/pingcap/ticdc/utils/chann" "github.com/pingcap/tidb/pkg/objstore/storeapi" "go.uber.org/atomic" @@ -68,6 +68,7 @@ func newDMLWriters( changefeedID, spool.WithRootDir(config.SpoolBaseDir), spool.WithDiskQuotaBytes(config.SpoolDiskQuota), + spool.WithMetrics(newSpoolMetrics(changefeedID)), ) if err != nil { return nil, err diff --git a/downstreamadapter/sink/cloudstorage/spool/budget.go b/downstreamadapter/sink/cloudstorage/spool/budget.go deleted file mode 100644 index 8fbc4a240d..0000000000 --- a/downstreamadapter/sink/cloudstorage/spool/budget.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package spool - -// budget stores the current queued byte counts and the byte limits derived from spool config. -type budget struct { - // diskQuotaBytes is the largest byte count allowed in local spool files. - diskQuotaBytes int64 - - // memoryQuotaBytes is the largest byte count we still keep in memory. - // If adding a new entry would cross this value, spool writes that entry - // to local spool files instead of keeping it in memory. - memoryQuotaBytes int64 - - // highWatermarkBytes is the byte count that makes spool stop running the - // new PostEnqueue callback immediately. The callback is saved in memory and - // will be run later. - highWatermarkBytes int64 - - // lowWatermarkBytes is the byte count that lets spool run the saved - // PostEnqueue callbacks again after some queued data has been flushed to the - // downstream storage or discarded locally. - lowWatermarkBytes int64 - - // memoryBytes is the number of queued bytes that are still kept in memory. - memoryBytes int64 - // diskBytes is the number of queued bytes that have already been written to local spool files. - diskBytes int64 -} - -func newBudget(options *options) *budget { - return &budget{ - diskQuotaBytes: options.diskQuotaBytes, - memoryQuotaBytes: int64(float64(options.diskQuotaBytes) * options.memoryRatio), - highWatermarkBytes: int64(float64(options.diskQuotaBytes) * options.highWatermarkRatio), - lowWatermarkBytes: int64(float64(options.diskQuotaBytes) * options.lowWatermarkRatio), - } -} - -// shouldSpill decides whether a new entry should stay in memory or be written to local spool files. -func (b *budget) shouldSpill(entryBytes int64) bool { - return b.memoryBytes+entryBytes > b.memoryQuotaBytes -} - -// entryExceedsDiskQuota returns true when a single spilled entry is larger -// than the configured disk quota by itself. -func (b *budget) entryExceedsDiskQuota(entryBytes int64) bool { - return entryBytes > b.diskQuotaBytes -} - -// spillWouldExceedDiskQuota returns true when adding one more spilled entry to -// the current on-disk usage would exceed the configured disk quota. -func (b *budget) spillWouldExceedDiskQuota(entryBytes int64) bool { - return b.diskBytes+entryBytes > b.diskQuotaBytes -} - -// acquire adds a newly accepted entry to the current byte counters and returns -// whether total queued bytes are now above the high watermark. -func (b *budget) acquire(entryBytes int64, spilled bool) bool { - if spilled { - b.diskBytes += entryBytes - return b.totalBytes() > b.highWatermarkBytes - } - b.memoryBytes += entryBytes - return b.totalBytes() > b.highWatermarkBytes -} - -// release removes an entry from the current byte counters after the entry has -// been flushed or discarded, and returns whether total queued bytes are now at -// or below the low watermark. -func (b *budget) release(entryBytes int64, spilled bool) bool { - if spilled { - b.diskBytes -= entryBytes - } - if !spilled { - b.memoryBytes -= entryBytes - } - if b.memoryBytes < 0 { - b.memoryBytes = 0 - } - if b.diskBytes < 0 { - b.diskBytes = 0 - } - return b.totalBytes() <= b.lowWatermarkBytes -} - -func (b *budget) totalBytes() int64 { - return b.memoryBytes + b.diskBytes -} diff --git a/downstreamadapter/sink/cloudstorage/spool/budget_test.go b/downstreamadapter/sink/cloudstorage/spool/budget_test.go deleted file mode 100644 index 700df31145..0000000000 --- a/downstreamadapter/sink/cloudstorage/spool/budget_test.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package spool - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestBudgetTracksMemoryAndDiskBytes(t *testing.T) { - t.Parallel() - - core := newBudget(&options{ - diskQuotaBytes: 100, - memoryRatio: 0.2, - highWatermarkRatio: 0.8, - lowWatermarkRatio: 0.6, - }) - - require.False(t, core.shouldSpill(10)) - - overHighWatermark := core.acquire(10, false) - require.False(t, overHighWatermark) - require.Equal(t, int64(10), core.memoryBytes) - require.Equal(t, int64(0), core.diskBytes) - require.Equal(t, int64(10), core.totalBytes()) - - require.True(t, core.shouldSpill(11)) - - overHighWatermark = core.acquire(11, true) - require.False(t, overHighWatermark) - require.Equal(t, int64(10), core.memoryBytes) - require.Equal(t, int64(11), core.diskBytes) - require.Equal(t, int64(21), core.totalBytes()) - - atOrBelowLowWatermark := core.release(50, false) - require.True(t, atOrBelowLowWatermark) - require.Equal(t, int64(0), core.memoryBytes) - require.Equal(t, int64(11), core.diskBytes) - require.Equal(t, int64(11), core.totalBytes()) - - atOrBelowLowWatermark = core.release(50, true) - require.True(t, atOrBelowLowWatermark) - require.Equal(t, int64(0), core.memoryBytes) - require.Equal(t, int64(0), core.diskBytes) - require.Equal(t, int64(0), core.totalBytes()) -} - -func TestBudgetTracksWatermarkState(t *testing.T) { - t.Parallel() - - core := newBudget(&options{ - diskQuotaBytes: 100, - memoryRatio: 0.2, - highWatermarkRatio: 0.8, - lowWatermarkRatio: 0.6, - }) - - require.LessOrEqual(t, core.totalBytes(), core.lowWatermarkBytes) - require.LessOrEqual(t, core.totalBytes(), core.highWatermarkBytes) - - overHighWatermark := core.acquire(81, true) - require.True(t, overHighWatermark) - require.Greater(t, core.totalBytes(), core.lowWatermarkBytes) - - atOrBelowLowWatermark := core.release(21, true) - require.True(t, atOrBelowLowWatermark) - require.LessOrEqual(t, core.totalBytes(), core.highWatermarkBytes) -} diff --git a/downstreamadapter/sink/cloudstorage/spool_metrics.go b/downstreamadapter/sink/cloudstorage/spool_metrics.go new file mode 100644 index 0000000000..54e569e518 --- /dev/null +++ b/downstreamadapter/sink/cloudstorage/spool_metrics.go @@ -0,0 +1,45 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package cloudstorage + +import ( + sinkmetrics "github.com/pingcap/ticdc/downstreamadapter/sink/metrics" + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/sink/spool" +) + +func newSpoolMetrics(changefeedID common.ChangeFeedID) *spool.Metrics { + keyspace := changefeedID.Keyspace() + changefeed := changefeedID.Name() + return &spool.Metrics{ + MemoryBytes: sinkmetrics.CloudStorageSpoolMemoryBytesGauge.WithLabelValues(keyspace, changefeed), + DiskBytes: sinkmetrics.CloudStorageSpoolDiskBytesGauge.WithLabelValues(keyspace, changefeed), + PendingPostEnqueue: sinkmetrics.CloudStoragePendingPostEnqueueGauge.WithLabelValues(keyspace, changefeed), + DiskQuotaWaiters: sinkmetrics.CloudStorageSpoolDiskQuotaWaitersGauge.WithLabelValues(keyspace, changefeed), + DiskQuotaWait: sinkmetrics.CloudStorageSpoolDiskQuotaWaitDurationHistogram.WithLabelValues(keyspace, changefeed), + LoadedBytes: sinkmetrics.CloudStorageLoadBytesHistogram.WithLabelValues(keyspace, changefeed), + RotatedCount: sinkmetrics.CloudStorageRotateCountCounter.WithLabelValues(keyspace, changefeed), + SegmentCount: sinkmetrics.CloudStorageSpoolSegmentCountGauge.WithLabelValues(keyspace, changefeed), + Close: func() { + sinkmetrics.CloudStorageSpoolMemoryBytesGauge.DeleteLabelValues(keyspace, changefeed) + sinkmetrics.CloudStorageSpoolDiskBytesGauge.DeleteLabelValues(keyspace, changefeed) + sinkmetrics.CloudStoragePendingPostEnqueueGauge.DeleteLabelValues(keyspace, changefeed) + sinkmetrics.CloudStorageSpoolDiskQuotaWaitersGauge.DeleteLabelValues(keyspace, changefeed) + sinkmetrics.CloudStorageSpoolDiskQuotaWaitDurationHistogram.DeleteLabelValues(keyspace, changefeed) + sinkmetrics.CloudStorageLoadBytesHistogram.DeleteLabelValues(keyspace, changefeed) + sinkmetrics.CloudStorageRotateCountCounter.DeleteLabelValues(keyspace, changefeed) + sinkmetrics.CloudStorageSpoolSegmentCountGauge.DeleteLabelValues(keyspace, changefeed) + }, + } +} diff --git a/downstreamadapter/sink/cloudstorage/writer.go b/downstreamadapter/sink/cloudstorage/writer.go index e31a1387c6..635de4e78b 100644 --- a/downstreamadapter/sink/cloudstorage/writer.go +++ b/downstreamadapter/sink/cloudstorage/writer.go @@ -20,12 +20,12 @@ import ( "time" "github.com/pingcap/log" - "github.com/pingcap/ticdc/downstreamadapter/sink/cloudstorage/spool" "github.com/pingcap/ticdc/downstreamadapter/sink/metrics" "github.com/pingcap/ticdc/pkg/cloudstorage" "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" pmetrics "github.com/pingcap/ticdc/pkg/metrics" + "github.com/pingcap/ticdc/pkg/sink/spool" "github.com/pingcap/tidb/pkg/objstore/storeapi" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" diff --git a/downstreamadapter/sink/cloudstorage/writer_test.go b/downstreamadapter/sink/cloudstorage/writer_test.go index 7aaf02dd82..7499167519 100644 --- a/downstreamadapter/sink/cloudstorage/writer_test.go +++ b/downstreamadapter/sink/cloudstorage/writer_test.go @@ -26,7 +26,6 @@ import ( "testing" "time" - "github.com/pingcap/ticdc/downstreamadapter/sink/cloudstorage/spool" "github.com/pingcap/ticdc/pkg/cloudstorage" commonType "github.com/pingcap/ticdc/pkg/common" commonEvent "github.com/pingcap/ticdc/pkg/common/event" @@ -34,6 +33,7 @@ import ( "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/ticdc/pkg/pdutil" "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/pingcap/ticdc/pkg/sink/spool" "github.com/pingcap/ticdc/pkg/util" "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/objstore/objectio" diff --git a/downstreamadapter/sink/helper/row_callback.go b/downstreamadapter/sink/helper/row_callback.go index 9acbc6e671..c9fe9ab480 100644 --- a/downstreamadapter/sink/helper/row_callback.go +++ b/downstreamadapter/sink/helper/row_callback.go @@ -21,10 +21,16 @@ import ( // NewPostFlushRowCallback returns a row-level callback that triggers txn-level // PostFlush exactly once when the callback has been invoked totalCount times. func NewPostFlushRowCallback(event *event.DMLEvent, totalCount uint64) func() { + return NewRowCallback(totalCount, event.PostFlush) +} + +// NewRowCallback returns a row-level callback that triggers callback exactly +// once after it has been invoked totalCount times. +func NewRowCallback(totalCount uint64, callback func()) func() { var calledCount atomic.Uint64 return func() { if calledCount.Inc() == totalCount { - event.PostFlush() + callback() } } } diff --git a/downstreamadapter/sink/redo/meta_test.go b/downstreamadapter/sink/redo/meta_test.go index 6d6378c39f..9b5eac76d0 100644 --- a/downstreamadapter/sink/redo/meta_test.go +++ b/downstreamadapter/sink/redo/meta_test.go @@ -386,7 +386,7 @@ func TestPreStartClosesExternalStorageOnFailure(t *testing.T) { cfg := &config.ConsistentConfig{ Storage: util.AddressOf(storageURI.String()), - MetaFlushIntervalInMs: util.AddressOf(int64(redo.MinFlushIntervalInMs)), + MetaFlushIntervalInMs: util.AddressOf(int64(redo.DefaultMetaFlushIntervalInMs)), } m := NewRedoMeta(common.NewChangeFeedIDWithName("test-changefeed", common.DefaultKeyspaceName), 1, cfg) err = m.PreStart(context.Background()) diff --git a/downstreamadapter/sink/redo/sink.go b/downstreamadapter/sink/redo/sink.go index d19fbf9c1e..3e01849f28 100644 --- a/downstreamadapter/sink/redo/sink.go +++ b/downstreamadapter/sink/redo/sink.go @@ -25,7 +25,6 @@ import ( "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/redo" "github.com/pingcap/ticdc/pkg/redo/writer" - "github.com/pingcap/ticdc/pkg/redo/writer/factory" "github.com/pingcap/ticdc/pkg/util" "github.com/pingcap/ticdc/utils/chann" "go.uber.org/atomic" @@ -103,7 +102,7 @@ func New(ctx context.Context, changefeedID common.ChangeFeedID, } }() - ddlWriter, err = factory.NewRedoDDLWriter(ctx, config) + ddlWriter, err = writer.NewDDLWriter(ctx, config) if err != nil { log.Error("redo: failed to create redo log writer", zap.String("keyspace", changefeedID.Keyspace()), @@ -112,7 +111,7 @@ func New(ctx context.Context, changefeedID common.ChangeFeedID, zap.Error(err)) return nil, err } - dmlWriter, err = factory.NewRedoDMLWriter(ctx, config) + dmlWriter, err = writer.NewDMLWriter(ctx, config) if err != nil { log.Error("redo: failed to create redo log writer", zap.String("keyspace", changefeedID.Keyspace()), @@ -168,7 +167,9 @@ func (s *Sink) WriteBlockEvent(event commonEvent.BlockEvent) error { func (s *Sink) AddDMLEvent(event *commonEvent.DMLEvent) { rowsCount := event.Len() events := make([]*commonEvent.RedoRowEvent, 0, rowsCount) - rowCallback := helper.NewPostFlushRowCallback(event, uint64(rowsCount)) + postEnqueue, postFlush := event.DetachPostCallbacks() + rowPostEnqueue := helper.NewRowCallback(uint64(rowsCount), postEnqueue) + rowPostFlush := helper.NewRowCallback(uint64(rowsCount), postFlush) var ( startTs = event.GetStartTs() @@ -187,7 +188,8 @@ func (s *Sink) AddDMLEvent(event *commonEvent.DMLEvent) { Event: row, PhysicalTableID: physicalTableID, TableInfo: event.TableInfo, - Callback: rowCallback, + Callback: rowPostFlush, + EnqueueCallback: rowPostEnqueue, }) } s.logBuffer.Push(events...) @@ -237,29 +239,21 @@ func (s *Sink) Close() { } func (s *Sink) sendMessages(ctx context.Context) error { - buffer := make([]*commonEvent.RedoRowEvent, 0, redo.DefaultFlushBatchSize) for { - select { - case <-ctx.Done(): - return errors.Trace(context.Cause(ctx)) - default: + event, ok, err := s.logBuffer.GetWithContext(ctx) + if err != nil { + return errors.Trace(err) } - events, ok := s.logBuffer.GetMultipleNoGroup(buffer) if !ok { return nil } - if len(events) == 0 { - continue - } - buffer = events[:0] start := time.Now() - err := s.dmlWriter.AddDMLEvents(ctx, events...) - if err != nil { + if err := s.dmlWriter.AddDMLEvents(ctx, event); err != nil { return err } if s.metricCollector != nil { - s.metricCollector.observeRowWrite(len(events), time.Since(start)) + s.metricCollector.observeRowWrite(1, time.Since(start)) } } } diff --git a/downstreamadapter/sink/redo/sink_test.go b/downstreamadapter/sink/redo/sink_test.go index 1207572f4d..3f325cb5f2 100644 --- a/downstreamadapter/sink/redo/sink_test.go +++ b/downstreamadapter/sink/redo/sink_test.go @@ -119,6 +119,52 @@ func TestRedoSinkBatchConfig(t *testing.T) { require.Equal(t, int(32*redo.Megabyte), sink.BatchBytes()) } +func TestRedoSinkTwoStageAck(t *testing.T) { + helper := commonEvent.NewEventTestHelper(t) + defer helper.Close() + + helper.Tk().MustExec("use test") + job := helper.DDL2Job("create table t (id int primary key)") + require.NotNil(t, job) + event := helper.DML2Event("test", "t", "insert into t values (1), (2), (3)") + + callbacks := make([]string, 0, 2) + event.AddPostEnqueueFunc(func() { + callbacks = append(callbacks, "enqueue") + }) + event.AddPostFlushFunc(func() { + callbacks = append(callbacks, "flush") + }) + + sink := &Sink{ + ctx: context.Background(), + logBuffer: chann.NewUnlimitedChannelDefault[*commonEvent.RedoRowEvent](), + } + sink.AddDMLEvent(event) + require.Empty(t, callbacks) + + sink.logBuffer.Close() + rowEvents, ok := sink.logBuffer.GetMultipleNoGroup( + make([]*commonEvent.RedoRowEvent, 0, event.Len())) + require.True(t, ok) + require.Len(t, rowEvents, int(event.Len())) + + for _, rowEvent := range rowEvents[:len(rowEvents)-1] { + rowEvent.PostEnqueue() + } + require.Empty(t, callbacks) + rowEvents[len(rowEvents)-1].PostEnqueue() + require.Equal(t, []string{"enqueue"}, callbacks) + + for _, rowEvent := range rowEvents[:len(rowEvents)-1] { + rowEvent.PostFlush() + } + require.Equal(t, []string{"enqueue"}, callbacks) + + rowEvents[len(rowEvents)-1].PostFlush() + require.Equal(t, []string{"enqueue", "flush"}, callbacks) +} + // TestRedoSinkInProcessor tests how redo log manager is used in processor. func TestRedoSinkInProcessor(t *testing.T) { helper := commonEvent.NewEventTestHelper(t) @@ -340,7 +386,7 @@ func runBenchTest(b *testing.B, storage string, useFileBackend bool) { require.ErrorIs(b, eg.Wait(), context.Canceled) } -func TestRedoSinkSendMessagesInBatch(t *testing.T) { +func TestRedoSinkSendMessages(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) @@ -350,25 +396,13 @@ func TestRedoSinkSendMessagesInBatch(t *testing.T) { defer ctrl.Finish() mockWriter := writer.NewMockRedoDMLWriter(ctrl) - expectWriteBatch := func(batchSize int) *gomock.Call { - args := make([]interface{}, 0, batchSize+1) - args = append(args, gomock.Any()) // context - for range batchSize { - args = append(args, gomock.Any()) - } - return mockWriter.EXPECT(). - AddDMLEvents(args[0], args[1:]...). - DoAndReturn(func(_ context.Context, events ...*commonEvent.RedoRowEvent) error { - require.Len(t, events, batchSize) - return nil - }) - } - - gomock.InOrder( - expectWriteBatch(redo.DefaultFlushBatchSize), - expectWriteBatch(redo.DefaultFlushBatchSize), - expectWriteBatch(17), - ) + mockWriter.EXPECT(). + AddDMLEvents(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, events ...*commonEvent.RedoRowEvent) error { + require.Len(t, events, 1) + return nil + }). + Times(3) s := &Sink{ dmlWriter: mockWriter, @@ -380,9 +414,8 @@ func TestRedoSinkSendMessagesInBatch(t *testing.T) { doneCh <- s.sendMessages(ctx) }() - totalEvents := redo.DefaultFlushBatchSize*2 + 17 - events := make([]*commonEvent.RedoRowEvent, 0, totalEvents) - for range totalEvents { + events := make([]*commonEvent.RedoRowEvent, 0, 3) + for range 3 { events = append(events, &commonEvent.RedoRowEvent{}) } s.logBuffer.Push(events...) diff --git a/pkg/common/event/redo.go b/pkg/common/event/redo.go index 36b4906588..2ef562b28f 100644 --- a/pkg/common/event/redo.go +++ b/pkg/common/event/redo.go @@ -117,6 +117,7 @@ type RedoRowEvent struct { TableInfo *common.TableInfo Event RowChange Callback func() + EnqueueCallback func() } const ( @@ -132,6 +133,13 @@ func (r *RedoRowEvent) PostFlush() { } } +// PostEnqueue marks this encoded row as accepted by the redo spool. +func (r *RedoRowEvent) PostEnqueue() { + if r.EnqueueCallback != nil { + r.EnqueueCallback() + } +} + func (r *RedoRowEvent) ToRedoLog() *RedoLog { redoRow := &RedoDMLEvent{ Row: &DMLEventInRedoLog{ diff --git a/pkg/config/consistent.go b/pkg/config/consistent.go index eb151d7598..10a9bea2eb 100644 --- a/pkg/config/consistent.go +++ b/pkg/config/consistent.go @@ -15,6 +15,7 @@ package config import ( "fmt" + "path/filepath" "github.com/pingcap/ticdc/pkg/compression" "github.com/pingcap/ticdc/pkg/errors" @@ -52,9 +53,8 @@ type ConsistentConfig struct { FlushWorkerNum *int `toml:"flush-worker-num" json:"flush-worker-num,omitempty"` // Storage is the storage path(uri) to store redo log. Storage *string `toml:"storage" json:"storage,omitempty"` - // UseFileBackend is a flag to enable file backend for redo log. - // file backend means before flush redo log to storage, it will be written to local file. - // Default is false. + // UseFileBackend is retained for compatibility and ignored. Redo always uses + // the spooled memory writer. UseFileBackend *bool `toml:"use-file-backend" json:"use-file-backend,omitempty"` // Compression is the compression algorithm used for redo log. // Default is "", it means no compression, equals to `none`. @@ -64,6 +64,12 @@ type ConsistentConfig struct { // Default is 1. It means a single log file will be flushed by only one worker. // The singe file concurrent flushing feature supports only `s3` storage. FlushConcurrency *int `toml:"flush-concurrency" json:"flush-concurrency,omitempty"` + // SpoolDiskQuota is the disk quota in bytes for redo spool files. + // Default is 10 GiB. + SpoolDiskQuota *int64 `toml:"spool-disk-quota" json:"spool-disk-quota,omitempty"` + // SpoolBaseDir is the base directory for redo spool files. + // It must be an absolute path when configured. + SpoolBaseDir *string `toml:"spool-base-dir" json:"spool-base-dir,omitempty"` // MemoryUsage represents the percentage of ReplicaConfig.MemoryQuota // that can be utilized by the redo log module. MemoryUsage *ConsistentMemoryUsage `toml:"memory-usage" json:"memory-usage,omitempty"` @@ -138,6 +144,19 @@ func (c *ConsistentConfig) validateAndAdjust(enableIOCheck bool) error { c.FlushWorkerNum = util.AddressOf(redo.DefaultFlushWorkerNum) } + if c.SpoolDiskQuota == nil { + c.SpoolDiskQuota = util.AddressOf(redo.DefaultSpoolDiskQuota) + } else if *c.SpoolDiskQuota <= 0 { + return errors.ErrInvalidReplicaConfig.FastGenByArgs( + "consistent.spool-disk-quota must be greater than 0") + } + if c.SpoolBaseDir == nil { + c.SpoolBaseDir = util.AddressOf("") + } else if *c.SpoolBaseDir != "" && !filepath.IsAbs(*c.SpoolBaseDir) { + return errors.ErrInvalidReplicaConfig.FastGenByArgs( + "consistent.spool-base-dir must be an absolute path") + } + uri, err := objstore.ParseRawURL(util.GetOrZero(c.Storage)) if err != nil { return errors.ErrInvalidReplicaConfig.GenWithStackByArgs( diff --git a/pkg/config/replica_config.go b/pkg/config/replica_config.go index 06eff031d5..348c4efdd8 100644 --- a/pkg/config/replica_config.go +++ b/pkg/config/replica_config.go @@ -98,6 +98,8 @@ var defaultReplicaConfig = &ReplicaConfig{ Storage: util.AddressOf(""), UseFileBackend: util.AddressOf(false), Compression: util.AddressOf(""), + SpoolDiskQuota: util.AddressOf(redo.DefaultSpoolDiskQuota), + SpoolBaseDir: util.AddressOf(""), MemoryUsage: &ConsistentMemoryUsage{ MemoryQuotaPercentage: 50, }, diff --git a/pkg/config/replica_config_test.go b/pkg/config/replica_config_test.go index 3a5aba8b58..16499ce900 100644 --- a/pkg/config/replica_config_test.go +++ b/pkg/config/replica_config_test.go @@ -18,6 +18,7 @@ import ( "testing" "time" + "github.com/pingcap/ticdc/pkg/redo" "github.com/pingcap/ticdc/pkg/util" "github.com/stretchr/testify/require" ) @@ -247,6 +248,37 @@ func TestReplicaConfig_EnableRedoIOCheck_DefaultValue(t *testing.T) { require.True(t, util.GetOrZero(config.EnableRedoIOCheck)) } +func TestConsistentConfigSpoolSettings(t *testing.T) { + newConfig := func() *ConsistentConfig { + cfg := GetDefaultReplicaConfig().Consistent + cfg.Level = util.AddressOf(string(redo.ConsistentLevelEventual)) + cfg.Storage = util.AddressOf("blackhole://") + return cfg + } + + cfg := newConfig() + cfg.SpoolDiskQuota = nil + cfg.SpoolBaseDir = nil + require.NoError(t, cfg.validateAndAdjust(false)) + require.Equal(t, redo.DefaultSpoolDiskQuota, util.GetOrZero(cfg.SpoolDiskQuota)) + require.Empty(t, util.GetOrZero(cfg.SpoolBaseDir)) + + for _, quota := range []int64{0, -1} { + cfg = newConfig() + cfg.SpoolDiskQuota = util.AddressOf(quota) + require.ErrorContains(t, cfg.validateAndAdjust(false), "consistent.spool-disk-quota") + } + + cfg = newConfig() + cfg.SpoolBaseDir = util.AddressOf("relative/path") + require.ErrorContains(t, cfg.validateAndAdjust(false), "consistent.spool-base-dir") + + cfg = newConfig() + cfg.SpoolDiskQuota = util.AddressOf(int64(1024)) + cfg.SpoolBaseDir = util.AddressOf(t.TempDir()) + require.NoError(t, cfg.validateAndAdjust(false)) +} + func TestReplicaConfig_EnableRedoIOCheck_DefaultEnabled(t *testing.T) { config := GetDefaultReplicaConfig() config.Consistent.Level = util.AddressOf("eventual") diff --git a/pkg/config/server.go b/pkg/config/server.go index 8606f203a0..be5460b045 100644 --- a/pkg/config/server.go +++ b/pkg/config/server.go @@ -35,9 +35,6 @@ const ( // DefaultSortDir is the default value of sort-dir, it will be a subordinate directory of data-dir. DefaultSortDir = "/tmp/sorter" - // DefaultRedoDir is a subordinate directory path of data-dir. - DefaultRedoDir = "/tmp/redo" - // DebugConfigurationItem is the name of debug configurations DebugConfigurationItem = "debug" diff --git a/pkg/redo/config.go b/pkg/redo/config.go index 6cce071a54..16abdc4dfa 100644 --- a/pkg/redo/config.go +++ b/pkg/redo/config.go @@ -49,9 +49,8 @@ const ( DefaultMetaFlushIntervalInMs = 200 // MinFlushIntervalInMs is the minimum flush interval for redo log. MinFlushIntervalInMs = 50 - // DefaultFlushBatchSize is the default flush batch size for redo log. - DefaultFlushBatchSize = 1024 - + // DefaultSpoolDiskQuota is the default disk quota in bytes for redo spool files. + DefaultSpoolDiskQuota = int64(10 * 1024 * 1024 * 1024) // DefaultEncodingWorkerNum is the default number of encoding workers. DefaultEncodingWorkerNum = 16 // DefaultEncodingInputChanSize is the default size of input channel for encoding worker. diff --git a/pkg/redo/reader/file.go b/pkg/redo/reader/file.go index 073f259ab6..dba5f60c3a 100644 --- a/pkg/redo/reader/file.go +++ b/pkg/redo/reader/file.go @@ -21,7 +21,6 @@ import ( "context" "encoding/binary" "io" - "math" "net/url" "os" "path/filepath" @@ -35,8 +34,6 @@ import ( "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/redo" "github.com/pingcap/ticdc/pkg/redo/codec" - "github.com/pingcap/ticdc/pkg/redo/writer" - "github.com/pingcap/ticdc/pkg/redo/writer/file" "github.com/pingcap/tidb/pkg/objstore/storeapi" "go.uber.org/zap" "golang.org/x/sync/errgroup" @@ -245,12 +242,11 @@ func sortAndWriteFile( fileName string, cfg *readerConfig, ) error { sortedName := getSortedFileName(fileName) - w, err := file.NewLocalFileWriter(cfg.dir, math.MaxInt32, cfg.fileType, writer.WithLogFileName(func() string { - return sortedName - })) + w, err := newFramedFileWriter(filepath.Join(cfg.dir, sortedName)) if err != nil { return err } + defer w.Abort() fileContent, err := extStorage.ReadFile(egCtx, fileName) if err != nil { @@ -293,8 +289,7 @@ func sortAndWriteFile( if err != nil { return errors.WrapError(errors.ErrMarshalFailed, err) } - _, err = w.Write(data) - if err != nil { + if err = w.Write(data); err != nil { return err } } @@ -320,8 +315,7 @@ func shouldOpen(startTs uint64, name, fixedType string) (bool, error) { return commitTs > startTs, nil } -// Read implement Read interface. -// TODO: more general reader pair with writer in writer pkg +// Read implements the fileReader interface. func (r *reader) Read() (*pevent.RedoLog, error) { r.mu.Lock() defer r.mu.Unlock() @@ -367,7 +361,7 @@ func readInt64(r io.Reader) (int64, error) { return n, err } -// decodeFrameSize pair with encodeFrameSize in writer.file +// decodeFrameSize pairs with writer.EncodeFrameSize. // the func use code from etcd wal/decoder.go func decodeFrameSize(lenField int64) (recBytes int64, padBytes int64) { // the record size is stored in the lower 56 bits of the 64-bit length diff --git a/pkg/redo/reader/file_writer.go b/pkg/redo/reader/file_writer.go new file mode 100644 index 0000000000..5824adc013 --- /dev/null +++ b/pkg/redo/reader/file_writer.go @@ -0,0 +1,115 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package reader + +import ( + "encoding/binary" + "os" + "path/filepath" + + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/redo" + "github.com/pingcap/ticdc/pkg/redo/writer" + pioutil "go.etcd.io/etcd/pkg/v3/ioutil" +) + +// framedFileWriter writes reader-owned temporary sorted files in redo framing. +type framedFileWriter struct { + path string + tempPath string + file *os.File + writer *pioutil.PageWriter + lenBuf [8]byte +} + +func newFramedFileWriter(path string) (*framedFileWriter, error) { + if err := os.MkdirAll(filepath.Dir(path), redo.DefaultDirMode); err != nil { + return nil, errors.WrapError(errors.ErrRedoFileOp, err) + } + return &framedFileWriter{ + path: path, + tempPath: path + redo.TmpEXT, + }, nil +} + +func (w *framedFileWriter) Write(data []byte) error { + if w.file == nil { + file, err := os.OpenFile( + w.tempPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, redo.DefaultFileMode) + if err != nil { + return errors.WrapError(errors.ErrRedoFileOp, err) + } + w.file = file + w.writer = pioutil.NewPageWriter(file, redo.PageBytes, 0) + } + + lenField, padBytes := writer.EncodeFrameSize(len(data)) + binary.LittleEndian.PutUint64(w.lenBuf[:], lenField) + if _, err := w.writer.Write(w.lenBuf[:]); err != nil { + return errors.WrapError(errors.ErrRedoFileOp, err) + } + if _, err := w.writer.Write(data); err != nil { + return errors.WrapError(errors.ErrRedoFileOp, err) + } + if padBytes != 0 { + var padding [8]byte + if _, err := w.writer.Write(padding[:padBytes]); err != nil { + return errors.WrapError(errors.ErrRedoFileOp, err) + } + } + return nil +} + +func (w *framedFileWriter) Close() error { + if w.file == nil { + return nil + } + + if _, err := w.writer.FlushN(); err != nil { + w.Abort() + return errors.WrapError(errors.ErrRedoFileOp, err) + } + if err := w.file.Sync(); err != nil { + w.Abort() + return errors.WrapError(errors.ErrRedoFileOp, err) + } + if err := w.file.Close(); err != nil { + w.file = nil + _ = os.Remove(w.tempPath) + return errors.WrapError(errors.ErrRedoFileOp, err) + } + w.file = nil + if err := os.Rename(w.tempPath, w.path); err != nil { + _ = os.Remove(w.tempPath) + return errors.WrapError(errors.ErrRedoFileOp, err) + } + + dir, err := os.Open(filepath.Dir(w.path)) + if err != nil { + return errors.WrapError(errors.ErrRedoFileOp, err) + } + defer dir.Close() + if err := dir.Sync(); err != nil { + return errors.WrapError(errors.ErrRedoFileOp, err) + } + return nil +} + +func (w *framedFileWriter) Abort() { + if w.file != nil { + _ = w.file.Close() + w.file = nil + } + _ = os.Remove(w.tempPath) +} diff --git a/pkg/redo/reader/reader_test.go b/pkg/redo/reader/reader_test.go index 337a94ff91..c07d1e3278 100644 --- a/pkg/redo/reader/reader_test.go +++ b/pkg/redo/reader/reader_test.go @@ -28,10 +28,6 @@ import ( "github.com/pingcap/ticdc/pkg/redo" "github.com/pingcap/ticdc/pkg/redo/codec" misc "github.com/pingcap/ticdc/pkg/redo/common" - "github.com/pingcap/ticdc/pkg/redo/testutil" - "github.com/pingcap/ticdc/pkg/redo/writer" - "github.com/pingcap/ticdc/pkg/redo/writer/file" - "github.com/pingcap/ticdc/pkg/util" "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/objstore/mockobjstore" "github.com/pingcap/tidb/pkg/objstore/storeapi" @@ -42,22 +38,13 @@ import ( ) func genLogFile( - ctx context.Context, t *testing.T, + _ context.Context, t *testing.T, dir string, logType string, minCommitTs, maxCommitTs uint64, ) { - consistentCfg := testutil.NewConsistentConfig("file://" + dir) - consistentCfg.MaxLogSize = util.AddressOf(int64(1)) - cfg, err := writer.NewConfig( - common.NewChangeFeedIDWithName("reader-test", common.DefaultKeyspaceName), - consistentCfg, - ) - require.NoError(t, err) fileName := fmt.Sprintf(redo.RedoLogFileFormatV2, "capture", "default", "changefeed", logType, maxCommitTs, uuid.NewString(), redo.LogEXT) - w, err := file.NewFileWriter(ctx, cfg, logType, writer.WithLogFileName(func() string { - return fileName - })) + w, err := newFramedFileWriter(filepath.Join(dir, fileName)) require.Nil(t, err) switch logType { case redo.RedoRowLogFileType: @@ -72,8 +59,7 @@ func genLogFile( log := event.ToRedoLog() rawData, err := codec.MarshalRedoLog(log, nil) require.Nil(t, err) - _, err = w.Write(rawData) - require.Nil(t, err) + require.NoError(t, w.Write(rawData)) } case redo.RedoDDLLogFileType: event := &pevent.DDLEvent{ @@ -83,8 +69,7 @@ func genLogFile( log := event.ToRedoLog() rawData, err := codec.MarshalRedoLog(log, nil) require.Nil(t, err) - _, err = w.Write(rawData) - require.Nil(t, err) + require.NoError(t, w.Write(rawData)) } err = w.Close() require.Nil(t, err) diff --git a/pkg/redo/testutil/config.go b/pkg/redo/testutil/config.go index ef1cfe35b1..6930c0845d 100644 --- a/pkg/redo/testutil/config.go +++ b/pkg/redo/testutil/config.go @@ -24,7 +24,7 @@ func NewConsistentConfig(storage string) *config.ConsistentConfig { level := string(redo.ConsistentLevelEventual) maxLogSize := int64(redo.DefaultMaxLogSize) flushIntervalInMs := int64(redo.DefaultFlushIntervalInMs) - metaFlushIntervalInMs := int64(redo.MinFlushIntervalInMs) + metaFlushIntervalInMs := int64(redo.DefaultMetaFlushIntervalInMs) encodingWorkerNum := redo.DefaultEncodingWorkerNum flushWorkerNum := redo.DefaultFlushWorkerNum compressionType := compression.None @@ -40,5 +40,7 @@ func NewConsistentConfig(storage string) *config.ConsistentConfig { FlushWorkerNum: util.AddressOf(flushWorkerNum), Compression: util.AddressOf(compressionType), FlushConcurrency: util.AddressOf(flushConcurrency), + SpoolDiskQuota: util.AddressOf(redo.DefaultSpoolDiskQuota), + SpoolBaseDir: util.AddressOf(""), } } diff --git a/pkg/redo/writer/blackhole/writer.go b/pkg/redo/writer/blackhole_writer.go similarity index 85% rename from pkg/redo/writer/blackhole/writer.go rename to pkg/redo/writer/blackhole_writer.go index ec276ac9f2..aeb075a9ec 100644 --- a/pkg/redo/writer/blackhole/writer.go +++ b/pkg/redo/writer/blackhole_writer.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package blackhole +package writer import ( "context" @@ -19,13 +19,12 @@ import ( "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/common/event" - "github.com/pingcap/ticdc/pkg/redo/writer" "go.uber.org/zap" ) var ( - _ writer.RedoDMLWriter = (*blackHoleDMLWriter)(nil) - _ writer.RedoDDLWriter = (*blackHoleDDLWriter)(nil) + _ RedoDMLWriter = (*blackHoleDMLWriter)(nil) + _ RedoDDLWriter = (*blackHoleDDLWriter)(nil) ) // blackHoleSink defines a blackHole storage, it receives events and persists @@ -38,15 +37,13 @@ type blackHoleDDLWriter struct { invalid bool } -// NewDMLWriter creates a blackHole DML writer. -func NewDMLWriter(invalid bool) *blackHoleDMLWriter { +func newBlackHoleDMLWriter(invalid bool) *blackHoleDMLWriter { return &blackHoleDMLWriter{ invalid: invalid, } } -// NewDDLWriter creates a blackHole DDL writer. -func NewDDLWriter(invalid bool) *blackHoleDDLWriter { +func newBlackHoleDDLWriter(invalid bool) *blackHoleDDLWriter { return &blackHoleDDLWriter{ invalid: invalid, } @@ -76,6 +73,7 @@ func (bs *blackHoleDMLWriter) AddDMLEvents(_ context.Context, events ...*event.R log.Debug("write redo events", fields...) for _, e := range events { if e != nil { + e.PostEnqueue() e.PostFlush() } } diff --git a/pkg/redo/writer/config.go b/pkg/redo/writer/config.go index b9781f7e07..c8415167df 100644 --- a/pkg/redo/writer/config.go +++ b/pkg/redo/writer/config.go @@ -16,7 +16,6 @@ package writer import ( "fmt" "net/url" - "path/filepath" "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/config" @@ -28,34 +27,31 @@ import ( // Config is the config for redo log writer. type Config struct { - // Shared by file and memory backends for log file naming. + // Used for redo log file naming. captureID config.CaptureID - // Shared by file and memory backends for metrics and log file naming. + // Used for metrics and redo log file naming. changefeedID common.ChangeFeedID - // Shared by factory and both backends to initialize storage. + // Used to initialize redo storage. uri *url.URL - // Shared by file and memory backends as the rotate threshold. + // Used as the redo log file rotate threshold. maxLogSizeInBytes int64 - // Used by the factory to choose the file backend. - useFileBackend bool - - // Shared by file and memory backends as the flush ticker interval. + // Used as the flush ticker interval. flushIntervalInMs int64 // Used only by the memory backend for encoding workers. encodingWorkerNum int - // Shared by file and memory backends for worker fanout sizing. + // Used for flush worker fanout sizing. flushWorkerNum int // Used only by the memory backend for file compression. compression string // Used only by the memory backend for flush concurrency. flushConcurrency int - - // Used only by the file backend as the local writer directory. - dir string + // Used by the memory backend to configure local spool storage. + spoolDiskQuota int64 + spoolBaseDir string } // NewConfig builds the runtime writer config from an adjusted ConsistentConfig. @@ -75,50 +71,25 @@ func NewConfig(changefeedID common.ChangeFeedID, consistentCfg *config.Consisten changefeedID: changefeedID, uri: uri, maxLogSizeInBytes: util.GetOrZero(consistentCfg.MaxLogSize) * redo.Megabyte, - useFileBackend: util.GetOrZero(consistentCfg.UseFileBackend), flushIntervalInMs: util.GetOrZero(consistentCfg.FlushIntervalInMs), encodingWorkerNum: util.GetOrZero(consistentCfg.EncodingWorkerNum), flushWorkerNum: util.GetOrZero(consistentCfg.FlushWorkerNum), compression: util.GetOrZero(consistentCfg.Compression), flushConcurrency: util.GetOrZero(consistentCfg.FlushConcurrency), + spoolDiskQuota: util.GetOrZero(consistentCfg.SpoolDiskQuota), + spoolBaseDir: util.GetOrZero(consistentCfg.SpoolBaseDir), } - cfg.dir = newWriterDir(cfg) return cfg, nil } -// newWriterDir returns the local working directory only when a file writer will -// actually use it. Remote memory backend writes do not need a local directory. -// file:// uses the configured path directly, while remote file backend writes -// stage local files under the server data dir before uploading them. -func newWriterDir(cfg *Config) string { - if cfg == nil || cfg.uri == nil { - return "" - } - if !cfg.UseExternalStorage() { - return cfg.uri.Path - } - if cfg.uri.Scheme == "file" { - return cfg.uri.Path - } - if !cfg.useFileBackend { - return "" - } - return filepath.Join( - config.GetGlobalServerConfig().DataDir, - config.DefaultRedoDir, - cfg.changefeedID.Keyspace(), - cfg.changefeedID.Name(), - ) -} - func (cfg Config) String() string { uri := "" if cfg.uri != nil { uri = cfg.uri.String() } - return fmt.Sprintf("%s:%s:%s:%s:%d:%s:%t", + return fmt.Sprintf("%s:%s:%s:%d:%s:%t", cfg.changefeedID.Keyspace(), cfg.changefeedID.Name(), cfg.captureID, - cfg.dir, cfg.maxLogSizeInBytes, uri, cfg.UseExternalStorage()) + cfg.maxLogSizeInBytes, uri, cfg.UseExternalStorage()) } func (cfg *Config) CaptureID() config.CaptureID { @@ -137,18 +108,10 @@ func (cfg *Config) UseExternalStorage() bool { return cfg.uri != nil && redo.IsExternalStorage(cfg.uri.Scheme) } -func (cfg *Config) Dir() string { - return cfg.dir -} - func (cfg *Config) MaxLogSizeInBytes() int64 { return cfg.maxLogSizeInBytes } -func (cfg *Config) UseFileBackend() bool { - return cfg.useFileBackend -} - func (cfg *Config) FlushIntervalInMs() int64 { return cfg.flushIntervalInMs } @@ -168,3 +131,11 @@ func (cfg *Config) Compression() string { func (cfg *Config) FlushConcurrency() int { return cfg.flushConcurrency } + +func (cfg *Config) SpoolDiskQuota() int64 { + return cfg.spoolDiskQuota +} + +func (cfg *Config) SpoolBaseDir() string { + return cfg.spoolBaseDir +} diff --git a/pkg/redo/writer/factory/factory_test.go b/pkg/redo/writer/constructor_test.go similarity index 70% rename from pkg/redo/writer/factory/factory_test.go rename to pkg/redo/writer/constructor_test.go index c78021286d..aac9100211 100644 --- a/pkg/redo/writer/factory/factory_test.go +++ b/pkg/redo/writer/constructor_test.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package factory +package writer import ( "context" @@ -19,24 +19,23 @@ import ( "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/redo/testutil" - "github.com/pingcap/ticdc/pkg/redo/writer" "github.com/stretchr/testify/require" ) -func TestNewRedoWriters(t *testing.T) { +func TestNewWritersWithBlackholeStorage(t *testing.T) { t.Parallel() - cfg, err := writer.NewConfig( + cfg, err := NewConfig( common.NewChangeFeedIDWithName("test-changefeed", common.DefaultKeyspaceName), testutil.NewConsistentConfig("blackhole://"), ) require.NoError(t, err) - dmlWriter, err := NewRedoDMLWriter(context.Background(), cfg) + dmlWriter, err := NewDMLWriter(context.Background(), cfg) require.NoError(t, err) - require.Implements(t, (*writer.RedoDMLWriter)(nil), dmlWriter) + require.IsType(t, &blackHoleDMLWriter{}, dmlWriter) - ddlWriter, err := NewRedoDDLWriter(context.Background(), cfg) + ddlWriter, err := NewDDLWriter(context.Background(), cfg) require.NoError(t, err) - require.Implements(t, (*writer.RedoDDLWriter)(nil), ddlWriter) + require.IsType(t, &blackHoleDDLWriter{}, ddlWriter) } diff --git a/pkg/redo/writer/memory/ddl_writer.go b/pkg/redo/writer/ddl_writer.go similarity index 89% rename from pkg/redo/writer/memory/ddl_writer.go rename to pkg/redo/writer/ddl_writer.go index e21c0ebbfe..f5b9d6d0d3 100644 --- a/pkg/redo/writer/memory/ddl_writer.go +++ b/pkg/redo/writer/ddl_writer.go @@ -11,13 +11,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package memory +package writer import ( "bytes" "context" "encoding/binary" "fmt" + "strings" "sync" "time" @@ -30,20 +31,19 @@ import ( "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/ticdc/pkg/redo" "github.com/pingcap/ticdc/pkg/redo/codec" - "github.com/pingcap/ticdc/pkg/redo/writer" "github.com/pingcap/ticdc/pkg/uuid" "github.com/pingcap/tidb/pkg/objstore/storeapi" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" ) -var _ writer.RedoDDLWriter = (*ddlWriter)(nil) +var _ RedoDDLWriter = (*ddlWriter)(nil) type ddlWriter struct { mu sync.Mutex - cfg *writer.Config - op *writer.LogWriterOptions + cfg *Config + op *LogWriterOptions extStorage storeapi.Storage uuidGen uuid.Generator @@ -53,16 +53,26 @@ type ddlWriter struct { flushMetric prometheus.Observer } -// NewDDLWriter creates a new memory DDL writer. +// NewDDLWriter creates a new redo DDL writer. func NewDDLWriter( - ctx context.Context, cfg *writer.Config, opts ...writer.Option, -) (writer.RedoDDLWriter, error) { + ctx context.Context, cfg *Config, opts ...Option, +) (RedoDDLWriter, error) { + uri := cfg.URI() + if redo.IsBlackholeStorage(uri.Scheme) { + return newBlackHoleDDLWriter(strings.HasSuffix(uri.Scheme, "invalid")), nil + } + return newDDLWriter(ctx, cfg, opts...) +} + +func newDDLWriter( + ctx context.Context, cfg *Config, opts ...Option, +) (RedoDDLWriter, error) { extStorage, err := redo.InitExternalStorage(ctx, *cfg.URI()) if err != nil { return nil, err } - op := &writer.LogWriterOptions{} + op := &LogWriterOptions{} for _, opt := range opts { opt(op) } @@ -217,14 +227,14 @@ func toPolymorphicDDLEvent( if err != nil { return nil, err } - lenField, padBytes := writer.EncodeFrameSize(len(rawData)) + lenField, padBytes := EncodeFrameSize(len(rawData)) data := make([]byte, 8+len(rawData)+padBytes) binary.LittleEndian.PutUint64(data[:8], lenField) copy(data[8:], rawData) return &polymorphicRedoEvent{ - commitTs: rl.GetCommitTs(), - callback: event.PostFlush, - data: data, + commitTs: rl.GetCommitTs(), + postFlush: event.PostFlush, + data: data, }, nil } diff --git a/pkg/redo/writer/memory/ddl_writer_test.go b/pkg/redo/writer/ddl_writer_test.go similarity index 90% rename from pkg/redo/writer/memory/ddl_writer_test.go rename to pkg/redo/writer/ddl_writer_test.go index f80308c97c..d424082531 100644 --- a/pkg/redo/writer/memory/ddl_writer_test.go +++ b/pkg/redo/writer/ddl_writer_test.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package memory +package writer import ( "context" @@ -20,7 +20,6 @@ import ( "github.com/pingcap/ticdc/pkg/common" pevent "github.com/pingcap/ticdc/pkg/common/event" "github.com/pingcap/ticdc/pkg/redo/testutil" - "github.com/pingcap/ticdc/pkg/redo/writer" "github.com/pingcap/ticdc/pkg/util" "github.com/stretchr/testify/require" ) @@ -33,14 +32,14 @@ func TestWriteDDL(t *testing.T) { extStorage, uri, err := util.GetTestExtStorage(ctx, t.TempDir()) require.NoError(t, err) - cfg, err := writer.NewConfig( + cfg, err := NewConfig( common.NewChangeFeedIDWithName("test-changefeed", common.DefaultKeyspaceName), testutil.NewConsistentConfig(uri.String()), ) require.NoError(t, err) filename := t.Name() - lw, err := NewDDLWriter(ctx, cfg, writer.WithLogFileName(func() string { + lw, err := NewDDLWriter(ctx, cfg, WithLogFileName(func() string { return filename })) require.NoError(t, err) diff --git a/pkg/redo/writer/dml_writer.go b/pkg/redo/writer/dml_writer.go new file mode 100644 index 0000000000..e3126a9e5e --- /dev/null +++ b/pkg/redo/writer/dml_writer.go @@ -0,0 +1,239 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package writer + +import ( + "context" + "encoding/binary" + "os" + "path/filepath" + "strings" + + "github.com/pingcap/log" + commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/redo" + "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/pingcap/ticdc/pkg/sink/spool" + "github.com/pingcap/ticdc/utils/chann" + "github.com/pingcap/tidb/pkg/objstore/storeapi" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +var _ RedoDMLWriter = (*dmlWriter)(nil) + +type dmlWriter struct { + cfg *Config + encodeWorkers *encodingWorkerGroup + fileWorkers *fileWorkerGroup + spool *spool.Spool + spoolEntries *chann.UnlimitedChannel[*redoSpoolEntry, any] + extStorage storeapi.Storage + cancel context.CancelFunc +} + +type redoSpoolEntry struct { + entry *spool.Entry + flushImmediately bool +} + +const redoSpoolDirectory = "redo-sink-spool" + +// NewDMLWriter creates a new redo DML writer. +func NewDMLWriter( + ctx context.Context, cfg *Config, opts ...Option, +) (RedoDMLWriter, error) { + uri := cfg.URI() + if redo.IsBlackholeStorage(uri.Scheme) { + return newBlackHoleDMLWriter(strings.HasSuffix(uri.Scheme, "invalid")), nil + } + return newDMLWriter(ctx, cfg, opts...) +} + +func newDMLWriter( + ctx context.Context, cfg *Config, opts ...Option, +) (RedoDMLWriter, error) { + extStorage, err := redo.InitExternalStorage(ctx, *cfg.URI()) + if err != nil { + return nil, err + } + + encodeWorkers := newEncodingWorkerGroup(cfg) + fileWorkerInput := make(chan *polymorphicRedoEvent, redo.DefaultEncodingOutputChanSize) + fileWorkers := newFileWorkerGroup( + cfg, fileWorkerInput, extStorage, opts...) + spoolBaseDir := cfg.SpoolBaseDir() + if spoolBaseDir == "" { + spoolBaseDir = config.GetGlobalServerConfig().DataDir + if spoolBaseDir == "" { + spoolBaseDir = os.TempDir() + } + spoolBaseDir = filepath.Join(spoolBaseDir, redoSpoolDirectory) + } + spoolBuffer, err := spool.New( + cfg.ChangeFeedID(), + spool.WithRootDir(spoolBaseDir), + spool.WithDiskQuotaBytes(cfg.SpoolDiskQuota()), + ) + if err != nil { + extStorage.Close() + return nil, err + } + + return &dmlWriter{ + cfg: cfg, + encodeWorkers: encodeWorkers, + fileWorkers: fileWorkers, + spool: spoolBuffer, + spoolEntries: chann.NewUnlimitedChannelDefault[*redoSpoolEntry](), + extStorage: extStorage, + }, nil +} + +func (l *dmlWriter) Run(ctx context.Context) error { + newCtx, cancel := context.WithCancel(ctx) + l.cancel = cancel + + eg, egCtx := errgroup.WithContext(newCtx) + eg.Go(func() error { + return l.encodeWorkers.Run(egCtx) + }) + eg.Go(func() error { + return l.writeEncodedEventsToSpool(egCtx) + }) + eg.Go(func() error { + return l.readEncodedEventsFromSpool(egCtx) + }) + eg.Go(func() error { + return l.fileWorkers.Run(egCtx) + }) + return eg.Wait() +} + +func (l *dmlWriter) writeEncodedEventsToSpool(ctx context.Context) error { + for { + select { + case <-ctx.Done(): + return errors.Trace(context.Cause(ctx)) + case event := <-l.encodeWorkers.outputCh: + if event == nil { + return errors.ErrUnexpected.FastGenByArgs("encoded redo event is nil") + } + key := make([]byte, 8) + binary.LittleEndian.PutUint64(key, event.commitTs) + msg := common.NewMsg(key, event.data) + msg.Callback = event.postFlush + + for { + action, entry, err := l.spool.TryEnqueue( + []*common.Message{msg}, event.postEnqueue) + if err != nil { + return err + } + if action == spool.EnqueueActionWaitDiskQuota { + if err := l.spool.WaitForDiskQuota(ctx, []*common.Message{msg}); err != nil { + return err + } + continue + } + l.spoolEntries.Push(&redoSpoolEntry{ + entry: entry, + flushImmediately: action == spool.EnqueueActionAcceptedOversized, + }) + break + } + } + } +} + +func (l *dmlWriter) readEncodedEventsFromSpool(ctx context.Context) error { + for { + spooled, ok, err := l.spoolEntries.GetWithContext(ctx) + if err != nil { + return err + } + if !ok { + return nil + } + entry := spooled.entry + reader, err := l.spool.NewMessageReader(entry) + if err != nil { + return err + } + key, data, _, ok, err := reader.Next() + if err != nil { + return err + } + if !ok || len(key) != 8 || len(data) == 0 { + return errors.ErrUnexpected.FastGenByArgs("invalid encoded redo spool entry") + } + _, _, _, hasMore, err := reader.Next() + if err != nil { + return err + } + if hasMore { + return errors.ErrUnexpected.FastGenByArgs("encoded redo spool entry contains multiple messages") + } + postFlushCallbacks := reader.PostFlushCallbacks() + encodedEvent := &polymorphicRedoEvent{ + commitTs: binary.LittleEndian.Uint64(key), + data: data, + flushImmediately: spooled.flushImmediately, + postFlush: func() { + for _, callback := range postFlushCallbacks { + callback() + } + l.spool.Release(entry) + }, + } + select { + case <-ctx.Done(): + return errors.Trace(context.Cause(ctx)) + case l.fileWorkers.inputCh <- encodedEvent: + } + } +} + +func (l *dmlWriter) AddDMLEvents(ctx context.Context, events ...*commonEvent.RedoRowEvent) error { + for _, event := range events { + if event == nil { + log.Warn("writing nil event to redo log, ignore this", + zap.String("keyspace", l.cfg.ChangeFeedID().Keyspace()), + zap.String("changefeed", l.cfg.ChangeFeedID().Name())) + continue + } + if err := l.encodeWorkers.AddEvent(ctx, event); err != nil { + return err + } + } + return nil +} + +func (l *dmlWriter) Close() error { + if l.cancel != nil { + l.cancel() + l.cancel = nil + } + if l.extStorage != nil { + l.extStorage.Close() + l.extStorage = nil + } + if l.spool != nil { + l.spool.Close() + l.spool = nil + } + return nil +} diff --git a/pkg/redo/writer/dml_writer_test.go b/pkg/redo/writer/dml_writer_test.go new file mode 100644 index 0000000000..1b44a2491e --- /dev/null +++ b/pkg/redo/writer/dml_writer_test.go @@ -0,0 +1,168 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package writer + +import ( + "context" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/redo/testutil" + "github.com/pingcap/ticdc/pkg/sink/spool" + "github.com/pingcap/ticdc/pkg/util" + "github.com/pingcap/ticdc/utils/chann" + "github.com/stretchr/testify/require" +) + +func TestNewDMLWriter(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + _, uri, err := util.GetTestExtStorage(ctx, t.TempDir()) + require.NoError(t, err) + changefeedID := common.NewChangeFeedIDWithName("test-changefeed", common.DefaultKeyspaceName) + consistentCfg := testutil.NewConsistentConfig(uri.String()) + spoolBaseDir := t.TempDir() + consistentCfg.SpoolBaseDir = util.AddressOf(spoolBaseDir) + consistentCfg.SpoolDiskQuota = util.AddressOf(int64(1024)) + cfg, err := NewConfig(changefeedID, consistentCfg) + require.NoError(t, err) + + lw, err := NewDMLWriter(ctx, cfg) + require.NoError(t, err) + spoolDir := filepath.Join(spoolBaseDir, changefeedID.Keyspace(), changefeedID.Name()) + require.DirExists(t, spoolDir) + require.NoError(t, lw.Close()) + require.NoDirExists(t, spoolDir) +} + +func TestDMLWriterSpoolsEncodedBytesBeforePostEnqueue(t *testing.T) { + changefeedID := common.NewChangeFeedIDWithName(t.Name(), common.DefaultKeyspaceName) + spoolBuffer, err := spool.New( + changefeedID, + spool.WithRootDir(t.TempDir()), + spool.WithDiskQuotaBytes(1000), + spool.WithSegmentBytes(1<<20), + spool.WithMemoryRatio(0.2), + spool.WithHighWatermarkRatio(0.6), + spool.WithLowWatermarkRatio(0.3), + ) + require.NoError(t, err) + defer spoolBuffer.Close() + + encodedCh := make(chan *polymorphicRedoEvent, 2) + dmlWriter := &dmlWriter{ + encodeWorkers: &encodingWorkerGroup{outputCh: encodedCh}, + spool: spoolBuffer, + spoolEntries: chann.NewUnlimitedChannelDefault[*redoSpoolEntry](), + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + done <- dmlWriter.writeEncodedEventsToSpool(ctx) + }() + + var firstEnqueued atomic.Int64 + var secondEnqueued atomic.Int64 + firstData := []byte(strings.Repeat("a", 350)) + secondData := []byte(strings.Repeat("b", 350)) + encodedCh <- &polymorphicRedoEvent{ + commitTs: 1, + data: firstData, + postEnqueue: func() { firstEnqueued.Add(1) }, + } + encodedCh <- &polymorphicRedoEvent{ + commitTs: 2, + data: secondData, + postEnqueue: func() { secondEnqueued.Add(1) }, + } + + readCtx, readCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer readCancel() + firstEntry, ok, err := dmlWriter.spoolEntries.GetWithContext(readCtx) + require.NoError(t, err) + require.True(t, ok) + secondEntry, ok, err := dmlWriter.spoolEntries.GetWithContext(readCtx) + require.NoError(t, err) + require.True(t, ok) + + require.True(t, firstEntry.entry.IsSpilled()) + require.True(t, secondEntry.entry.IsSpilled()) + require.False(t, firstEntry.flushImmediately) + require.False(t, secondEntry.flushImmediately) + require.Equal(t, int64(1), firstEnqueued.Load()) + require.Equal(t, int64(0), secondEnqueued.Load()) + + reader, err := spoolBuffer.NewMessageReader(firstEntry.entry) + require.NoError(t, err) + _, encodedData, _, ok, err := reader.Next() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, firstData, encodedData) + + spoolBuffer.Release(firstEntry.entry) + require.Equal(t, int64(0), secondEnqueued.Load()) + spoolBuffer.Release(secondEntry.entry) + require.Equal(t, int64(1), secondEnqueued.Load()) + + cancel() + require.ErrorIs(t, <-done, context.Canceled) +} + +func TestDMLWriterMarksOversizedEncodedBytesForImmediateFlush(t *testing.T) { + changefeedID := common.NewChangeFeedIDWithName(t.Name(), common.DefaultKeyspaceName) + spoolBuffer, err := spool.New( + changefeedID, + spool.WithRootDir(t.TempDir()), + spool.WithDiskQuotaBytes(100), + ) + require.NoError(t, err) + defer spoolBuffer.Close() + + encodedCh := make(chan *polymorphicRedoEvent, 1) + dmlWriter := &dmlWriter{ + encodeWorkers: &encodingWorkerGroup{outputCh: encodedCh}, + spool: spoolBuffer, + spoolEntries: chann.NewUnlimitedChannelDefault[*redoSpoolEntry](), + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + done <- dmlWriter.writeEncodedEventsToSpool(ctx) + }() + + encodedCh <- &polymorphicRedoEvent{ + commitTs: 1, + data: []byte(strings.Repeat("a", 200)), + } + readCtx, readCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer readCancel() + entry, ok, err := dmlWriter.spoolEntries.GetWithContext(readCtx) + require.NoError(t, err) + require.True(t, ok) + require.True(t, entry.entry.InMemory()) + require.True(t, entry.flushImmediately) + spoolBuffer.Release(entry.entry) + + cancel() + require.ErrorIs(t, <-done, context.Canceled) +} diff --git a/pkg/redo/writer/memory/encoding_worker.go b/pkg/redo/writer/encoding_worker.go similarity index 90% rename from pkg/redo/writer/memory/encoding_worker.go rename to pkg/redo/writer/encoding_worker.go index 8787dea0c3..9be71546a9 100644 --- a/pkg/redo/writer/memory/encoding_worker.go +++ b/pkg/redo/writer/encoding_worker.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package memory +package writer import ( "context" @@ -23,7 +23,6 @@ import ( "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/redo" "github.com/pingcap/ticdc/pkg/redo/codec" - "github.com/pingcap/ticdc/pkg/redo/writer" "go.uber.org/atomic" "go.uber.org/zap" "golang.org/x/sync/errgroup" @@ -31,14 +30,16 @@ import ( // polymorphicRedoEvent wraps RedoLog and callback for file worker. type polymorphicRedoEvent struct { - commitTs common.Ts - data []byte - callback func() + commitTs common.Ts + data []byte + postEnqueue func() + postFlush func() + flushImmediately bool } func (e *polymorphicRedoEvent) PostFlush() { - if e.callback != nil { - e.callback() + if e.postFlush != nil { + e.postFlush() } } @@ -51,14 +52,15 @@ func toPolymorphicDMLEvent( if err != nil { return nil, errors.WrapError(errors.ErrMarshalFailed, err) } - lenField, padBytes := writer.EncodeFrameSize(len(rawData)) + lenField, padBytes := EncodeFrameSize(len(rawData)) data := make([]byte, 8+len(rawData)+padBytes) binary.LittleEndian.PutUint64(data[:8], lenField) copy(data[8:], rawData) return &polymorphicRedoEvent{ - commitTs: rl.GetCommitTs(), - callback: event.PostFlush, - data: data, + commitTs: rl.GetCommitTs(), + postEnqueue: event.PostEnqueue, + postFlush: event.PostFlush, + data: data, }, nil } @@ -73,7 +75,7 @@ type encodingWorkerGroup struct { closed chan error } -func newEncodingWorkerGroup(cfg *writer.Config) *encodingWorkerGroup { +func newEncodingWorkerGroup(cfg *Config) *encodingWorkerGroup { workerNum := cfg.EncodingWorkerNum() if workerNum <= 0 { workerNum = redo.DefaultEncodingWorkerNum diff --git a/pkg/redo/writer/memory/encoding_worker_test.go b/pkg/redo/writer/encoding_worker_test.go similarity index 86% rename from pkg/redo/writer/memory/encoding_worker_test.go rename to pkg/redo/writer/encoding_worker_test.go index 949f59348e..eabde9cc83 100644 --- a/pkg/redo/writer/memory/encoding_worker_test.go +++ b/pkg/redo/writer/encoding_worker_test.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package memory +package writer import ( "testing" @@ -19,7 +19,6 @@ import ( "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/redo" "github.com/pingcap/ticdc/pkg/redo/testutil" - "github.com/pingcap/ticdc/pkg/redo/writer" "github.com/pingcap/ticdc/pkg/util" "github.com/stretchr/testify/require" ) @@ -30,13 +29,13 @@ func TestNewEncodingWorkerGroup(t *testing.T) { changefeed := common.NewChangeFeedIDWithName("test-cf", common.DefaultKeyspaceName) cfg := testutil.NewConsistentConfig("nfs:///tmp/redo") cfg.EncodingWorkerNum = util.AddressOf(3) - writerCfg, err := writer.NewConfig(changefeed, cfg) + writerCfg, err := NewConfig(changefeed, cfg) require.NoError(t, err) g := newEncodingWorkerGroup(writerCfg) require.Equal(t, 3, g.workerNum) require.Len(t, g.inputChs, 3) - defaultCfg, err := writer.NewConfig(changefeed, testutil.NewConsistentConfig("nfs:///tmp/redo")) + defaultCfg, err := NewConfig(changefeed, testutil.NewConsistentConfig("nfs:///tmp/redo")) require.NoError(t, err) g = newEncodingWorkerGroup(defaultCfg) require.Equal(t, redo.DefaultEncodingWorkerNum, g.workerNum) diff --git a/pkg/redo/writer/factory/factory.go b/pkg/redo/writer/factory/factory.go deleted file mode 100644 index beba889ff7..0000000000 --- a/pkg/redo/writer/factory/factory.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2023 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package factory - -import ( - "context" - "strings" - - "github.com/pingcap/ticdc/pkg/redo" - "github.com/pingcap/ticdc/pkg/redo/writer" - "github.com/pingcap/ticdc/pkg/redo/writer/blackhole" - "github.com/pingcap/ticdc/pkg/redo/writer/file" - "github.com/pingcap/ticdc/pkg/redo/writer/memory" -) - -// NewRedoDMLWriter creates a new RedoDMLWriter. -func NewRedoDMLWriter( - ctx context.Context, cfg *writer.Config, -) (writer.RedoDMLWriter, error) { - uri := cfg.URI() - if redo.IsBlackholeStorage(uri.Scheme) { - invalid := strings.HasSuffix(uri.Scheme, "invalid") - return blackhole.NewDMLWriter(invalid), nil - } - - if cfg.UseFileBackend() { - return file.NewDMLWriter(ctx, cfg) - } - return memory.NewDMLWriter(ctx, cfg) -} - -// NewRedoDDLWriter creates a new RedoDDLWriter. -func NewRedoDDLWriter( - ctx context.Context, cfg *writer.Config, -) (writer.RedoDDLWriter, error) { - uri := cfg.URI() - if redo.IsBlackholeStorage(uri.Scheme) { - invalid := strings.HasSuffix(uri.Scheme, "invalid") - return blackhole.NewDDLWriter(invalid), nil - } - - if cfg.UseFileBackend() { - return file.NewDDLWriter(ctx, cfg) - } - return memory.NewDDLWriter(ctx, cfg) -} diff --git a/pkg/redo/writer/file/file.go b/pkg/redo/writer/file/file.go deleted file mode 100644 index 759d429337..0000000000 --- a/pkg/redo/writer/file/file.go +++ /dev/null @@ -1,622 +0,0 @@ -// Copyright 2023 PingCAP, Inc. -// Copyright 2015 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package file - -import ( - "context" - "encoding/binary" - "fmt" - "io" - "os" - "path/filepath" - "sync" - "time" - - "github.com/pingcap/ticdc/pkg/common" - commonEvent "github.com/pingcap/ticdc/pkg/common/event" - "github.com/pingcap/ticdc/pkg/config" - "github.com/pingcap/ticdc/pkg/errors" - "github.com/pingcap/ticdc/pkg/fsutil" - "github.com/pingcap/ticdc/pkg/metrics" - "github.com/pingcap/ticdc/pkg/redo" - "github.com/pingcap/ticdc/pkg/redo/codec" - "github.com/pingcap/ticdc/pkg/redo/writer" - "github.com/pingcap/ticdc/pkg/uuid" - "github.com/pingcap/tidb/pkg/objstore/storeapi" - "github.com/prometheus/client_golang/prometheus" - "github.com/uber-go/atomic" - pioutil "go.etcd.io/etcd/pkg/v3/ioutil" - "golang.org/x/sync/errgroup" -) - -//go:generate mockery --name=fileWriter --filename=file_mock.go --inpackage --quiet -type fileWriter interface { - Run(ctx context.Context) error - IsRunning() bool - SyncWrite(event writer.RedoEvent) error - GetInputCh() chan writer.RedoEvent - Flush() error - Close() error - SetTableSchemaStore(*commonEvent.TableSchemaStore) -} - -type fileWriterConfig interface { - CaptureID() config.CaptureID - ChangeFeedID() common.ChangeFeedID - Dir() string - MaxLogSizeInBytes() int64 - FlushIntervalInMs() int64 - FlushWorkerNum() int - UseExternalStorage() bool -} - -type localFileConfig struct { - dir string - maxLogSizeInBytes int64 - flushIntervalInMs int64 - flushWorkerNum int -} - -func (cfg *localFileConfig) CaptureID() config.CaptureID { - return "" -} - -func (cfg *localFileConfig) ChangeFeedID() common.ChangeFeedID { - return common.ChangeFeedID{} -} - -func (cfg *localFileConfig) Dir() string { - return cfg.dir -} - -func (cfg *localFileConfig) MaxLogSizeInBytes() int64 { - return cfg.maxLogSizeInBytes -} - -func (cfg *localFileConfig) FlushIntervalInMs() int64 { - return cfg.flushIntervalInMs -} - -func (cfg *localFileConfig) FlushWorkerNum() int { - return cfg.flushWorkerNum -} - -func (cfg *localFileConfig) UseExternalStorage() bool { - return false -} - -// fileWriter is a redo log event fileWriter which writes redo log events to a file. -type Writer struct { - cfg fileWriterConfig - logType string - op *writer.LogWriterOptions - inputCh chan writer.RedoEvent - // maxCommitTS is the max commitTS among the events in one log file - maxCommitTS atomic.Uint64 - // the ts used in file name - commitTS atomic.Uint64 - // the ts send with the event - eventCommitTS atomic.Uint64 - running atomic.Bool - size int64 - file *os.File - // record the filepath that is being written, and has not been flushed - ongoingFilePath string - bw *pioutil.PageWriter - uint64buf []byte - storage storeapi.Storage - sync.RWMutex - uuidGenerator uuid.Generator - allocator *fsutil.FileAllocator - - metricFsyncDuration prometheus.Observer - metricFlushAllDuration prometheus.Observer - metricWriteBytes prometheus.Gauge - tableSchemaStore *commonEvent.TableSchemaStore -} - -func newWriter( - cfg fileWriterConfig, - logType string, - extStorage storeapi.Storage, - opts ...writer.Option, -) (*Writer, error) { - op := &writer.LogWriterOptions{} - for _, opt := range opts { - opt(op) - } - - w := &Writer{ - cfg: cfg, - logType: logType, - op: op, - inputCh: make(chan writer.RedoEvent, redo.DefaultEncodingInputChanSize*cfg.FlushWorkerNum()), - uint64buf: make([]byte, 8), - storage: extStorage, - - metricFsyncDuration: metrics.RedoFsyncDurationHistogram. - WithLabelValues(cfg.ChangeFeedID().Keyspace(), cfg.ChangeFeedID().Name(), logType), - metricFlushAllDuration: metrics.RedoFlushAllDurationHistogram. - WithLabelValues(cfg.ChangeFeedID().Keyspace(), cfg.ChangeFeedID().Name(), logType), - metricWriteBytes: metrics.RedoWriteBytesGauge. - WithLabelValues(cfg.ChangeFeedID().Keyspace(), cfg.ChangeFeedID().Name(), logType), - } - if w.op.GetUUIDGenerator != nil { - w.uuidGenerator = w.op.GetUUIDGenerator() - } else { - w.uuidGenerator = uuid.NewGenerator() - } - - if len(cfg.Dir()) == 0 { - return nil, errors.WrapError(errors.ErrRedoFileOp, errors.New("invalid redo dir path")) - } - - err := os.MkdirAll(cfg.Dir(), redo.DefaultDirMode) - if err != nil { - return nil, errors.WrapError(errors.ErrRedoFileOp, - errors.Annotatef(err, "can't make dir: %s for redo writing", cfg.Dir())) - } - - if w.cfg.UseExternalStorage() { - w.allocator = fsutil.NewFileAllocator(cfg.Dir(), logType, cfg.MaxLogSizeInBytes()) - } - - w.running.Store(true) - return w, nil -} - -// NewFileWriter returns a file rotated writer for the normal redo writer path. -func NewFileWriter( - ctx context.Context, cfg *writer.Config, logType string, opts ...writer.Option, -) (w *Writer, err error) { - var extStorage storeapi.Storage - if cfg.UseExternalStorage() { - extStorage, err = redo.InitExternalStorage(ctx, *cfg.URI()) - if err != nil { - return nil, err - } - } - return newWriter(cfg, logType, extStorage, opts...) -} - -// NewLocalFileWriter is used by reader-side local sorting. It keeps the -// temporary sorted-file path local and avoids the external-storage write path. -func NewLocalFileWriter( - dir string, - maxLogSizeInBytes int64, - logType string, - opts ...writer.Option, -) (*Writer, error) { - return newWriter(&localFileConfig{ - dir: dir, - maxLogSizeInBytes: maxLogSizeInBytes, - flushIntervalInMs: redo.DefaultFlushIntervalInMs, - flushWorkerNum: redo.DefaultFlushWorkerNum, - }, logType, nil, opts...) -} - -func (w *Writer) SetTableSchemaStore(tableSchemaStore *commonEvent.TableSchemaStore) { - w.tableSchemaStore = tableSchemaStore -} - -func (w *Writer) Run(ctx context.Context) error { - eg, ctx := errgroup.WithContext(ctx) - eg.Go(func() error { - return w.encode(ctx) - }) - return eg.Wait() -} - -// Write implement write interface -// TODO: more general api with fileName generated by caller -func (w *Writer) Write(rawData []byte) (int, error) { - w.Lock() - defer w.Unlock() - - writeLen := int64(len(rawData)) - if writeLen > w.cfg.MaxLogSizeInBytes() { - return 0, errors.ErrRedoFileSizeExceed.GenWithStackByArgs(writeLen, w.cfg.MaxLogSizeInBytes()) - } - - if w.file == nil { - if err := w.openNew(); err != nil { - return 0, err - } - } - - if w.size+writeLen > w.cfg.MaxLogSizeInBytes() { - if err := w.rotate(); err != nil { - return 0, err - } - } - - if w.maxCommitTS.Load() < w.eventCommitTS.Load() { - w.maxCommitTS.Store(w.eventCommitTS.Load()) - } - // ref: https://github.com/etcd-io/etcd/pull/5250 - lenField, padBytes := writer.EncodeFrameSize(len(rawData)) - if err := w.writeUint64(lenField, w.uint64buf); err != nil { - return 0, err - } - - if padBytes != 0 { - rawData = append(rawData, make([]byte, padBytes)...) - } - - n, err := w.bw.Write(rawData) - if err != nil { - return 0, err - } - w.metricWriteBytes.Add(float64(n)) - w.size += int64(n) - - return n, err -} - -// AdvanceTs implement Advance interface -func (w *Writer) AdvanceTs(commitTs uint64) { - w.eventCommitTS.Store(commitTs) -} - -func (w *Writer) writeUint64(n uint64, buf []byte) error { - binary.LittleEndian.PutUint64(buf, n) - v, err := w.bw.Write(buf) - w.metricWriteBytes.Add(float64(v)) - - return err -} - -// Close implements fileWriter.Close. -func (w *Writer) Close() error { - w.Lock() - defer w.Unlock() - // always set to false when closed, since if having err may not get fixed just by retry - defer w.running.Store(false) - defer func() { - if w.storage != nil { - w.storage.Close() - w.storage = nil - } - }() - - if w.allocator != nil { - w.allocator.Close() - w.allocator = nil - } - - if !w.IsRunning() { - return nil - } - - metrics.RedoFlushAllDurationHistogram. - DeleteLabelValues(w.cfg.ChangeFeedID().Keyspace(), w.cfg.ChangeFeedID().Name(), w.logType) - metrics.RedoFsyncDurationHistogram. - DeleteLabelValues(w.cfg.ChangeFeedID().Keyspace(), w.cfg.ChangeFeedID().Name(), w.logType) - metrics.RedoWriteBytesGauge. - DeleteLabelValues(w.cfg.ChangeFeedID().Keyspace(), w.cfg.ChangeFeedID().Name(), w.logType) - - ctx, cancel := context.WithTimeout(context.Background(), redo.CloseTimeout) - defer cancel() - return w.close(ctx) -} - -// IsRunning implement IsRunning interface -func (w *Writer) IsRunning() bool { - return w.running.Load() -} - -func (w *Writer) GetInputCh() chan writer.RedoEvent { - return w.inputCh -} - -func (w *Writer) write(event writer.RedoEvent) error { - rl := event.ToRedoLog() - if rl.Type == commonEvent.RedoLogTypeDDL { - rl.RedoDDL.SetTableSchemaStore(w.tableSchemaStore) - } - data, err := codec.MarshalRedoLog(rl, nil) - if err != nil { - return errors.WrapError(errors.ErrMarshalFailed, err) - } - w.AdvanceTs(rl.GetCommitTs()) - _, err = w.Write(data) - if err != nil { - return err - } - return nil -} - -func (w *Writer) SyncWrite(event writer.RedoEvent) error { - err := w.write(event) - if err != nil { - return err - } - err = w.Flush() - if err != nil { - return errors.Trace(err) - } - event.PostFlush() - return nil -} - -func (w *Writer) encode(ctx context.Context) error { - d := time.Duration(w.cfg.FlushIntervalInMs()) * time.Millisecond - ticker := time.NewTicker(d) - defer ticker.Stop() - num := 0 - cacheEventPostFlush := make([]func(), 0, redo.DefaultFlushBatchSize) - flush := func() error { - err := w.Flush() - if err != nil { - return err - } - for _, fn := range cacheEventPostFlush { - fn() - } - num = 0 - cacheEventPostFlush = cacheEventPostFlush[:0] - return nil - } - for { - select { - case <-ctx.Done(): - return ctx.Err() - case <-ticker.C: - err := flush() - if err != nil { - return errors.Trace(err) - } - case e := <-w.inputCh: - err := w.write(e) - if err != nil { - return err - } - num++ - if num >= redo.DefaultFlushBatchSize { - err := flush() - if err != nil { - return errors.Trace(err) - } - e.PostFlush() - } else { - cacheEventPostFlush = append(cacheEventPostFlush, e.PostFlush) - } - } - } -} - -func (w *Writer) close(ctx context.Context) error { - if w.file == nil { - return nil - } - - if err := w.flush(); err != nil { - return err - } - - if w.cfg.UseExternalStorage() { - off, err := w.file.Seek(0, io.SeekCurrent) - if err != nil { - return err - } - // offset equals to 0 means that no written happened for current file, - // we can simply return - if off == 0 { - return nil - } - // a file created by a file allocator needs to be truncated - // to save disk space and network bandwidth. - if err := w.file.Truncate(off); err != nil { - return err - } - } - - // rename the file name from commitTs.log.tmp to maxCommitTS.log if closed safely - // after rename, the file name could be used for search, since the ts is the max ts for all events in the file. - w.commitTS.Store(w.maxCommitTS.Load()) - err := os.Rename(w.file.Name(), w.filePath()) - if err != nil { - return errors.WrapError(errors.ErrRedoFileOp, err) - } - - dirFile, err := os.Open(w.cfg.Dir()) - if err != nil { - return errors.WrapError(errors.ErrRedoFileOp, err) - } - defer dirFile.Close() - // sync the dir to guarantee the renamed file is persisted to disk. - err = dirFile.Sync() - if err != nil { - return errors.WrapError(errors.ErrRedoFileOp, err) - } - - // We only write content to S3 before closing the local file. - // By this way, we no longer need renaming object in S3. - if w.cfg.UseExternalStorage() { - err = w.writeToS3(ctx, w.ongoingFilePath) - if err != nil { - w.file.Close() - w.file = nil - return errors.WrapError(errors.ErrExternalStorageAPI, err) - } - } - - err = w.file.Close() - w.file = nil - return errors.WrapError(errors.ErrRedoFileOp, err) -} - -func (w *Writer) getLogFileName() string { - if w.op != nil && w.op.GetLogFileName != nil { - return w.op.GetLogFileName() - } - uid := w.uuidGenerator.NewString() - if common.DefaultKeyspaceName == w.cfg.ChangeFeedID().Keyspace() { - return fmt.Sprintf(redo.RedoLogFileFormatV1, - w.cfg.CaptureID(), w.cfg.ChangeFeedID().Name(), w.logType, - w.commitTS.Load(), uid, redo.LogEXT) - } - return fmt.Sprintf(redo.RedoLogFileFormatV2, - w.cfg.CaptureID(), w.cfg.ChangeFeedID().Keyspace(), w.cfg.ChangeFeedID().Name(), - w.logType, w.commitTS.Load(), uid, redo.LogEXT) -} - -// filePath always creates a new, unique file path, note this function is not -// thread-safe, writer needs to ensure lock is acquired when calling it. -func (w *Writer) filePath() string { - fp := filepath.Join(w.cfg.Dir(), w.getLogFileName()) - w.ongoingFilePath = fp - return fp -} - -func openTruncFile(name string) (*os.File, error) { - return os.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, redo.DefaultFileMode) -} - -func (w *Writer) openNew() error { - err := os.MkdirAll(w.cfg.Dir(), redo.DefaultDirMode) - if err != nil { - return errors.WrapError(errors.ErrRedoFileOp, - errors.Annotatef(err, "can't make dir: %s for new redo logfile", w.cfg.Dir())) - } - - // reset ts used in file name when new file - var f *os.File - if w.allocator == nil { - w.commitTS.Store(w.eventCommitTS.Load()) - w.maxCommitTS.Store(w.eventCommitTS.Load()) - path := w.filePath() + redo.TmpEXT - f, err = openTruncFile(path) - if err != nil { - return errors.WrapError(errors.ErrRedoFileOp, - errors.Annotate(err, "can't open new redolog file")) - } - } else { - // if there is a file allocator, we use the pre-created file - // supplied by the allocator to boost performance - f, err = w.allocator.Open() - if err != nil { - return errors.WrapError(errors.ErrRedoFileOp, - errors.Annotate(err, "can't open new redolog file with file allocator")) - } - } - w.file = f - w.size = 0 - err = w.newPageWriter() - if err != nil { - return err - } - return nil -} - -func (w *Writer) newPageWriter() error { - offset, err := w.file.Seek(0, io.SeekCurrent) - if err != nil { - return errors.WrapError(errors.ErrRedoFileOp, err) - } - w.bw = pioutil.NewPageWriter(w.file, redo.PageBytes, int(offset)) - - return nil -} - -func (w *Writer) rotate() error { - ctx, cancel := context.WithTimeout(context.Background(), redo.DefaultTimeout) - defer cancel() - if err := w.close(ctx); err != nil { - return err - } - return w.openNew() -} - -// flushAndRotateFile flushes the file to disk and rotate it if S3 storage is used. -func (w *Writer) flushAndRotateFile() error { - if w.file == nil { - return nil - } - - start := time.Now() - err := w.flush() - if err != nil { - return err - } - - if !w.cfg.UseExternalStorage() { - return nil - } - - if w.size == 0 { - return nil - } - - // for s3 storage, when the file is flushed to disk, we need an immediate - // file rotate. Otherwise, the existing file content would be repeatedly written to S3, - // which could cause considerable network bandwidth waste. - err = w.rotate() - if err != nil { - return err - } - w.metricFlushAllDuration.Observe(time.Since(start).Seconds()) - - return err -} - -// Flush implement Flush interface -func (w *Writer) Flush() error { - w.Lock() - defer w.Unlock() - - return w.flushAndRotateFile() -} - -func (w *Writer) flush() error { - if w.file == nil { - return nil - } - - n, err := w.bw.FlushN() - w.metricWriteBytes.Add(float64(n)) - if err != nil { - return errors.WrapError(errors.ErrRedoFileOp, err) - } - - start := time.Now() - err = w.file.Sync() - w.metricFsyncDuration.Observe(time.Since(start).Seconds()) - - return errors.WrapError(errors.ErrRedoFileOp, err) -} - -func (w *Writer) writeToS3(ctx context.Context, name string) error { - fileData, err := os.ReadFile(name) - if err != nil { - return errors.WrapError(errors.ErrRedoFileOp, err) - } - - // Key in s3: aws.String(rs.options.Prefix + name), prefix should be changefeed name - err = w.storage.WriteFile(ctx, filepath.Base(name), fileData) - if err != nil { - return errors.WrapError(errors.ErrExternalStorageAPI, err) - } - - // in case the page cache piling up triggered the OS memory reclaming which may cause - // I/O latency spike, we mandatorily drop the page cache of the file when it is successfully - // written to S3. - err = fsutil.DropPageCache(name) - if err != nil { - return errors.WrapError(errors.ErrRedoFileOp, err) - } - - return nil -} diff --git a/pkg/redo/writer/file/file_log_writer.go b/pkg/redo/writer/file/file_log_writer.go deleted file mode 100644 index 1fedb9b0c8..0000000000 --- a/pkg/redo/writer/file/file_log_writer.go +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright 2023 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package file - -import ( - "context" - - "github.com/pingcap/log" - commonEvent "github.com/pingcap/ticdc/pkg/common/event" - "github.com/pingcap/ticdc/pkg/errors" - "github.com/pingcap/ticdc/pkg/redo" - "github.com/pingcap/ticdc/pkg/redo/writer" - "go.uber.org/zap" -) - -var ( - _ writer.RedoDMLWriter = (*dmlWriter)(nil) - _ writer.RedoDDLWriter = (*ddlWriter)(nil) -) - -type logWriter struct { - cfg *writer.Config - backendWriter fileWriter -} - -type dmlWriter struct { - *logWriter -} - -type ddlWriter struct { - *logWriter -} - -func newLogWriter( - ctx context.Context, cfg *writer.Config, fileType string, opts ...writer.Option, -) (l *logWriter, err error) { - l = &logWriter{cfg: cfg} - if l.backendWriter, err = NewFileWriter(ctx, cfg, fileType, opts...); err != nil { - return nil, err - } - return -} - -// NewDMLWriter creates a new file DML writer. -func NewDMLWriter( - ctx context.Context, cfg *writer.Config, opts ...writer.Option, -) (writer.RedoDMLWriter, error) { - l, err := newLogWriter(ctx, cfg, redo.RedoRowLogFileType, opts...) - if err != nil { - return nil, err - } - return &dmlWriter{logWriter: l}, nil -} - -// NewDDLWriter creates a new file DDL writer. -func NewDDLWriter( - ctx context.Context, cfg *writer.Config, opts ...writer.Option, -) (writer.RedoDDLWriter, error) { - l, err := newLogWriter(ctx, cfg, redo.RedoDDLLogFileType, opts...) - if err != nil { - return nil, err - } - return &ddlWriter{logWriter: l}, nil -} - -func (l *logWriter) SetTableSchemaStore(tableSchemaStore *commonEvent.TableSchemaStore) { - l.backendWriter.SetTableSchemaStore(tableSchemaStore) -} - -func (l *logWriter) Run(ctx context.Context) error { - return l.backendWriter.Run(ctx) -} - -func (l *dmlWriter) AddDMLEvents(ctx context.Context, events ...*commonEvent.RedoRowEvent) error { - for _, event := range events { - if event == nil { - log.Warn("writing nil event to redo log, ignore this", - zap.String("keyspace", l.cfg.ChangeFeedID().Keyspace()), - zap.String("changefeed", l.cfg.ChangeFeedID().Name()), - zap.String("capture", l.cfg.CaptureID())) - continue - } - select { - case <-ctx.Done(): - return ctx.Err() - case l.backendWriter.GetInputCh() <- event: - } - } - return nil -} - -func (l *ddlWriter) WriteDDLEvent(ctx context.Context, event *commonEvent.DDLEvent) error { - select { - case <-ctx.Done(): - return errors.Trace(ctx.Err()) - default: - } - - if l.isStopped() { - return errors.ErrRedoWriterStopped.GenWithStackByArgs() - } - if event == nil { - log.Warn("writing nil event to redo log, ignore this", - zap.String("keyspace", l.cfg.ChangeFeedID().Keyspace()), - zap.String("changefeed", l.cfg.ChangeFeedID().Name()), - zap.String("capture", l.cfg.CaptureID())) - return nil - } - if err := l.backendWriter.SyncWrite(event); err != nil { - return errors.Trace(err) - } - return nil -} - -func (l *logWriter) Close() (err error) { - return l.backendWriter.Close() -} - -func (l *logWriter) isStopped() bool { - return !l.backendWriter.IsRunning() -} diff --git a/pkg/redo/writer/file/file_log_writer_test.go b/pkg/redo/writer/file/file_log_writer_test.go deleted file mode 100644 index a83dc5a2dc..0000000000 --- a/pkg/redo/writer/file/file_log_writer_test.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2023 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package file - -import ( - "context" - "testing" - - "github.com/pingcap/ticdc/pkg/common" - pevent "github.com/pingcap/ticdc/pkg/common/event" - "github.com/pingcap/ticdc/pkg/errors" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" -) - -func TestLogWriterWriteDDL(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - ctx context.Context - ddl *pevent.DDLEvent - isRunning bool - writerErr error - wantErr error - }{ - { - name: "happy", - ctx: context.Background(), - ddl: &pevent.DDLEvent{FinishedTs: 1}, - isRunning: true, - writerErr: nil, - }, - { - name: "writer err", - ctx: context.Background(), - ddl: &pevent.DDLEvent{FinishedTs: 1}, - writerErr: errors.New("err"), - wantErr: errors.New("err"), - isRunning: true, - }, - { - name: "ddl nil", - ctx: context.Background(), - ddl: nil, - writerErr: errors.New("err"), - isRunning: true, - }, - { - name: "isStopped", - ctx: context.Background(), - ddl: &pevent.DDLEvent{FinishedTs: 1}, - writerErr: errors.ErrRedoWriterStopped, - isRunning: false, - wantErr: errors.ErrRedoWriterStopped, - }, - { - name: "context cancel", - ctx: context.Background(), - ddl: &pevent.DDLEvent{FinishedTs: 1}, - writerErr: nil, - isRunning: true, - wantErr: context.Canceled, - }, - } - - for _, tt := range tests { - mockWriter := &mockFileWriter{} - mockWriter.On("IsRunning").Return(tt.isRunning) - mockWriter.On("SyncWrite", mock.Anything).Return(tt.writerErr) - w := ddlWriter{logWriter: &logWriter{ - cfg: newTestWriterConfig(t, common.ChangeFeedID{}, nil), - backendWriter: mockWriter, - }} - - if tt.name == "context cancel" { - ctx, cancel := context.WithCancel(context.Background()) - cancel() - tt.ctx = ctx - } - - err := w.WriteDDLEvent(tt.ctx, tt.ddl) - if tt.wantErr != nil { - require.Equal(t, tt.wantErr.Error(), err.Error(), tt.name) - } else { - require.Nil(t, err, tt.name) - } - } -} diff --git a/pkg/redo/writer/file/file_mock.go b/pkg/redo/writer/file/file_mock.go deleted file mode 100644 index 40679247ce..0000000000 --- a/pkg/redo/writer/file/file_mock.go +++ /dev/null @@ -1,146 +0,0 @@ -// Code generated by mockery v2.53.4. DO NOT EDIT. - -package file - -import ( - context "context" - - event "github.com/pingcap/ticdc/pkg/common/event" - mock "github.com/stretchr/testify/mock" - - writer "github.com/pingcap/ticdc/pkg/redo/writer" -) - -// mockFileWriter is an autogenerated mock type for the fileWriter type -type mockFileWriter struct { - mock.Mock -} - -// Close provides a mock function with no fields -func (_m *mockFileWriter) Close() error { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for Close") - } - - var r0 error - if rf, ok := ret.Get(0).(func() error); ok { - r0 = rf() - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Flush provides a mock function with no fields -func (_m *mockFileWriter) Flush() error { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for Flush") - } - - var r0 error - if rf, ok := ret.Get(0).(func() error); ok { - r0 = rf() - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// GetInputCh provides a mock function with no fields -func (_m *mockFileWriter) GetInputCh() chan writer.RedoEvent { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for GetInputCh") - } - - var r0 chan writer.RedoEvent - if rf, ok := ret.Get(0).(func() chan writer.RedoEvent); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(chan writer.RedoEvent) - } - } - - return r0 -} - -// IsRunning provides a mock function with no fields -func (_m *mockFileWriter) IsRunning() bool { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for IsRunning") - } - - var r0 bool - if rf, ok := ret.Get(0).(func() bool); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -// Run provides a mock function with given fields: ctx -func (_m *mockFileWriter) Run(ctx context.Context) error { - ret := _m.Called(ctx) - - if len(ret) == 0 { - panic("no return value specified for Run") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context) error); ok { - r0 = rf(ctx) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// SetTableSchemaStore provides a mock function with given fields: _a0 -func (_m *mockFileWriter) SetTableSchemaStore(_a0 *event.TableSchemaStore) { - _m.Called(_a0) -} - -// SyncWrite provides a mock function with given fields: _a0 -func (_m *mockFileWriter) SyncWrite(_a0 writer.RedoEvent) error { - ret := _m.Called(_a0) - - if len(ret) == 0 { - panic("no return value specified for SyncWrite") - } - - var r0 error - if rf, ok := ret.Get(0).(func(writer.RedoEvent) error); ok { - r0 = rf(_a0) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// newMockFileWriter creates a new instance of mockFileWriter. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func newMockFileWriter(t interface { - mock.TestingT - Cleanup(func()) -}) *mockFileWriter { - mock := &mockFileWriter{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/pkg/redo/writer/file/file_test.go b/pkg/redo/writer/file/file_test.go deleted file mode 100644 index 02f890ec47..0000000000 --- a/pkg/redo/writer/file/file_test.go +++ /dev/null @@ -1,499 +0,0 @@ -// Copyright 2023 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package file - -import ( - "context" - "fmt" - "math" - "os" - "path/filepath" - "testing" - "time" - - "github.com/pingcap/ticdc/pkg/common" - pevent "github.com/pingcap/ticdc/pkg/common/event" - "github.com/pingcap/ticdc/pkg/config" - "github.com/pingcap/ticdc/pkg/fsutil" - "github.com/pingcap/ticdc/pkg/metrics" - "github.com/pingcap/ticdc/pkg/redo" - "github.com/pingcap/ticdc/pkg/redo/writer" - "github.com/pingcap/ticdc/pkg/util" - "github.com/pingcap/ticdc/pkg/uuid" - "github.com/pingcap/tidb/pkg/objstore/mockobjstore" - "github.com/stretchr/testify/require" - "github.com/uber-go/atomic" - "go.uber.org/mock/gomock" -) - -func expectedLogFileName(cfg fileWriterConfig, logType string, commitTs uint64, uid string) string { - if cfg.ChangeFeedID().Keyspace() == common.DefaultKeyspaceName { - return fmt.Sprintf(redo.RedoLogFileFormatV1, - cfg.CaptureID(), cfg.ChangeFeedID().Name(), logType, commitTs, uid, redo.LogEXT) - } - return fmt.Sprintf(redo.RedoLogFileFormatV2, - cfg.CaptureID(), cfg.ChangeFeedID().Keyspace(), cfg.ChangeFeedID().Name(), - logType, commitTs, uid, redo.LogEXT) -} - -func TestWriterWrite(t *testing.T) { - t.Parallel() - - dir := t.TempDir() - extStorage := newTestLocalExternalStorage(t, dir) - cfs := []common.ChangeFeedID{ - common.NewChangeFeedIDWithName("test-cf", common.DefaultKeyspaceName), - common.NewChangeFeedIDWithDisplayName(common.ChangeFeedDisplayName{ - Keyspace: "abcd", - Name: "test-cf", - }), - } - - cf11s := []common.ChangeFeedID{ - common.NewChangeFeedIDWithName("test-cf11", common.DefaultKeyspaceName), - common.NewChangeFeedIDWithDisplayName(common.ChangeFeedDisplayName{ - Keyspace: "abcd", - Name: "test-cf11", - }), - } - - for idx, cf := range cfs { - largePayload := make([]byte, redo.Megabyte) - uuidGen := uuid.NewConstGenerator("const-uuid") - writerCfg := newTestWriterConfig( - t, - cf, - &config.ConsistentConfig{ - MaxLogSize: util.AddressOf(int64(1)), - Storage: util.AddressOf("file://" + dir), - }, - ) - w := &Writer{ - logType: redo.RedoRowLogFileType, - cfg: writerCfg, - uint64buf: make([]byte, 8), - running: *atomic.NewBool(true), - metricWriteBytes: metrics.RedoWriteBytesGauge. - WithLabelValues("default", "test-cf", redo.RedoRowLogFileType), - metricFsyncDuration: metrics.RedoFsyncDurationHistogram. - WithLabelValues("default", "test-cf", redo.RedoRowLogFileType), - metricFlushAllDuration: metrics.RedoFlushAllDurationHistogram. - WithLabelValues("default", "test-cf", redo.RedoRowLogFileType), - storage: extStorage, - uuidGenerator: uuidGen, - } - - w.eventCommitTS.Store(1) - _, err := w.Write(largePayload) - require.Nil(t, err) - var fileName string - // create a .tmp file - if w.cfg.ChangeFeedID().Keyspace() == common.DefaultKeyspaceName { - fileName = fmt.Sprintf(redo.RedoLogFileFormatV1, w.cfg.CaptureID(), - w.cfg.ChangeFeedID().Name(), - w.logType, 1, uuidGen.NewString(), redo.LogEXT) + redo.TmpEXT - } else { - fileName = fmt.Sprintf(redo.RedoLogFileFormatV2, w.cfg.CaptureID(), - w.cfg.ChangeFeedID().Keyspace(), w.cfg.ChangeFeedID().Name(), - w.logType, 1, uuidGen.NewString(), redo.LogEXT) + redo.TmpEXT - } - path := filepath.Join(w.cfg.Dir(), fileName) - info, err := os.Stat(path) - require.Nil(t, err) - require.Equal(t, fileName, info.Name()) - - w.eventCommitTS.Store(12) - _, err = w.Write([]byte("tt")) - require.Nil(t, err) - w.eventCommitTS.Store(22) - _, err = w.Write([]byte("t")) - require.Nil(t, err) - - // after rotate, rename to .log - if w.cfg.ChangeFeedID().Keyspace() == common.DefaultKeyspaceName { - fileName = fmt.Sprintf(redo.RedoLogFileFormatV1, w.cfg.CaptureID(), - w.cfg.ChangeFeedID().Name(), - w.logType, 1, uuidGen.NewString(), redo.LogEXT) - } else { - fileName = fmt.Sprintf(redo.RedoLogFileFormatV2, w.cfg.CaptureID(), - w.cfg.ChangeFeedID().Keyspace(), w.cfg.ChangeFeedID().Name(), - w.logType, 1, uuidGen.NewString(), redo.LogEXT) - } - path = filepath.Join(w.cfg.Dir(), fileName) - info, err = os.Stat(path) - require.Nil(t, err) - require.Equal(t, fileName, info.Name()) - // create a .tmp file with first eventCommitTS as name - if w.cfg.ChangeFeedID().Keyspace() == common.DefaultKeyspaceName { - fileName = fmt.Sprintf(redo.RedoLogFileFormatV1, w.cfg.CaptureID(), - w.cfg.ChangeFeedID().Name(), - w.logType, 12, uuidGen.NewString(), redo.LogEXT) + redo.TmpEXT - } else { - fileName = fmt.Sprintf(redo.RedoLogFileFormatV2, w.cfg.CaptureID(), - w.cfg.ChangeFeedID().Keyspace(), w.cfg.ChangeFeedID().Name(), - w.logType, 12, uuidGen.NewString(), redo.LogEXT) + redo.TmpEXT - } - path = filepath.Join(w.cfg.Dir(), fileName) - info, err = os.Stat(path) - require.Nil(t, err) - require.Equal(t, fileName, info.Name()) - err = w.Close() - require.Nil(t, err) - require.False(t, w.IsRunning()) - // safe close, rename to .log with max eventCommitTS as name - if w.cfg.ChangeFeedID().Keyspace() == common.DefaultKeyspaceName { - fileName = fmt.Sprintf(redo.RedoLogFileFormatV1, w.cfg.CaptureID(), - w.cfg.ChangeFeedID().Name(), - w.logType, 22, uuidGen.NewString(), redo.LogEXT) - } else { - fileName = fmt.Sprintf(redo.RedoLogFileFormatV2, w.cfg.CaptureID(), - w.cfg.ChangeFeedID().Keyspace(), w.cfg.ChangeFeedID().Name(), - w.logType, 22, uuidGen.NewString(), redo.LogEXT) - } - path = filepath.Join(w.cfg.Dir(), fileName) - info, err = os.Stat(path) - require.Nil(t, err) - require.Equal(t, fileName, info.Name()) - - writerCfg11 := newTestWriterConfig( - t, - cf11s[idx], - &config.ConsistentConfig{ - MaxLogSize: util.AddressOf(int64(1)), - Storage: util.AddressOf("file://" + dir), - }, - ) - w1 := &Writer{ - logType: redo.RedoRowLogFileType, - cfg: writerCfg11, - uint64buf: make([]byte, 8), - running: *atomic.NewBool(true), - metricWriteBytes: metrics.RedoWriteBytesGauge. - WithLabelValues("default", "test-cf11", redo.RedoRowLogFileType), - metricFsyncDuration: metrics.RedoFsyncDurationHistogram. - WithLabelValues("default", "test-cf11", redo.RedoRowLogFileType), - metricFlushAllDuration: metrics.RedoFlushAllDurationHistogram. - WithLabelValues("default", "test-cf11", redo.RedoRowLogFileType), - storage: extStorage, - uuidGenerator: uuidGen, - } - - w1.eventCommitTS.Store(1) - _, err = w1.Write(largePayload) - require.Nil(t, err) - // create a .tmp file - if w1.cfg.ChangeFeedID().Keyspace() == common.DefaultKeyspaceName { - fileName = fmt.Sprintf(redo.RedoLogFileFormatV1, w1.cfg.CaptureID(), - w1.cfg.ChangeFeedID().Name(), - w1.logType, 1, uuidGen.NewString(), redo.LogEXT) + redo.TmpEXT - } else { - fileName = fmt.Sprintf(redo.RedoLogFileFormatV2, w1.cfg.CaptureID(), - w1.cfg.ChangeFeedID().Keyspace(), w1.cfg.ChangeFeedID().Name(), - w1.logType, 1, uuidGen.NewString(), redo.LogEXT) + redo.TmpEXT - } - path = filepath.Join(w1.cfg.Dir(), fileName) - info, err = os.Stat(path) - require.Nil(t, err) - require.Equal(t, fileName, info.Name()) - // change the file name, should cause CLose err - err = os.Rename(path, path+"new") - require.Nil(t, err) - err = w1.Close() - require.NotNil(t, err) - // closed anyway - require.False(t, w1.IsRunning()) - } -} - -func TestNewWriter(t *testing.T) { - t.Parallel() - - storageDir := t.TempDir() - dir := t.TempDir() - - uuidGen := uuid.NewConstGenerator("const-uuid") - writerCfg := newTestWriterConfig( - t, - common.NewChangeFeedIDWithName("test-row-writer", common.DefaultKeyspaceName), - &config.ConsistentConfig{ - Storage: util.AddressOf("file://" + storageDir), - }, - ) - w, err := NewFileWriter(context.Background(), writerCfg, redo.RedoRowLogFileType, - writer.WithUUIDGenerator(func() uuid.Generator { return uuidGen }), - ) - require.Nil(t, err) - require.NotNil(t, w.allocator) - err = w.Close() - require.Nil(t, err) - require.False(t, w.IsRunning()) - - controller := gomock.NewController(t) - mockStorage := mockobjstore.NewMockStorage(controller) - - changefeed := common.NewChangeFeedIDWithDisplayName(common.ChangeFeedDisplayName{ - Keyspace: "abcd", - Name: "test", - }) - ddlWriterCfg := newTestWriterConfig( - t, - changefeed, - &config.ConsistentConfig{ - Storage: util.AddressOf("file://" + dir), - }, - ) - mockStorage.EXPECT().WriteFile( - gomock.Any(), - expectedLogFileName(ddlWriterCfg, redo.RedoDDLLogFileType, 0, "const-uuid"), - gomock.Any(), - ).Return(nil).Times(1) - mockStorage.EXPECT().Close().Times(1) - w = &Writer{ - logType: redo.RedoDDLLogFileType, - cfg: ddlWriterCfg, - uint64buf: make([]byte, 8), - storage: mockStorage, - metricWriteBytes: metrics.RedoWriteBytesGauge. - WithLabelValues("default", "test", redo.RedoRowLogFileType), - metricFsyncDuration: metrics.RedoFsyncDurationHistogram. - WithLabelValues("default", "test", redo.RedoRowLogFileType), - metricFlushAllDuration: metrics.RedoFlushAllDurationHistogram. - WithLabelValues("default", "test", redo.RedoRowLogFileType), - uuidGenerator: uuidGen, - } - w.running.Store(true) - _, err = w.Write([]byte("test")) - require.Nil(t, err) - err = w.Flush() - require.Nil(t, err) - - err = w.Close() - require.Nil(t, err) - require.Equal(t, w.running.Load(), false) -} - -func TestNewLocalFileWriterKeepsLocalOnlySemantics(t *testing.T) { - t.Parallel() - - dir := t.TempDir() - fileName := "reader-sort.log" - - w, err := NewLocalFileWriter(dir, math.MaxInt32, redo.RedoRowLogFileType, writer.WithLogFileName(func() string { - return fileName - })) - require.NoError(t, err) - require.Nil(t, w.storage) - require.Nil(t, w.allocator) - require.Equal(t, dir, w.cfg.Dir()) - require.False(t, w.cfg.UseExternalStorage()) - _, err = w.Write([]byte("test")) - require.NoError(t, err) - require.NoError(t, w.Close()) - _, err = os.Stat(filepath.Join(dir, fileName)) - require.NoError(t, err) -} - -func TestRotateFileWithFileAllocator(t *testing.T) { - t.Parallel() - - controller := gomock.NewController(t) - mockStorage := mockobjstore.NewMockStorage(controller) - - dir := t.TempDir() - uuidGen := uuid.NewMock() - uuidGen.Push("uuid-1") - uuidGen.Push("uuid-2") - uuidGen.Push("uuid-3") - uuidGen.Push("uuid-4") - uuidGen.Push("uuid-5") - changefeed := common.NewChangeFeedIDWithDisplayName(common.ChangeFeedDisplayName{ - Keyspace: "abcd", - Name: "test", - }) - rowWriterCfg := newTestWriterConfig( - t, - changefeed, - &config.ConsistentConfig{ - Storage: util.AddressOf("file://" + dir), - }, - ) - mockStorage.EXPECT().WriteFile( - gomock.Any(), - expectedLogFileName(rowWriterCfg, redo.RedoRowLogFileType, 0, "uuid-1"), - gomock.Any(), - ).Return(nil).Times(1) - mockStorage.EXPECT().WriteFile( - gomock.Any(), - expectedLogFileName(rowWriterCfg, redo.RedoRowLogFileType, 100, "uuid-2"), - gomock.Any(), - ).Return(nil).Times(1) - mockStorage.EXPECT().Close().Times(1) - w := &Writer{ - logType: redo.RedoRowLogFileType, - cfg: rowWriterCfg, - uint64buf: make([]byte, 8), - metricWriteBytes: metrics.RedoWriteBytesGauge. - WithLabelValues("default", "test", redo.RedoRowLogFileType), - metricFsyncDuration: metrics.RedoFsyncDurationHistogram. - WithLabelValues("default", "test", redo.RedoRowLogFileType), - metricFlushAllDuration: metrics.RedoFlushAllDurationHistogram. - WithLabelValues("default", "test", redo.RedoRowLogFileType), - storage: mockStorage, - uuidGenerator: uuidGen, - } - w.allocator = fsutil.NewFileAllocator( - w.cfg.Dir(), redo.RedoRowLogFileType, redo.DefaultMaxLogSize*redo.Megabyte) - - w.running.Store(true) - _, err := w.Write([]byte("test")) - require.Nil(t, err) - - err = w.rotate() - require.Nil(t, err) - - w.AdvanceTs(100) - _, err = w.Write([]byte("test")) - require.Nil(t, err) - err = w.rotate() - require.Nil(t, err) - - w.Close() -} - -func TestRotateFileWithoutFileAllocator(t *testing.T) { - t.Parallel() - - controller := gomock.NewController(t) - mockStorage := mockobjstore.NewMockStorage(controller) - - dir := t.TempDir() - uuidGen := uuid.NewMock() - uuidGen.Push("uuid-1") - uuidGen.Push("uuid-2") - uuidGen.Push("uuid-3") - uuidGen.Push("uuid-4") - uuidGen.Push("uuid-5") - uuidGen.Push("uuid-6") - changefeed := common.NewChangeFeedIDWithDisplayName(common.ChangeFeedDisplayName{ - Keyspace: "abcd", - Name: "test", - }) - ddlWriterCfg := newTestWriterConfig( - t, - changefeed, - &config.ConsistentConfig{ - Storage: util.AddressOf("file://" + dir), - }, - ) - mockStorage.EXPECT().WriteFile( - gomock.Any(), - expectedLogFileName(ddlWriterCfg, redo.RedoDDLLogFileType, 0, "uuid-2"), - gomock.Any(), - ).Return(nil).Times(1) - mockStorage.EXPECT().WriteFile( - gomock.Any(), - expectedLogFileName(ddlWriterCfg, redo.RedoDDLLogFileType, 100, "uuid-4"), - gomock.Any(), - ).Return(nil).Times(1) - mockStorage.EXPECT().Close().Times(1) - w := &Writer{ - logType: redo.RedoDDLLogFileType, - cfg: ddlWriterCfg, - uint64buf: make([]byte, 8), - metricWriteBytes: metrics.RedoWriteBytesGauge. - WithLabelValues("default", "test", redo.RedoDDLLogFileType), - metricFsyncDuration: metrics.RedoFsyncDurationHistogram. - WithLabelValues("default", "test", redo.RedoDDLLogFileType), - metricFlushAllDuration: metrics.RedoFlushAllDurationHistogram. - WithLabelValues("default", "test", redo.RedoDDLLogFileType), - storage: mockStorage, - uuidGenerator: uuidGen, - } - w.running.Store(true) - _, err := w.Write([]byte("test")) - require.Nil(t, err) - - err = w.rotate() - require.Nil(t, err) - - w.AdvanceTs(100) - _, err = w.Write([]byte("test")) - require.Nil(t, err) - err = w.rotate() - require.Nil(t, err) - - w.Close() -} - -func TestRunFlushesOnBatchBoundaryAndExecutesPostFlush(t *testing.T) { - t.Parallel() - - dir := t.TempDir() - flushIntervalInMs := int64(60 * 1000) - flushWorkerNum := 9 - batchWriterCfg := newTestWriterConfig( - t, - common.NewChangeFeedIDWithName("test-run-batch", common.DefaultKeyspaceName), - &config.ConsistentConfig{ - FlushIntervalInMs: &flushIntervalInMs, - FlushWorkerNum: &flushWorkerNum, - Storage: util.AddressOf("file://" + dir), - }, - ) - w, err := NewFileWriter(context.Background(), batchWriterCfg, redo.RedoRowLogFileType) - require.NoError(t, err) - - ctx, cancel := context.WithCancel(context.Background()) - runErrCh := make(chan error, 1) - go func() { - runErrCh <- w.Run(ctx) - }() - - postFlushCnt := atomic.NewInt64(0) - for i := 0; i < redo.DefaultFlushBatchSize-1; i++ { - ts := uint64(i + 1) - w.GetInputCh() <- &pevent.RedoRowEvent{ - StartTs: ts, - CommitTs: ts, - Callback: func() { - postFlushCnt.Inc() - }, - } - } - - // The callback should not be executed before the batch reaches the boundary. - require.Equal(t, int64(0), postFlushCnt.Load()) - select { - case err := <-runErrCh: - require.Failf(t, "run exited unexpectedly", "run returned before cancel: %v", err) - default: - } - - ts := uint64(redo.DefaultFlushBatchSize) - w.GetInputCh() <- &pevent.RedoRowEvent{ - StartTs: ts, - CommitTs: ts, - Callback: func() { - postFlushCnt.Inc() - }, - } - - require.Eventually(t, func() bool { - return postFlushCnt.Load() == int64(redo.DefaultFlushBatchSize) - }, 10*time.Second, 20*time.Millisecond) - - cancel() - require.ErrorIs(t, <-runErrCh, context.Canceled) - require.NoError(t, w.Close()) -} diff --git a/pkg/redo/writer/file/test_helper_test.go b/pkg/redo/writer/file/test_helper_test.go deleted file mode 100644 index 56c4ff1055..0000000000 --- a/pkg/redo/writer/file/test_helper_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package file - -import ( - "context" - "testing" - - "github.com/pingcap/ticdc/pkg/common" - "github.com/pingcap/ticdc/pkg/config" - "github.com/pingcap/ticdc/pkg/redo/testutil" - "github.com/pingcap/ticdc/pkg/redo/writer" - "github.com/pingcap/ticdc/pkg/util" - "github.com/pingcap/tidb/pkg/objstore/storeapi" - "github.com/stretchr/testify/require" -) - -func newTestWriterConfig( - t *testing.T, - changefeedID common.ChangeFeedID, - consistentCfg *config.ConsistentConfig, -) *writer.Config { - defaultCfg := testutil.NewConsistentConfig("file://" + t.TempDir()) - if consistentCfg == nil { - consistentCfg = defaultCfg - } - if len(util.GetOrZero(consistentCfg.Storage)) == 0 { - consistentCfg.Storage = defaultCfg.Storage - } - if util.GetOrZero(consistentCfg.MaxLogSize) == 0 { - consistentCfg.MaxLogSize = defaultCfg.MaxLogSize - } - if util.GetOrZero(consistentCfg.FlushIntervalInMs) == 0 { - consistentCfg.FlushIntervalInMs = defaultCfg.FlushIntervalInMs - } - if util.GetOrZero(consistentCfg.EncodingWorkerNum) == 0 { - consistentCfg.EncodingWorkerNum = defaultCfg.EncodingWorkerNum - } - if util.GetOrZero(consistentCfg.FlushWorkerNum) == 0 { - consistentCfg.FlushWorkerNum = defaultCfg.FlushWorkerNum - } - if len(util.GetOrZero(consistentCfg.Compression)) == 0 { - consistentCfg.Compression = defaultCfg.Compression - } - if util.GetOrZero(consistentCfg.FlushConcurrency) == 0 { - consistentCfg.FlushConcurrency = defaultCfg.FlushConcurrency - } - cfg, err := writer.NewConfig(changefeedID, consistentCfg) - require.NoError(t, err) - return cfg -} - -func newTestLocalExternalStorage(t *testing.T, dir string) storeapi.Storage { - extStorage, _, err := util.GetTestExtStorage(context.Background(), dir) - require.NoError(t, err) - return extStorage -} diff --git a/pkg/redo/writer/memory/file_worker.go b/pkg/redo/writer/file_worker.go similarity index 95% rename from pkg/redo/writer/memory/file_worker.go rename to pkg/redo/writer/file_worker.go index ba67651544..751f922894 100644 --- a/pkg/redo/writer/memory/file_worker.go +++ b/pkg/redo/writer/file_worker.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package memory +package writer import ( "bytes" @@ -28,7 +28,6 @@ import ( "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/ticdc/pkg/redo" - "github.com/pingcap/ticdc/pkg/redo/writer" "github.com/pingcap/ticdc/pkg/uuid" "github.com/pingcap/tidb/pkg/objstore/storeapi" "github.com/prometheus/client_golang/prometheus" @@ -84,8 +83,8 @@ func (f *fileCache) markFlushed() { } type fileWorkerGroup struct { - cfg *writer.Config - op *writer.LogWriterOptions + cfg *Config + op *LogWriterOptions workerNum int inputCh chan *polymorphicRedoEvent extStorage storeapi.Storage @@ -103,17 +102,17 @@ type fileWorkerGroup struct { // fileWorkerGroup receives encoded redo events and writes them to cache, with // background goroutines handling file flush. func newFileWorkerGroup( - cfg *writer.Config, + cfg *Config, inputCh chan *polymorphicRedoEvent, extStorage storeapi.Storage, - opts ...writer.Option, + opts ...Option, ) *fileWorkerGroup { workerNum := cfg.FlushWorkerNum() if workerNum <= 0 { workerNum = redo.DefaultFlushWorkerNum } - op := &writer.LogWriterOptions{} + op := &LogWriterOptions{} for _, opt := range opts { opt(op) } @@ -212,8 +211,7 @@ func (f *fileWorkerGroup) bgWriteLogs( d := time.Duration(f.cfg.FlushIntervalInMs()) * time.Millisecond ticker := time.NewTicker(d) defer ticker.Stop() - num := 0 - cacheEventPostFlush := make([]func(), 0, redo.DefaultFlushBatchSize) + var cacheEventPostFlush []func() flush := func() error { err := f.flushAll(egCtx) if err != nil { @@ -222,7 +220,6 @@ func (f *fileWorkerGroup) bgWriteLogs( for _, fn := range cacheEventPostFlush { fn() } - num = 0 cacheEventPostFlush = cacheEventPostFlush[:0] return nil } @@ -244,15 +241,11 @@ func (f *fileWorkerGroup) bgWriteLogs( if err != nil { return errors.Trace(err) } - num++ - if num > redo.DefaultFlushBatchSize { - err := flush() - if err != nil { + cacheEventPostFlush = append(cacheEventPostFlush, event.PostFlush) + if event.flushImmediately { + if err := flush(); err != nil { return errors.Trace(err) } - event.PostFlush() - } else { - cacheEventPostFlush = append(cacheEventPostFlush, event.PostFlush) } } } diff --git a/pkg/redo/writer/file/main_test.go b/pkg/redo/writer/main_test.go similarity index 97% rename from pkg/redo/writer/file/main_test.go rename to pkg/redo/writer/main_test.go index 8fb571e1ec..400593de61 100644 --- a/pkg/redo/writer/file/main_test.go +++ b/pkg/redo/writer/main_test.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package file +package writer import ( "testing" diff --git a/pkg/redo/writer/memory/dml_writer.go b/pkg/redo/writer/memory/dml_writer.go deleted file mode 100644 index 649126f7b5..0000000000 --- a/pkg/redo/writer/memory/dml_writer.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package memory - -import ( - "context" - - "github.com/pingcap/log" - commonEvent "github.com/pingcap/ticdc/pkg/common/event" - "github.com/pingcap/ticdc/pkg/redo" - "github.com/pingcap/ticdc/pkg/redo/writer" - "github.com/pingcap/tidb/pkg/objstore/storeapi" - "go.uber.org/zap" - "golang.org/x/sync/errgroup" -) - -var _ writer.RedoDMLWriter = (*dmlWriter)(nil) - -type dmlWriter struct { - cfg *writer.Config - encodeWorkers *encodingWorkerGroup - fileWorkers *fileWorkerGroup - extStorage storeapi.Storage - cancel context.CancelFunc -} - -// NewDMLWriter creates a new memory DML writer. -func NewDMLWriter( - ctx context.Context, cfg *writer.Config, opts ...writer.Option, -) (writer.RedoDMLWriter, error) { - extStorage, err := redo.InitExternalStorage(ctx, *cfg.URI()) - if err != nil { - return nil, err - } - - encodeWorkers := newEncodingWorkerGroup(cfg) - fileWorkers := newFileWorkerGroup( - cfg, encodeWorkers.outputCh, extStorage, opts...) - - return &dmlWriter{ - cfg: cfg, - encodeWorkers: encodeWorkers, - fileWorkers: fileWorkers, - extStorage: extStorage, - }, nil -} - -func (l *dmlWriter) Run(ctx context.Context) error { - newCtx, cancel := context.WithCancel(ctx) - l.cancel = cancel - - eg, egCtx := errgroup.WithContext(newCtx) - eg.Go(func() error { - return l.encodeWorkers.Run(egCtx) - }) - eg.Go(func() error { - return l.fileWorkers.Run(egCtx) - }) - return eg.Wait() -} - -func (l *dmlWriter) AddDMLEvents(ctx context.Context, events ...*commonEvent.RedoRowEvent) error { - for _, event := range events { - if event == nil { - log.Warn("writing nil event to redo log, ignore this", - zap.String("keyspace", l.cfg.ChangeFeedID().Keyspace()), - zap.String("changefeed", l.cfg.ChangeFeedID().Name())) - continue - } - if err := l.encodeWorkers.AddEvent(ctx, event); err != nil { - return err - } - } - return nil -} - -func (l *dmlWriter) Close() error { - if l.cancel != nil { - l.cancel() - l.cancel = nil - } - if l.extStorage != nil { - l.extStorage.Close() - l.extStorage = nil - } - return nil -} diff --git a/pkg/redo/writer/memory/dml_writer_test.go b/pkg/redo/writer/memory/dml_writer_test.go deleted file mode 100644 index f7d79241cc..0000000000 --- a/pkg/redo/writer/memory/dml_writer_test.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package memory - -import ( - "context" - "testing" - - "github.com/pingcap/ticdc/pkg/common" - "github.com/pingcap/ticdc/pkg/redo/testutil" - "github.com/pingcap/ticdc/pkg/redo/writer" - "github.com/pingcap/ticdc/pkg/util" - "github.com/stretchr/testify/require" -) - -func TestNewDMLWriter(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - _, uri, err := util.GetTestExtStorage(ctx, t.TempDir()) - require.NoError(t, err) - cfg, err := writer.NewConfig( - common.NewChangeFeedIDWithName("test-changefeed", common.DefaultKeyspaceName), - testutil.NewConsistentConfig(uri.String()), - ) - require.NoError(t, err) - - lw, err := NewDMLWriter(ctx, cfg) - require.NoError(t, err) - require.NoError(t, lw.Close()) -} diff --git a/pkg/redo/writer/memory/main_test.go b/pkg/redo/writer/memory/main_test.go deleted file mode 100644 index 528ecb0197..0000000000 --- a/pkg/redo/writer/memory/main_test.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2023 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package memory - -import ( - "testing" - - "github.com/pingcap/ticdc/pkg/leakutil" -) - -func TestMain(m *testing.M) { - leakutil.SetUpLeakTest(m) -} diff --git a/pkg/redo/writer/writer_test.go b/pkg/redo/writer/writer_test.go index c49d022951..f2bea3156e 100644 --- a/pkg/redo/writer/writer_test.go +++ b/pkg/redo/writer/writer_test.go @@ -14,7 +14,6 @@ package writer import ( - "path/filepath" "testing" "github.com/pingcap/ticdc/pkg/common" @@ -35,6 +34,8 @@ func TestNewConfigUsesConsistentConfigValues(t *testing.T) { flushWorkerNum := 6 compressionType := "lz4" flushConcurrency := 7 + spoolDiskQuota := int64(8 * 1024 * 1024) + spoolBaseDir := t.TempDir() consistentCfg := testutil.NewConsistentConfig("nfs:///tmp/redo") consistentCfg.MaxLogSize = util.AddressOf(maxLogSize) consistentCfg.FlushIntervalInMs = util.AddressOf(flushIntervalInMs) @@ -42,6 +43,8 @@ func TestNewConfigUsesConsistentConfigValues(t *testing.T) { consistentCfg.FlushWorkerNum = util.AddressOf(flushWorkerNum) consistentCfg.Compression = util.AddressOf(compressionType) consistentCfg.FlushConcurrency = util.AddressOf(flushConcurrency) + consistentCfg.SpoolDiskQuota = util.AddressOf(spoolDiskQuota) + consistentCfg.SpoolBaseDir = util.AddressOf(spoolBaseDir) cfg, err := NewConfig(changefeedID, consistentCfg) require.NoError(t, err) @@ -49,7 +52,6 @@ func TestNewConfigUsesConsistentConfigValues(t *testing.T) { require.Equal(t, config.GetGlobalServerConfig().AdvertiseAddr, cfg.CaptureID()) require.NotNil(t, cfg.URI()) require.Equal(t, "file", cfg.URI().Scheme) - require.Equal(t, "/tmp/redo", cfg.Dir()) require.True(t, cfg.UseExternalStorage()) require.Equal(t, maxLogSize*redo.Megabyte, cfg.MaxLogSizeInBytes()) require.Equal(t, flushIntervalInMs, cfg.FlushIntervalInMs()) @@ -57,39 +59,8 @@ func TestNewConfigUsesConsistentConfigValues(t *testing.T) { require.Equal(t, flushWorkerNum, cfg.FlushWorkerNum()) require.Equal(t, flushConcurrency, cfg.FlushConcurrency()) require.Equal(t, compressionType, cfg.Compression()) - require.False(t, cfg.UseFileBackend()) -} - -func TestNewConfigInitializesFileBackendDirForExternalStorage(t *testing.T) { - t.Parallel() - - changefeedID := common.NewChangeFeedIDWithName("test-cf", common.DefaultKeyspaceName) - consistentCfg := testutil.NewConsistentConfig("s3://bucket/prefix") - consistentCfg.UseFileBackend = util.AddressOf(true) - cfg, err := NewConfig(changefeedID, consistentCfg) - require.NoError(t, err) - - require.NotNil(t, cfg.URI()) - require.Equal(t, "s3", cfg.URI().Scheme) - require.True(t, cfg.UseExternalStorage()) - require.True(t, cfg.UseFileBackend()) - require.Equal(t, - filepath.Join(config.GetGlobalServerConfig().DataDir, config.DefaultRedoDir, changefeedID.Keyspace(), changefeedID.Name()), - cfg.Dir()) -} - -func TestNewConfigLeavesDirEmptyForRemoteMemoryBackend(t *testing.T) { - t.Parallel() - - changefeedID := common.NewChangeFeedIDWithName("test-cf", common.DefaultKeyspaceName) - cfg, err := NewConfig(changefeedID, testutil.NewConsistentConfig("s3://bucket/prefix")) - require.NoError(t, err) - - require.NotNil(t, cfg.URI()) - require.Equal(t, "s3", cfg.URI().Scheme) - require.True(t, cfg.UseExternalStorage()) - require.False(t, cfg.UseFileBackend()) - require.Empty(t, cfg.Dir()) + require.Equal(t, spoolDiskQuota, cfg.SpoolDiskQuota()) + require.Equal(t, spoolBaseDir, cfg.SpoolBaseDir()) } func TestNewConfigReturnsErrorForInvalidStorageURI(t *testing.T) { diff --git a/pkg/sink/spool/budget.go b/pkg/sink/spool/budget.go new file mode 100644 index 0000000000..5e3f1cc4e6 --- /dev/null +++ b/pkg/sink/spool/budget.go @@ -0,0 +1,107 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package spool + +// Limits defines the byte limits used by a spool budget. +type Limits struct { + DiskQuotaBytes int64 + MemoryQuotaBytes int64 + HighWatermarkBytes int64 + LowWatermarkBytes int64 +} + +// Budget tracks the memory and disk bytes owned by a spool. It is not +// thread-safe; callers should synchronize compound admission decisions. +type Budget struct { + limits Limits + + memoryBytes int64 + diskBytes int64 +} + +// NewBudget creates an empty spool budget with the supplied limits. +func NewBudget(limits Limits) *Budget { + return &Budget{limits: limits} +} + +// CanFitMemory reports whether an entry can be admitted into memory. +func (b *Budget) CanFitMemory(entryBytes int64) bool { + return b.memoryBytes+entryBytes <= b.limits.MemoryQuotaBytes +} + +// ShouldSpill reports whether a new entry should be written to disk instead +// of being retained in memory. +func (b *Budget) ShouldSpill(entryBytes int64) bool { + return !b.CanFitMemory(entryBytes) +} + +// EntryExceedsDiskQuota reports whether one entry is larger than the entire +// disk quota. +func (b *Budget) EntryExceedsDiskQuota(entryBytes int64) bool { + return entryBytes > b.limits.DiskQuotaBytes +} + +// SpillWouldExceedDiskQuota reports whether admitting one more spilled entry +// would exceed the disk quota. +func (b *Budget) SpillWouldExceedDiskQuota(entryBytes int64) bool { + return b.diskBytes+entryBytes > b.limits.DiskQuotaBytes +} + +// Acquire records an admitted entry and reports whether total staged bytes +// are above the high watermark. +func (b *Budget) Acquire(entryBytes int64, spilled bool) bool { + if spilled { + b.diskBytes += entryBytes + } else { + b.memoryBytes += entryBytes + } + return b.TotalBytes() > b.limits.HighWatermarkBytes +} + +// Release removes a flushed or discarded entry and reports whether total +// staged bytes are at or below the low watermark. +func (b *Budget) Release(entryBytes int64, spilled bool) bool { + if spilled { + b.diskBytes -= entryBytes + } else { + b.memoryBytes -= entryBytes + } + if b.memoryBytes < 0 { + b.memoryBytes = 0 + } + if b.diskBytes < 0 { + b.diskBytes = 0 + } + return b.TotalBytes() <= b.limits.LowWatermarkBytes +} + +// MemoryBytes returns currently staged in-memory bytes. +func (b *Budget) MemoryBytes() int64 { + return b.memoryBytes +} + +// DiskBytes returns currently staged on-disk bytes. +func (b *Budget) DiskBytes() int64 { + return b.diskBytes +} + +// TotalBytes returns all currently staged bytes. +func (b *Budget) TotalBytes() int64 { + return b.memoryBytes + b.diskBytes +} + +// Limits returns the immutable limits of this budget. +func (b *Budget) Limits() Limits { + return b.limits +} diff --git a/pkg/sink/spool/budget_test.go b/pkg/sink/spool/budget_test.go new file mode 100644 index 0000000000..53a7aca6a4 --- /dev/null +++ b/pkg/sink/spool/budget_test.go @@ -0,0 +1,67 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package spool + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBudgetTracksMemoryAndDiskBytes(t *testing.T) { + t.Parallel() + + budget := NewBudget(Limits{ + DiskQuotaBytes: 100, + MemoryQuotaBytes: 20, + HighWatermarkBytes: 80, + LowWatermarkBytes: 60, + }) + + require.False(t, budget.ShouldSpill(10)) + require.False(t, budget.Acquire(10, false)) + require.Equal(t, int64(10), budget.MemoryBytes()) + require.Equal(t, int64(0), budget.DiskBytes()) + require.Equal(t, int64(10), budget.TotalBytes()) + + require.True(t, budget.ShouldSpill(11)) + require.False(t, budget.Acquire(11, true)) + require.Equal(t, int64(10), budget.MemoryBytes()) + require.Equal(t, int64(11), budget.DiskBytes()) + require.Equal(t, int64(21), budget.TotalBytes()) + + require.True(t, budget.Release(50, false)) + require.Equal(t, int64(0), budget.MemoryBytes()) + require.Equal(t, int64(11), budget.DiskBytes()) + + require.True(t, budget.Release(50, true)) + require.Equal(t, int64(0), budget.TotalBytes()) +} + +func TestBudgetTracksWatermarkAndDiskQuota(t *testing.T) { + t.Parallel() + + budget := NewBudget(Limits{ + DiskQuotaBytes: 100, + MemoryQuotaBytes: 20, + HighWatermarkBytes: 80, + LowWatermarkBytes: 60, + }) + + require.True(t, budget.EntryExceedsDiskQuota(101)) + require.False(t, budget.SpillWouldExceedDiskQuota(81)) + require.True(t, budget.Acquire(81, true)) + require.True(t, budget.SpillWouldExceedDiskQuota(20)) + require.True(t, budget.Release(21, true)) +} diff --git a/downstreamadapter/sink/cloudstorage/spool/codec.go b/pkg/sink/spool/codec.go similarity index 100% rename from downstreamadapter/sink/cloudstorage/spool/codec.go rename to pkg/sink/spool/codec.go diff --git a/downstreamadapter/sink/cloudstorage/spool/codec_test.go b/pkg/sink/spool/codec_test.go similarity index 100% rename from downstreamadapter/sink/cloudstorage/spool/codec_test.go rename to pkg/sink/spool/codec_test.go diff --git a/downstreamadapter/sink/cloudstorage/spool/quota.go b/pkg/sink/spool/quota.go similarity index 69% rename from downstreamadapter/sink/cloudstorage/spool/quota.go rename to pkg/sink/spool/quota.go index 1178bcfac8..a4851e4ddb 100644 --- a/downstreamadapter/sink/cloudstorage/spool/quota.go +++ b/pkg/sink/spool/quota.go @@ -16,8 +16,6 @@ package spool import ( "sync" - "github.com/pingcap/ticdc/downstreamadapter/sink/metrics" - "github.com/pingcap/ticdc/pkg/common" "github.com/prometheus/client_golang/prometheus" ) @@ -26,7 +24,7 @@ import ( // state are we in"; this adapter decides how spool reacts to that state. type quotaController struct { // budget owns threshold math and byte accounting. - budget *budget + budget *Budget // postEnqueuePaused is true will hold PostEnqueue callbacks in memory postEnqueuePaused bool @@ -40,31 +38,30 @@ type quotaController struct { metricDiskQuotaWaiters prometheus.Gauge metricDiskQuotaWait prometheus.Observer - keyspace string - changefeed string + closeMetrics func() waitersMu sync.Mutex nextWaiterID uint64 waiters map[uint64]chan struct{} } -func newQuotaController( - changefeedID common.ChangeFeedID, - options *options, -) *quotaController { - keyspace := changefeedID.Keyspace() - changefeed := changefeedID.Name() +func newQuotaController(options *options) *quotaController { + spoolMetrics := normalizeMetrics(options.metrics) controller := "aController{ - keyspace: keyspace, - changefeed: changefeed, - - budget: newBudget(options), - - metricMemoryBytes: metrics.CloudStorageSpoolMemoryBytesGauge.WithLabelValues(keyspace, changefeed), - metricDiskBytes: metrics.CloudStorageSpoolDiskBytesGauge.WithLabelValues(keyspace, changefeed), - metricPendingPostEnqueue: metrics.CloudStoragePendingPostEnqueueGauge.WithLabelValues(keyspace, changefeed), - metricDiskQuotaWaiters: metrics.CloudStorageSpoolDiskQuotaWaitersGauge.WithLabelValues(keyspace, changefeed), - metricDiskQuotaWait: metrics.CloudStorageSpoolDiskQuotaWaitDurationHistogram.WithLabelValues(keyspace, changefeed), + closeMetrics: spoolMetrics.Close, + + budget: NewBudget(Limits{ + DiskQuotaBytes: options.diskQuotaBytes, + MemoryQuotaBytes: int64(float64(options.diskQuotaBytes) * options.memoryRatio), + HighWatermarkBytes: int64(float64(options.diskQuotaBytes) * options.highWatermarkRatio), + LowWatermarkBytes: int64(float64(options.diskQuotaBytes) * options.lowWatermarkRatio), + }), + + metricMemoryBytes: spoolMetrics.MemoryBytes, + metricDiskBytes: spoolMetrics.DiskBytes, + metricPendingPostEnqueue: spoolMetrics.PendingPostEnqueue, + metricDiskQuotaWaiters: spoolMetrics.DiskQuotaWaiters, + metricDiskQuotaWait: spoolMetrics.DiskQuotaWait, waiters: make(map[uint64]chan struct{}), } controller.metricDiskQuotaWaiters.Set(0) @@ -72,15 +69,15 @@ func newQuotaController( } func (q *quotaController) shouldSpill(entryBytes int64) bool { - return q.budget.shouldSpill(entryBytes) + return q.budget.ShouldSpill(entryBytes) } func (q *quotaController) entryExceedsDiskQuota(entryBytes int64) bool { - return q.budget.entryExceedsDiskQuota(entryBytes) + return q.budget.EntryExceedsDiskQuota(entryBytes) } func (q *quotaController) spillWouldExceedDiskQuota(entryBytes int64) bool { - return q.budget.spillWouldExceedDiskQuota(entryBytes) + return q.budget.SpillWouldExceedDiskQuota(entryBytes) } func (q *quotaController) addDiskQuotaWaiter() (uint64, <-chan struct{}) { @@ -111,7 +108,7 @@ func (q *quotaController) acquire( spilled bool, postEnqueue func(), ) func() { - if q.budget.acquire(entryBytes, spilled) { + if q.budget.Acquire(entryBytes, spilled) { q.postEnqueuePaused = true } @@ -130,7 +127,7 @@ func (q *quotaController) acquire( // discarded. It returns all pending PostEnqueue callbacks once local usage has // dropped back to the low watermark. func (q *quotaController) release(entryBytes int64, spilled bool) []func() { - atOrBelowLowWatermark := q.budget.release(entryBytes, spilled) + atOrBelowLowWatermark := q.budget.Release(entryBytes, spilled) if spilled { q.wakeDiskQuotaWaiters() } @@ -151,16 +148,12 @@ func (q *quotaController) release(entryBytes int64, spilled bool) []func() { // deleteMetrics removes per-changefeed label values owned by this adapter. func (q *quotaController) deleteMetrics() { - metrics.CloudStorageSpoolMemoryBytesGauge.DeleteLabelValues(q.keyspace, q.changefeed) - metrics.CloudStorageSpoolDiskBytesGauge.DeleteLabelValues(q.keyspace, q.changefeed) - metrics.CloudStoragePendingPostEnqueueGauge.DeleteLabelValues(q.keyspace, q.changefeed) - metrics.CloudStorageSpoolDiskQuotaWaitersGauge.DeleteLabelValues(q.keyspace, q.changefeed) - metrics.CloudStorageSpoolDiskQuotaWaitDurationHistogram.DeleteLabelValues(q.keyspace, q.changefeed) + q.closeMetrics() } func (q *quotaController) updateMetrics() { - q.metricMemoryBytes.Set(float64(q.budget.memoryBytes)) - q.metricDiskBytes.Set(float64(q.budget.diskBytes)) + q.metricMemoryBytes.Set(float64(q.budget.MemoryBytes())) + q.metricDiskBytes.Set(float64(q.budget.DiskBytes())) q.metricPendingPostEnqueue.Set(float64(len(q.pendingPostEnqueue))) } diff --git a/downstreamadapter/sink/cloudstorage/spool/spool.go b/pkg/sink/spool/spool.go similarity index 89% rename from downstreamadapter/sink/cloudstorage/spool/spool.go rename to pkg/sink/spool/spool.go index ffe5dbe7ca..10c0a9f58a 100644 --- a/downstreamadapter/sink/cloudstorage/spool/spool.go +++ b/pkg/sink/spool/spool.go @@ -22,7 +22,6 @@ import ( "time" "github.com/pingcap/log" - "github.com/pingcap/ticdc/downstreamadapter/sink/metrics" commonType "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/config" "github.com/pingcap/ticdc/pkg/errors" @@ -79,17 +78,17 @@ type options struct { highWatermarkRatio float64 // lowWatermarkRatio is the ratio that resumes pending PostEnqueue callbacks. lowWatermarkRatio float64 -} -type option func(*options) + metrics *Metrics +} -func WithRootDir(rootDir string) option { +func WithRootDir(rootDir string) func(*options) { return func(options *options) { options.rootDir = rootDir } } -func WithDiskQuotaBytes(quotaBytes int64) option { +func WithDiskQuotaBytes(quotaBytes int64) func(*options) { return func(options *options) { if quotaBytes == 0 { return @@ -107,7 +106,7 @@ func WithDiskQuotaBytes(quotaBytes int64) option { } } -func WithSegmentBytes(segmentBytes int64) option { +func WithSegmentBytes(segmentBytes int64) func(*options) { return func(options *options) { if segmentBytes == 0 { return @@ -125,7 +124,7 @@ func WithSegmentBytes(segmentBytes int64) option { } } -func WithMemoryRatio(memoryRatio float64) option { +func WithMemoryRatio(memoryRatio float64) func(*options) { return func(options *options) { if memoryRatio == 0 { return @@ -143,7 +142,7 @@ func WithMemoryRatio(memoryRatio float64) option { } } -func WithHighWatermarkRatio(highWatermarkRatio float64) option { +func WithHighWatermarkRatio(highWatermarkRatio float64) func(*options) { return func(options *options) { if highWatermarkRatio == 0 { return @@ -161,7 +160,7 @@ func WithHighWatermarkRatio(highWatermarkRatio float64) option { } } -func WithLowWatermarkRatio(lowWatermarkRatio float64) option { +func WithLowWatermarkRatio(lowWatermarkRatio float64) func(*options) { return func(options *options) { if lowWatermarkRatio == 0 { return @@ -179,19 +178,31 @@ func WithLowWatermarkRatio(lowWatermarkRatio float64) option { } } +// Metrics contains component-owned metric handles updated by a spool. +type Metrics struct { + MemoryBytes prometheus.Gauge + DiskBytes prometheus.Gauge + PendingPostEnqueue prometheus.Gauge + DiskQuotaWaiters prometheus.Gauge + DiskQuotaWait prometheus.Observer + LoadedBytes prometheus.Observer + RotatedCount prometheus.Counter + SegmentCount prometheus.Gauge + Close func() +} + +// WithMetrics supplies component-owned metrics to the shared spool. +func WithMetrics(metrics *Metrics) func(*options) { + return func(options *options) { + options.metrics = metrics + } +} + type segmentID uint64 -// Spool keeps encoded DML messages after a writer shard has accepted them and -// before that writer shard has flushed them to external storage. -// -// The producer is the cloud storage writer path: after encoderGroup has -// produced encoded messages for a task, writer.Enqueue calls Spool.Enqueue to -// hand those messages to local spool storage. -// -// The consumer is also the cloud storage writer path: when the writer flushes a -// batch, it calls Spool.Load to read the queued messages back, then calls -// Spool.Release after a successful flush or Spool.Discard when the batch is -// ignored. +// Spool keeps encoded sink messages after the sink has accepted them and before +// it has flushed them to external storage. A sink releases an entry only after +// a successful flush, or discards it when the corresponding data is ignored. type Spool struct { keyspace string changefeed string @@ -308,7 +319,7 @@ func (e *Entry) InMemory() bool { // New return a spool that manages unflushed data. func New( changefeedID commonType.ChangeFeedID, - opts ...option, + opts ...func(*options), ) (*Spool, error) { cfg := defaultOptions() for _, opt := range opts { @@ -326,20 +337,55 @@ func New( keyspace = changefeedID.Keyspace() changefeed = changefeedID.Name() ) + spoolMetrics := normalizeMetrics(cfg.metrics) spool := &Spool{ keyspace: keyspace, changefeed: changefeed, workDir: workDir, - quota: newQuotaController(changefeedID, cfg), + quota: newQuotaController(cfg), segmentCapacity: cfg.segmentCapacity, - metricLoadedBytes: metrics.CloudStorageLoadBytesHistogram.WithLabelValues(keyspace, changefeed), - metricRotatedCount: metrics.CloudStorageRotateCountCounter.WithLabelValues(keyspace, changefeed), - metricSegmentCount: metrics.CloudStorageSpoolSegmentCountGauge.WithLabelValues(keyspace, changefeed), + metricLoadedBytes: spoolMetrics.LoadedBytes, + metricRotatedCount: spoolMetrics.RotatedCount, + metricSegmentCount: spoolMetrics.SegmentCount, segments: make(map[segmentID]*segment), } return spool, nil } +func normalizeMetrics(spoolMetrics *Metrics) *Metrics { + if spoolMetrics == nil { + spoolMetrics = &Metrics{} + } + if spoolMetrics.MemoryBytes == nil { + spoolMetrics.MemoryBytes = prometheus.NewGauge(prometheus.GaugeOpts{}) + } + if spoolMetrics.DiskBytes == nil { + spoolMetrics.DiskBytes = prometheus.NewGauge(prometheus.GaugeOpts{}) + } + if spoolMetrics.PendingPostEnqueue == nil { + spoolMetrics.PendingPostEnqueue = prometheus.NewGauge(prometheus.GaugeOpts{}) + } + if spoolMetrics.DiskQuotaWaiters == nil { + spoolMetrics.DiskQuotaWaiters = prometheus.NewGauge(prometheus.GaugeOpts{}) + } + if spoolMetrics.DiskQuotaWait == nil { + spoolMetrics.DiskQuotaWait = prometheus.NewHistogram(prometheus.HistogramOpts{}) + } + if spoolMetrics.LoadedBytes == nil { + spoolMetrics.LoadedBytes = prometheus.NewHistogram(prometheus.HistogramOpts{}) + } + if spoolMetrics.RotatedCount == nil { + spoolMetrics.RotatedCount = prometheus.NewCounter(prometheus.CounterOpts{}) + } + if spoolMetrics.SegmentCount == nil { + spoolMetrics.SegmentCount = prometheus.NewGauge(prometheus.GaugeOpts{}) + } + if spoolMetrics.Close == nil { + spoolMetrics.Close = func() {} + } + return spoolMetrics +} + func defaultOptions() *options { return &options{ diskQuotaBytes: defaultDiskQuotaBytes, @@ -678,9 +724,6 @@ func (s *Spool) Close() { zap.String("keyspace", s.keyspace), zap.String("changefeed", s.changefeed), zap.String("path", s.workDir), zap.Error(err)) } - metrics.CloudStorageLoadBytesHistogram.DeleteLabelValues(s.keyspace, s.changefeed) - metrics.CloudStorageRotateCountCounter.DeleteLabelValues(s.keyspace, s.changefeed) - metrics.CloudStorageSpoolSegmentCountGauge.DeleteLabelValues(s.keyspace, s.changefeed) s.quota.deleteMetrics() } diff --git a/downstreamadapter/sink/cloudstorage/spool/spool_test.go b/pkg/sink/spool/spool_test.go similarity index 97% rename from downstreamadapter/sink/cloudstorage/spool/spool_test.go rename to pkg/sink/spool/spool_test.go index 4c230305fc..62aa8e727f 100644 --- a/downstreamadapter/sink/cloudstorage/spool/spool_test.go +++ b/pkg/sink/spool/spool_test.go @@ -201,9 +201,10 @@ func TestNewUsesDefaultOptionsWhenValuesAreMissing(t *testing.T) { require.NoError(t, err) require.NotNil(t, manager) require.Equal(t, defaultSegmentCapacity, manager.segmentCapacity) - require.Equal(t, int64(float64(expectedQuotaBytes)*defaultMemoryRatio), manager.quota.budget.memoryQuotaBytes) - require.Equal(t, int64(float64(expectedQuotaBytes)*defaultHighWatermarkRatio), manager.quota.budget.highWatermarkBytes) - require.Equal(t, int64(float64(expectedQuotaBytes)*defaultLowWatermarkRatio), manager.quota.budget.lowWatermarkBytes) + limits := manager.quota.budget.Limits() + require.Equal(t, int64(float64(expectedQuotaBytes)*defaultMemoryRatio), limits.MemoryQuotaBytes) + require.Equal(t, int64(float64(expectedQuotaBytes)*defaultHighWatermarkRatio), limits.HighWatermarkBytes) + require.Equal(t, int64(float64(expectedQuotaBytes)*defaultLowWatermarkRatio), limits.LowWatermarkBytes) manager.Close() } @@ -272,9 +273,10 @@ func TestNewSanitizesInvalidOptions(t *testing.T) { require.NoError(t, err) require.NotNil(t, manager) require.Equal(t, defaultSegmentCapacity, manager.segmentCapacity) - require.Equal(t, int64(float64(expectedQuotaBytes)*defaultMemoryRatio), manager.quota.budget.memoryQuotaBytes) - require.Equal(t, int64(float64(expectedQuotaBytes)*defaultHighWatermarkRatio), manager.quota.budget.highWatermarkBytes) - require.Equal(t, int64(float64(expectedQuotaBytes)*defaultLowWatermarkRatio), manager.quota.budget.lowWatermarkBytes) + limits := manager.quota.budget.Limits() + require.Equal(t, int64(float64(expectedQuotaBytes)*defaultMemoryRatio), limits.MemoryQuotaBytes) + require.Equal(t, int64(float64(expectedQuotaBytes)*defaultHighWatermarkRatio), limits.HighWatermarkBytes) + require.Equal(t, int64(float64(expectedQuotaBytes)*defaultLowWatermarkRatio), limits.LowWatermarkBytes) manager.Close() } @@ -295,9 +297,10 @@ func TestNewAppliesFunctionalOptions(t *testing.T) { require.NoError(t, err) require.Equal(t, filepath.Join(baseDir, changefeedID.Keyspace(), changefeedID.Name()), manager.workDir) require.Equal(t, int64(4096), manager.segmentCapacity) - require.Equal(t, int64(512), manager.quota.budget.memoryQuotaBytes) - require.Equal(t, int64(1536), manager.quota.budget.highWatermarkBytes) - require.Equal(t, int64(1024), manager.quota.budget.lowWatermarkBytes) + limits := manager.quota.budget.Limits() + require.Equal(t, int64(512), limits.MemoryQuotaBytes) + require.Equal(t, int64(1536), limits.HighWatermarkBytes) + require.Equal(t, int64(1024), limits.LowWatermarkBytes) manager.Close() } diff --git a/tests/integration_tests/api_v2/cases.go b/tests/integration_tests/api_v2/cases.go index c0e489b598..bc2900339f 100644 --- a/tests/integration_tests/api_v2/cases.go +++ b/tests/integration_tests/api_v2/cases.go @@ -24,6 +24,7 @@ import ( "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/ticdc/pkg/redo" "github.com/pingcap/ticdc/pkg/util" "go.uber.org/zap" ) @@ -107,6 +108,8 @@ var customReplicaConfig = &ReplicaConfig{ UseFileBackend: false, EncoderWorkerNum: 31, FlushWorkerNum: 18, + SpoolDiskQuota: redo.DefaultSpoolDiskQuota, + SpoolBaseDir: "", }, } @@ -160,6 +163,8 @@ var defaultReplicaConfig = &ReplicaConfig{ FlushWorkerNum: 8, Storage: "", UseFileBackend: false, + SpoolDiskQuota: redo.DefaultSpoolDiskQuota, + SpoolBaseDir: "", }, } diff --git a/tests/integration_tests/api_v2/model.go b/tests/integration_tests/api_v2/model.go index 12ea18c63b..da36306aa5 100644 --- a/tests/integration_tests/api_v2/model.go +++ b/tests/integration_tests/api_v2/model.go @@ -279,6 +279,8 @@ type ConsistentConfig struct { FlushWorkerNum int `json:"flush_worker_num"` Storage string `json:"storage"` UseFileBackend bool `json:"use_file_backend"` + SpoolDiskQuota int64 `json:"spool_disk_quota"` + SpoolBaseDir string `json:"spool_base_dir"` } // ChangefeedSchedulerConfig is per changefeed scheduler settings.