From cf5dc4505f978fa3596da67e3544fe97c3164573 Mon Sep 17 00:00:00 2001 From: gengliqi Date: Mon, 13 Oct 2025 19:44:56 +0800 Subject: [PATCH 01/16] fix Signed-off-by: gengliqi --- dbms/src/Storages/StorageDisaggregated.h | 2 +- .../Storages/StorageDisaggregatedRemote.cpp | 83 ++++++++++--------- 2 files changed, 47 insertions(+), 38 deletions(-) diff --git a/dbms/src/Storages/StorageDisaggregated.h b/dbms/src/Storages/StorageDisaggregated.h index 33c81d94db8..727c70caa63 100644 --- a/dbms/src/Storages/StorageDisaggregated.h +++ b/dbms/src/Storages/StorageDisaggregated.h @@ -107,7 +107,7 @@ class StorageDisaggregated : public IStorage std::tuple buildRSOperatorAndColumnRange( const Context & db_context, const DM::ColumnDefinesPtr & columns_to_read); - std::variant packSegmentReadTasks( + std::tuple, DM::ColumnDefinesPtr> packSegmentReadTasks( const Context & db_context, DM::SegmentReadTasks && read_tasks, const DM::ColumnDefinesPtr & column_defines, diff --git a/dbms/src/Storages/StorageDisaggregatedRemote.cpp b/dbms/src/Storages/StorageDisaggregatedRemote.cpp index e7101dc6036..ad29805f765 100644 --- a/dbms/src/Storages/StorageDisaggregatedRemote.cpp +++ b/dbms/src/Storages/StorageDisaggregatedRemote.cpp @@ -564,12 +564,13 @@ std::tuple StorageDisaggregated::buildRSO return {rs_operator, column_range}; } -std::variant StorageDisaggregated::packSegmentReadTasks( - const Context & db_context, - DM::SegmentReadTasks && read_tasks, - const DM::ColumnDefinesPtr & column_defines, - size_t num_streams, - int extra_table_id_index) +std::tuple, DM::ColumnDefinesPtr> StorageDisaggregated:: + packSegmentReadTasks( + const Context & db_context, + DM::SegmentReadTasks && read_tasks, + const DM::ColumnDefinesPtr & column_defines, + size_t num_streams, + int extra_table_id_index) { const auto & executor_id = table_scan.getTableScanExecutorID(); @@ -604,48 +605,56 @@ std::variant StorageDisagg push_down_executor); const UInt64 start_ts = sender_target_mpp_task_id.gather_id.query_id.start_ts; const auto enable_read_thread = db_context.getSettingsRef().dt_enable_read_thread; + const auto & final_columns_defines = push_down_executor && push_down_executor->extra_cast + ? push_down_executor->columns_after_cast + : column_defines; LOG_INFO( log, "packSegmentReadTasks: enable_read_thread={} read_mode={} is_fast_scan={} keep_order={} task_count={} " - "num_streams={} column_defines={}", + "num_streams={} column_defines={}, final_columns_defines={}", enable_read_thread, magic_enum::enum_name(read_mode), table_scan.isFastScan(), table_scan.keepOrder(), read_tasks.size(), num_streams, - *column_defines); + *column_defines, + *final_columns_defines); if (enable_read_thread) { - return std::make_shared( - extra_table_id_index, - *column_defines, - push_down_executor, - start_ts, - db_context.getSettingsRef().max_block_size, - read_mode, - std::move(read_tasks), - /*after_segment_read*/ [](const DM::DMContextPtr &, const DM::SegmentPtr &) {}, - executor_id, - /*enable_read_thread*/ true, - num_streams, - context.getDAGContext()->getKeyspaceID(), - context.getDAGContext()->getResourceGroupName()); + return { + std::make_shared( + extra_table_id_index, + *final_columns_defines, + push_down_executor, + start_ts, + db_context.getSettingsRef().max_block_size, + read_mode, + std::move(read_tasks), + /*after_segment_read*/ [](const DM::DMContextPtr &, const DM::SegmentPtr &) {}, + executor_id, + /*enable_read_thread*/ true, + num_streams, + context.getDAGContext()->getKeyspaceID(), + context.getDAGContext()->getResourceGroupName()), + final_columns_defines}; } else { - return DM::Remote::RNWorkers::create( - db_context, - std::move(read_tasks), - { - .log = log->getChild(executor_id), - .columns_to_read = column_defines, - .start_ts = start_ts, - .push_down_executor = push_down_executor, - .read_mode = read_mode, - }, - num_streams); + return { + DM::Remote::RNWorkers::create( + db_context, + std::move(read_tasks), + { + .log = log->getChild(executor_id), + .columns_to_read = final_columns_defines, + .start_ts = start_ts, + .push_down_executor = push_down_executor, + .read_mode = read_mode, + }, + num_streams), + final_columns_defines}; } } @@ -687,14 +696,14 @@ void StorageDisaggregated::buildRemoteSegmentInputStreams( { // Build the input streams to read blocks from remote segments auto [column_defines, extra_table_id_index] = genColumnDefinesForDisaggregatedRead(table_scan); - auto packed_read_tasks + auto [packed_read_tasks, final_column_defines] = packSegmentReadTasks(db_context, std::move(read_tasks), column_defines, num_streams, extra_table_id_index); RUNTIME_CHECK(num_streams > 0, num_streams); pipeline.streams.reserve(num_streams); InputStreamBuilder builder{ .tracing_id = log->identifier(), - .columns_to_read = column_defines, + .columns_to_read = final_column_defines, .extra_table_id_index = extra_table_id_index, }; for (size_t stream_idx = 0; stream_idx < num_streams; ++stream_idx) @@ -753,13 +762,13 @@ void StorageDisaggregated::buildRemoteSegmentSourceOps( { // Build the input streams to read blocks from remote segments auto [column_defines, extra_table_id_index] = genColumnDefinesForDisaggregatedRead(table_scan); - auto packed_read_tasks + auto [packed_read_tasks, final_column_defines] = packSegmentReadTasks(db_context, std::move(read_tasks), column_defines, num_streams, extra_table_id_index); RUNTIME_CHECK(num_streams > 0, num_streams); SrouceOpBuilder builder{ .tracing_id = log->identifier(), - .column_defines = column_defines, + .column_defines = final_column_defines, .extra_table_id_index = extra_table_id_index, .exec_context = exec_context, }; From 892e01e2e01aeee5efa1fc89e55ba967170e3d0f Mon Sep 17 00:00:00 2001 From: gengliqi Date: Tue, 14 Oct 2025 23:46:30 +0800 Subject: [PATCH 02/16] fix more bugs Signed-off-by: gengliqi --- dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp | 4 ---- tests/fullstack-test/mpp/extra_physical_table_column.test | 3 --- 2 files changed, 7 deletions(-) diff --git a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp index 9502a7d6ebf..4eb6479d0da 100644 --- a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp +++ b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp @@ -117,10 +117,6 @@ std::tuple genColumnDefinesForDisaggregatedRead(const break; case MutSup::extra_table_id_col_id: { - column_defines->emplace_back(DM::ColumnDefine{ - MutSup::extra_table_id_col_id, - output_name, // MutSup::extra_table_id_column_name - MutSup::getExtraTableIdColumnType()}); extra_table_id_index = i; break; } diff --git a/tests/fullstack-test/mpp/extra_physical_table_column.test b/tests/fullstack-test/mpp/extra_physical_table_column.test index 2fe6af2d1f0..098f95c7d4e 100644 --- a/tests/fullstack-test/mpp/extra_physical_table_column.test +++ b/tests/fullstack-test/mpp/extra_physical_table_column.test @@ -12,9 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# line 27: Pessimistic lock response corrupted -#SKIP_FOR_NEXT_GEN - # Preparation. => DBGInvoke __init_fail_point() From b74596984086b0180f777c21f65e494ef3756825 Mon Sep 17 00:00:00 2001 From: gengliqi Date: Fri, 7 Nov 2025 03:12:40 +0800 Subject: [PATCH 03/16] fix the generated column bug Signed-off-by: gengliqi --- dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp | 14 ++++++++++++-- dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h | 5 +++-- dbms/src/Storages/StorageDisaggregated.h | 2 ++ dbms/src/Storages/StorageDisaggregatedRemote.cpp | 14 ++++++++++++-- 4 files changed, 29 insertions(+), 6 deletions(-) diff --git a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp index 4eb6479d0da..86844cb7710 100644 --- a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp +++ b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include @@ -94,14 +95,23 @@ NamesAndTypes genNamesAndTypes(const TiDBTableScan & table_scan, const StringRef return genNamesAndTypes(table_scan.getColumns(), column_prefix); } -std::tuple genColumnDefinesForDisaggregatedRead(const TiDBTableScan & table_scan) +std::tuple>> genColumnDefinesForDisaggregatedRead( + const TiDBTableScan & table_scan) { auto column_defines = std::make_shared(); int extra_table_id_index = MutSup::invalid_col_id; column_defines->reserve(table_scan.getColumnSize()); + std::vector> generated_column_infos; for (Int32 i = 0; i < table_scan.getColumnSize(); ++i) { const auto & column_info = table_scan.getColumns()[i]; + if (column_info.hasGeneratedColumnFlag()) + { + const auto & data_type = getDataTypeByColumnInfoForComputingLayer(column_info); + const auto & col_name = GeneratedColumnPlaceholderBlockInputStream::getColumnName(i); + generated_column_infos.push_back(std::make_tuple(i, col_name, data_type)); + continue; + } // Now the upper level seems treat disagg read as an ExchangeReceiver output, so // use this as output column prefix. // Even if the id is pk_column or extra_table_id, we still output it as @@ -129,7 +139,7 @@ std::tuple genColumnDefinesForDisaggregatedRead(const break; } } - return {std::move(column_defines), extra_table_id_index}; + return {std::move(column_defines), extra_table_id_index, std::move(generated_column_infos)}; } ColumnsWithTypeAndName getColumnWithTypeAndName(const NamesAndTypes & names_and_types) diff --git a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h index 4b6281e8a77..1eb87891ba9 100644 --- a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h +++ b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h @@ -34,7 +34,8 @@ NamesAndTypes genNamesAndTypes(const TiDB::ColumnInfos & column_infos, const Str ColumnsWithTypeAndName getColumnWithTypeAndName(const NamesAndTypes & names_and_types); NamesAndTypes toNamesAndTypes(const DAGSchema & dag_schema); -// The column defines and `extra table id index` -std::tuple genColumnDefinesForDisaggregatedRead(const TiDBTableScan & table_scan); +// The column defines, `extra table id index` and `generated columns info` for disaggregated read. +std::tuple>> genColumnDefinesForDisaggregatedRead( + const TiDBTableScan & table_scan); } // namespace DB diff --git a/dbms/src/Storages/StorageDisaggregated.h b/dbms/src/Storages/StorageDisaggregated.h index 727c70caa63..8748de8d2e4 100644 --- a/dbms/src/Storages/StorageDisaggregated.h +++ b/dbms/src/Storages/StorageDisaggregated.h @@ -157,5 +157,7 @@ class StorageDisaggregated : public IStorage std::unique_ptr analyzer; static constexpr auto ZONE_LABEL_KEY = "zone"; std::optional zone_label; + // For generated column, just need a placeholder, and TiDB will fill this column. + std::vector> generated_column_infos; }; } // namespace DB diff --git a/dbms/src/Storages/StorageDisaggregatedRemote.cpp b/dbms/src/Storages/StorageDisaggregatedRemote.cpp index ad29805f765..945801756c3 100644 --- a/dbms/src/Storages/StorageDisaggregatedRemote.cpp +++ b/dbms/src/Storages/StorageDisaggregatedRemote.cpp @@ -105,6 +105,8 @@ BlockInputStreams StorageDisaggregated::readThroughS3(const Context & db_context // Build InputStream according to the remote segment read tasks DAGPipeline pipeline; buildRemoteSegmentInputStreams(db_context, buildReadTaskWithBackoff(db_context), num_streams, pipeline); + // handle generated column if necessary. + executeGeneratedColumnPlaceholder(generated_column_infos, log, pipeline); NamesAndTypes source_columns; source_columns.reserve(table_scan.getColumnSize()); @@ -134,6 +136,8 @@ void StorageDisaggregated::readThroughS3( db_context, buildReadTaskWithBackoff(db_context), num_streams); + // handle generated column if necessary. + executeGeneratedColumnPlaceholder(exec_context, group_builder, generated_column_infos, log); NamesAndTypes source_columns; auto header = group_builder.getCurrentHeader(); @@ -695,7 +699,10 @@ void StorageDisaggregated::buildRemoteSegmentInputStreams( DAGPipeline & pipeline) { // Build the input streams to read blocks from remote segments - auto [column_defines, extra_table_id_index] = genColumnDefinesForDisaggregatedRead(table_scan); + DM::ColumnDefinesPtr column_defines; + int extra_table_id_index; + std::tie(column_defines, extra_table_id_index, generated_column_infos) + = genColumnDefinesForDisaggregatedRead(table_scan); auto [packed_read_tasks, final_column_defines] = packSegmentReadTasks(db_context, std::move(read_tasks), column_defines, num_streams, extra_table_id_index); RUNTIME_CHECK(num_streams > 0, num_streams); @@ -761,7 +768,10 @@ void StorageDisaggregated::buildRemoteSegmentSourceOps( size_t num_streams) { // Build the input streams to read blocks from remote segments - auto [column_defines, extra_table_id_index] = genColumnDefinesForDisaggregatedRead(table_scan); + DM::ColumnDefinesPtr column_defines; + int extra_table_id_index; + std::tie(column_defines, extra_table_id_index, generated_column_infos) + = genColumnDefinesForDisaggregatedRead(table_scan); auto [packed_read_tasks, final_column_defines] = packSegmentReadTasks(db_context, std::move(read_tasks), column_defines, num_streams, extra_table_id_index); From 160b035aa2aed0751b1702de7979e014f2038ce2 Mon Sep 17 00:00:00 2001 From: gengliqi Date: Fri, 7 Nov 2025 03:22:13 +0800 Subject: [PATCH 04/16] enable next-gen tests for expr and mpp Signed-off-by: gengliqi --- tests/fullstack-test-next-gen/run.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/fullstack-test-next-gen/run.sh b/tests/fullstack-test-next-gen/run.sh index d7192820896..eff81b40fb6 100755 --- a/tests/fullstack-test-next-gen/run.sh +++ b/tests/fullstack-test-next-gen/run.sh @@ -75,9 +75,9 @@ if [[ -n "$ENABLE_NEXT_GEN" && "$ENABLE_NEXT_GEN" != "false" && "$ENABLE_NEXT_GE ${COMPOSE} -f next-gen-cluster.yaml -f "${DISAGG_TIFLASH_YAML}" exec -T tiflash-cn0 bash -c "cd /tests ; ${ENV_ARGS} ./run-test.sh fullstack-test2/dml" ${COMPOSE} -f next-gen-cluster.yaml -f "${DISAGG_TIFLASH_YAML}" exec -T tiflash-cn0 bash -c "cd /tests ; ${ENV_ARGS} ./run-test.sh fullstack-test2/variables" ${COMPOSE} -f next-gen-cluster.yaml -f "${DISAGG_TIFLASH_YAML}" exec -T tiflash-cn0 bash -c "cd /tests ; ${ENV_ARGS} ./run-test.sh fullstack-test2/mpp" - # TODO: enable the following tests after they are fixed. And maybe we need to split them into parallel pipelines because they take too long to run. - #${COMPOSE} -f next-gen-cluster.yaml -f "${DISAGG_TIFLASH_YAML}" exec -T tiflash-cn0 bash -c "cd /tests ; ${ENV_ARGS} ./run-test.sh fullstack-test/expr" - #${COMPOSE} -f next-gen-cluster.yaml -f "${DISAGG_TIFLASH_YAML}" exec -T tiflash-cn0 bash -c "cd /tests ; ${ENV_ARGS} ./run-test.sh fullstack-test/mpp" + # maybe we need to split them into parallel pipelines because they take too long to run. + ${COMPOSE} -f next-gen-cluster.yaml -f "${DISAGG_TIFLASH_YAML}" exec -T tiflash-cn0 bash -c "cd /tests ; ${ENV_ARGS} ./run-test.sh fullstack-test/expr" + ${COMPOSE} -f next-gen-cluster.yaml -f "${DISAGG_TIFLASH_YAML}" exec -T tiflash-cn0 bash -c "cd /tests ; ${ENV_ARGS} ./run-test.sh fullstack-test/mpp" ${COMPOSE} -f next-gen-cluster.yaml -f "${DISAGG_TIFLASH_YAML}" down clean_data_log From d6e0a476bc99767049e258e28b1a7e68f2056aaa Mon Sep 17 00:00:00 2001 From: gengliqi Date: Fri, 7 Nov 2025 03:28:03 +0800 Subject: [PATCH 05/16] enable more tests Signed-off-by: gengliqi --- tests/fullstack-test-next-gen/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fullstack-test-next-gen/run.sh b/tests/fullstack-test-next-gen/run.sh index eff81b40fb6..5f828cf4b66 100755 --- a/tests/fullstack-test-next-gen/run.sh +++ b/tests/fullstack-test-next-gen/run.sh @@ -69,7 +69,7 @@ if [[ -n "$ENABLE_NEXT_GEN" && "$ENABLE_NEXT_GEN" != "false" && "$ENABLE_NEXT_GE ENV_ARGS="ENABLE_NEXT_GEN=true verbose=${verbose} " # most failpoints are expected to be set on the compute layer, use tiflash-cn0 to run tests ${COMPOSE} -f next-gen-cluster.yaml -f "${DISAGG_TIFLASH_YAML}" exec -T tiflash-cn0 bash -c "cd /tests ; ${ENV_ARGS} ./run-test.sh fullstack-test/sample.test" - ${COMPOSE} -f next-gen-cluster.yaml -f "${DISAGG_TIFLASH_YAML}" exec -T tiflash-cn0 bash -c "cd /tests ; ${ENV_ARGS} ./run-test.sh fullstack-test-index/vector" + ${COMPOSE} -f next-gen-cluster.yaml -f "${DISAGG_TIFLASH_YAML}" exec -T tiflash-cn0 bash -c "cd /tests ; ${ENV_ARGS} ./run-test.sh fullstack-test-index" ${COMPOSE} -f next-gen-cluster.yaml -f "${DISAGG_TIFLASH_YAML}" exec -T tiflash-cn0 bash -c "cd /tests ; ${ENV_ARGS} ./run-test.sh fullstack-test-next-gen/placement" ${COMPOSE} -f next-gen-cluster.yaml -f "${DISAGG_TIFLASH_YAML}" exec -T tiflash-cn0 bash -c "cd /tests ; ${ENV_ARGS} ./run-test.sh fullstack-test2/clustered_index" ${COMPOSE} -f next-gen-cluster.yaml -f "${DISAGG_TIFLASH_YAML}" exec -T tiflash-cn0 bash -c "cd /tests ; ${ENV_ARGS} ./run-test.sh fullstack-test2/dml" From 45875acbfefd5367ad00977de330509e9e36c0c1 Mon Sep 17 00:00:00 2001 From: gengliqi Date: Fri, 7 Nov 2025 14:17:09 +0800 Subject: [PATCH 06/16] add tikv-worker for next-gen tests Signed-off-by: gengliqi --- tests/docker/next-gen-config/tikv-worker.toml | 11 +++++++++++ tests/docker/next-gen-yaml/cluster.yaml | 14 ++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 tests/docker/next-gen-config/tikv-worker.toml diff --git a/tests/docker/next-gen-config/tikv-worker.toml b/tests/docker/next-gen-config/tikv-worker.toml new file mode 100644 index 00000000000..89864d1d826 --- /dev/null +++ b/tests/docker/next-gen-config/tikv-worker.toml @@ -0,0 +1,11 @@ +[dfs] +prefix = "tikv" +s3-endpoint = "http://minio0:9000" +s3-key-id = "minioadmin" +s3-secret-key = "minioadmin" +s3-bucket = "tiflash-test" +s3-region = "local" + +[ia] +mem-cap = "5GB" # or "10%" +disk-cap = "20GB" # or "20%" diff --git a/tests/docker/next-gen-yaml/cluster.yaml b/tests/docker/next-gen-yaml/cluster.yaml index 28bd63f2748..5234bf580e8 100644 --- a/tests/docker/next-gen-yaml/cluster.yaml +++ b/tests/docker/next-gen-yaml/cluster.yaml @@ -49,6 +49,19 @@ services: - "pd0" - "minio0" restart: on-failure + tikv-worker0: + image: ${TIKV_IMAGE:-us-docker.pkg.dev/pingcap-testing-account/hub/tikv/tikv/image:dedicated-next-gen} + security_opt: + - seccomp:unconfined + volumes: + - ./next-gen-config/tikv-worker.toml:/tikv-worker.toml:ro + - ./data/tikv-worker0:/data + - ./log/tikv-worker0:/log + entrypoint: /tikv-worker + command: --config=/tikv-worker.toml --data-dir=/data --addr=0.0.0.0:19000 --advertise-addr=tikv-worker0:19000 --pd-endpoints=pd0:2379 --log-file=/log/tikv-worker.log + depends_on: + - "pd0" + restart: on-failure tidb0: image: ${TIDB_IMAGE:-us-docker.pkg.dev/pingcap-testing-account/hub/pingcap/tidb/images/tidb-server:master-next-gen} security_opt: @@ -60,3 +73,4 @@ services: depends_on: - "tikv0" restart: on-failure + From 5eadeaaec2e0a7d2eec1f6c7d87098adbf624579 Mon Sep 17 00:00:00 2001 From: gengliqi Date: Fri, 7 Nov 2025 14:47:03 +0800 Subject: [PATCH 07/16] add license Signed-off-by: gengliqi --- tests/docker/next-gen-config/tikv-worker.toml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/docker/next-gen-config/tikv-worker.toml b/tests/docker/next-gen-config/tikv-worker.toml index 89864d1d826..8bbd0475aaf 100644 --- a/tests/docker/next-gen-config/tikv-worker.toml +++ b/tests/docker/next-gen-config/tikv-worker.toml @@ -1,3 +1,18 @@ +# Copyright 2025 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# object storage for next-gen [dfs] prefix = "tikv" s3-endpoint = "http://minio0:9000" From 8ab0b8849223ffc769e5d8c3c2a60f1e2c8f7414 Mon Sep 17 00:00:00 2001 From: gengliqi Date: Tue, 18 Nov 2025 16:01:50 +0800 Subject: [PATCH 08/16] u Signed-off-by: gengliqi --- tests/docker/next-gen-yaml/cluster.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/docker/next-gen-yaml/cluster.yaml b/tests/docker/next-gen-yaml/cluster.yaml index 5234bf580e8..c0acdf2418c 100644 --- a/tests/docker/next-gen-yaml/cluster.yaml +++ b/tests/docker/next-gen-yaml/cluster.yaml @@ -58,7 +58,7 @@ services: - ./data/tikv-worker0:/data - ./log/tikv-worker0:/log entrypoint: /tikv-worker - command: --config=/tikv-worker.toml --data-dir=/data --addr=0.0.0.0:19000 --advertise-addr=tikv-worker0:19000 --pd-endpoints=pd0:2379 --log-file=/log/tikv-worker.log + command: --addr=0.0.0.0:19000 --pd-endpoints=pd0:2379 --config=/tikv-worker.toml --data-dir=/data --log-file=/log/tikv-worker.log depends_on: - "pd0" restart: on-failure @@ -73,4 +73,3 @@ services: depends_on: - "tikv0" restart: on-failure - From 1d23e5e84e5eab74a77d6e02f4d2b132c9ed778d Mon Sep 17 00:00:00 2001 From: gengliqi Date: Tue, 18 Nov 2025 16:52:26 +0800 Subject: [PATCH 09/16] update tikv-worker.toml Signed-off-by: gengliqi --- tests/docker/next-gen-config/tikv-worker.toml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/docker/next-gen-config/tikv-worker.toml b/tests/docker/next-gen-config/tikv-worker.toml index 8bbd0475aaf..d637964c02a 100644 --- a/tests/docker/next-gen-config/tikv-worker.toml +++ b/tests/docker/next-gen-config/tikv-worker.toml @@ -21,6 +21,8 @@ s3-secret-key = "minioadmin" s3-bucket = "tiflash-test" s3-region = "local" -[ia] -mem-cap = "5GB" # or "10%" -disk-cap = "20GB" # or "20%" +[schema-manager] +dir = "/data/schemas" +enabled = true +keyspace-refresh-interval = "10s" +schema-refresh-threshold = 1 \ No newline at end of file From fda22b2827f1714e87c47f8e37974069dfb730de Mon Sep 17 00:00:00 2001 From: gengliqi Date: Tue, 18 Nov 2025 17:06:57 +0800 Subject: [PATCH 10/16] update tidb.toml Signed-off-by: gengliqi --- tests/docker/next-gen-config/tidb.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/docker/next-gen-config/tidb.toml b/tests/docker/next-gen-config/tidb.toml index 4c1c7e6c764..3e87edbddf0 100644 --- a/tests/docker/next-gen-config/tidb.toml +++ b/tests/docker/next-gen-config/tidb.toml @@ -19,6 +19,8 @@ disaggregated-tiflash = true # Now tests are ran on the SYSTEM keyspace tidb. keyspace-name = "SYSTEM" +tikv-worker-url = "0.0.0.0:19000" + enable-telemetry = false temp-dir = "/data/tmp" [performance] From cb326d519a7bc4584f9d39a560a526a6805cbde2 Mon Sep 17 00:00:00 2001 From: gengliqi Date: Tue, 18 Nov 2025 17:15:56 +0800 Subject: [PATCH 11/16] update tidb.toml Signed-off-by: gengliqi --- tests/docker/next-gen-config/tidb.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/docker/next-gen-config/tidb.toml b/tests/docker/next-gen-config/tidb.toml index 3e87edbddf0..40cc6e9e68a 100644 --- a/tests/docker/next-gen-config/tidb.toml +++ b/tests/docker/next-gen-config/tidb.toml @@ -19,7 +19,7 @@ disaggregated-tiflash = true # Now tests are ran on the SYSTEM keyspace tidb. keyspace-name = "SYSTEM" -tikv-worker-url = "0.0.0.0:19000" +tikv-worker-url = "tikv-worker0:19000" enable-telemetry = false temp-dir = "/data/tmp" From 90fa5376d5651f30af94eb2693428da375697792 Mon Sep 17 00:00:00 2001 From: gengliqi Date: Tue, 18 Nov 2025 18:03:57 +0800 Subject: [PATCH 12/16] update tikv.toml Signed-off-by: gengliqi --- tests/docker/next-gen-config/tikv.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/docker/next-gen-config/tikv.toml b/tests/docker/next-gen-config/tikv.toml index 0e0fe67baee..3daca2b0099 100644 --- a/tests/docker/next-gen-config/tikv.toml +++ b/tests/docker/next-gen-config/tikv.toml @@ -18,6 +18,7 @@ reserve-space = "0" # Enable keyspace and ttl for next-gen api-version = 2 enable-ttl = true +low_space_threshold = 0 [raftstore] capacity = "100GB" From dfe784ef285d50793520e6a1ec17a124594f6c88 Mon Sep 17 00:00:00 2001 From: gengliqi Date: Tue, 18 Nov 2025 18:06:59 +0800 Subject: [PATCH 13/16] update tikv.toml again Signed-off-by: gengliqi --- tests/docker/next-gen-config/tikv.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/docker/next-gen-config/tikv.toml b/tests/docker/next-gen-config/tikv.toml index 3daca2b0099..d00976dc9ca 100644 --- a/tests/docker/next-gen-config/tikv.toml +++ b/tests/docker/next-gen-config/tikv.toml @@ -18,7 +18,7 @@ reserve-space = "0" # Enable keyspace and ttl for next-gen api-version = 2 enable-ttl = true -low_space_threshold = 0 +low-space-threshold = 0 [raftstore] capacity = "100GB" From f269ba2e2035318c634d7c583b76e8025d6685ed Mon Sep 17 00:00:00 2001 From: gengliqi Date: Tue, 18 Nov 2025 19:02:41 +0800 Subject: [PATCH 14/16] disable unixTimeStamp.test for next-gen Signed-off-by: gengliqi --- tests/fullstack-test/expr/unixTimeStamp.test | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/fullstack-test/expr/unixTimeStamp.test b/tests/fullstack-test/expr/unixTimeStamp.test index 7858baab2ad..9262e64cf15 100644 --- a/tests/fullstack-test/expr/unixTimeStamp.test +++ b/tests/fullstack-test/expr/unixTimeStamp.test @@ -12,6 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +# next-gen does not support unix_timestamp for now. +# will enable after fixing this issue. +#SKIP_FOR_NEXT_GEN + mysql> drop table if exists test.t mysql> create table test.t(a date, b datetime, c timestamp(3), d timestamp(6)); mysql> alter table test.t set tiflash replica 1; From 2378d5ffb036c5cd13e54cf75da34aca869a3e37 Mon Sep 17 00:00:00 2001 From: gengliqi Date: Tue, 18 Nov 2025 20:12:34 +0800 Subject: [PATCH 15/16] u Signed-off-by: gengliqi --- tests/docker/next-gen-config/tikv-worker.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/docker/next-gen-config/tikv-worker.toml b/tests/docker/next-gen-config/tikv-worker.toml index d637964c02a..bb2a52ba88f 100644 --- a/tests/docker/next-gen-config/tikv-worker.toml +++ b/tests/docker/next-gen-config/tikv-worker.toml @@ -25,4 +25,4 @@ s3-region = "local" dir = "/data/schemas" enabled = true keyspace-refresh-interval = "10s" -schema-refresh-threshold = 1 \ No newline at end of file +schema-refresh-threshold = 1 From 39459a1d7f4d69b009e44a74ab6236c3c6f02d0a Mon Sep 17 00:00:00 2001 From: gengliqi Date: Wed, 26 Nov 2025 20:38:46 +0800 Subject: [PATCH 16/16] address comments Signed-off-by: gengliqi --- dbms/src/Storages/StorageDisaggregatedRemote.cpp | 2 +- tests/fullstack-test/expr/unixTimeStamp.test | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/dbms/src/Storages/StorageDisaggregatedRemote.cpp b/dbms/src/Storages/StorageDisaggregatedRemote.cpp index 5eb8111a70a..0f870a83813 100644 --- a/dbms/src/Storages/StorageDisaggregatedRemote.cpp +++ b/dbms/src/Storages/StorageDisaggregatedRemote.cpp @@ -663,7 +663,7 @@ std::tuple, D LOG_INFO( log, "packSegmentReadTasks: enable_read_thread={} read_mode={} is_fast_scan={} keep_order={} task_count={} " - "num_streams={} column_defines={}, final_columns_defines={}", + "num_streams={} column_defines={} final_columns_defines={}", enable_read_thread, magic_enum::enum_name(read_mode), table_scan.isFastScan(), diff --git a/tests/fullstack-test/expr/unixTimeStamp.test b/tests/fullstack-test/expr/unixTimeStamp.test index 9262e64cf15..7858baab2ad 100644 --- a/tests/fullstack-test/expr/unixTimeStamp.test +++ b/tests/fullstack-test/expr/unixTimeStamp.test @@ -12,10 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# next-gen does not support unix_timestamp for now. -# will enable after fixing this issue. -#SKIP_FOR_NEXT_GEN - mysql> drop table if exists test.t mysql> create table test.t(a date, b datetime, c timestamp(3), d timestamp(6)); mysql> alter table test.t set tiflash replica 1;