From 96055e4e6f7cab9da7184695717a9fbb619b868b Mon Sep 17 00:00:00 2001 From: Dylan Date: Tue, 22 Sep 2026 11:38:18 +0800 Subject: [PATCH 1/2] [fix](iceberg) Allow nested decimal precision widening ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: Iceberg allows a DECIMAL precision increase when the scale is unchanged, but the connector's complex-column diff only allowed integer and floating-point promotions. As a result, ALTER TABLE MODIFY COLUMN rejected STRUCT to STRUCT, breaking Trino Iceberg compatibility tests. Delegate nested primitive promotion validation to Iceberg TypeUtil and cover the persisted top-level complex MODIFY path. Update the existing test that previously asserted the now-supported decimal widening must fail. ### Release note Iceberg complex column changes now support safe nested decimal precision widening. ### Check List (For Author) - Test: FE Iceberg unit tests (84 tests) and external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy (1 suite) - Behavior changed: Yes (safe nested Iceberg decimal precision widening is accepted) - Does this need documentation: No --- .../iceberg/IcebergComplexTypeDiff.java | 21 +++++++------------ ...dIcebergCatalogOpsColumnEvolutionTest.java | 17 ++++++++------- .../IcebergNestedColumnEvolutionTest.java | 20 ++++++++++++++++++ 3 files changed, 36 insertions(+), 22 deletions(-) diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergComplexTypeDiff.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergComplexTypeDiff.java index ad8ab08a917ce2..a7fae7b79683fd 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergComplexTypeDiff.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergComplexTypeDiff.java @@ -22,6 +22,7 @@ import org.apache.iceberg.UpdateSchema; import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; import java.util.HashSet; @@ -50,7 +51,7 @@ * {@code UpdateSchema.commit()}, so any guard throwing aborts the whole change atomically.

* *

Supported shape changes (legacy parity): widen an existing nested field's primitive type (only the - * iceberg-representable safe promotions int→long, float→double, or an exact match), change a nested + * iceberg-representable safe promotions, including same-scale decimal precision widening), change a nested * field's comment, widen a NOT NULL nested field to nullable, and append new (nullable) STRUCT fields. The * category of every nested level must stay the same (struct/array/map); struct fields may not be renamed, * reordered, dropped, or narrowed to NOT NULL; a MAP key type may not change.

@@ -310,21 +311,13 @@ private static void requireValueNotNarrowed(Types.MapType oldMap, Types.MapType /** * Whether changing a nested primitive {@code oldType} to {@code newType} is a legal promotion, mirroring - * legacy {@code ColumnType.checkSupportSchemaChangeForNestedPrimitive} restricted to the iceberg-representable - * cases: an exact match (covers VARCHAR length growth, which both map to iceberg STRING), INT→BIGINT - * (iceberg INTEGER→LONG), and FLOAT→DOUBLE. Everything else (e.g. a nested DECIMAL precision change, - * any narrowing, a category change) is rejected — matching legacy's restrictive nested rule. + * Iceberg's primitive-promotion rules. Besides INTEGER→LONG and FLOAT→DOUBLE, Iceberg allows a + * DECIMAL precision increase when scale is unchanged. Delegating to Iceberg keeps the connector's validation + * aligned with the UpdateSchema operation it is about to commit. */ private static boolean isLegalNestedPrimitivePromotion(Type oldType, Type newType) { - if (oldType.equals(newType)) { - return true; - } - Type.TypeID oldId = oldType.typeId(); - Type.TypeID newId = newType.typeId(); - if (oldId == Type.TypeID.INTEGER && newId == Type.TypeID.LONG) { - return true; - } - return oldId == Type.TypeID.FLOAT && newId == Type.TypeID.DOUBLE; + return newType.isPrimitiveType() + && TypeUtil.isPromotionAllowed(oldType, newType.asPrimitiveType()); } /** The iceberg type category (struct/list/map) of {@code newType} must equal {@code oldType}'s. */ diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/CatalogBackedIcebergCatalogOpsColumnEvolutionTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/CatalogBackedIcebergCatalogOpsColumnEvolutionTest.java index 450ae232695798..e23b0cfb309aae 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/CatalogBackedIcebergCatalogOpsColumnEvolutionTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/CatalogBackedIcebergCatalogOpsColumnEvolutionTest.java @@ -606,17 +606,18 @@ public void testModifyStructNewFieldNotNullableFailsLoud() { } @Test - public void testModifyNestedDecimalPrecisionFailsLoud() { - // Legacy parity: a nested primitive change is restricted to int->long / float->double / exact; a - // DECIMAL precision change inside a struct is rejected (checkSupportSchemaChangeForNestedPrimitive). + public void testModifyNestedDecimalPrecisionWidens() { + // Iceberg permits a nested DECIMAL precision increase when the scale stays fixed. createTable("s_dec", new ConnectorColumn("st", structType(Arrays.asList("a"), Arrays.asList(ConnectorType.of("DECIMALV3", 10, 2)), Arrays.asList(true), Arrays.asList((String) null)), "", true, null, false)); - DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class, - () -> modifyComplex("s_dec", "st", - structType(Arrays.asList("a"), Arrays.asList(ConnectorType.of("DECIMALV3", 20, 2)), - Arrays.asList(true), Arrays.asList((String) null)), true)); - Assertions.assertTrue(ex.getMessage().contains("nested")); + modifyComplex("s_dec", "st", + structType(Arrays.asList("a"), Arrays.asList(ConnectorType.of("DECIMALV3", 20, 2)), + Arrays.asList(true), Arrays.asList((String) null)), true); + Types.DecimalType amount = (Types.DecimalType) reload("s_dec").findField("st") + .type().asStructType().field("a").type(); + Assertions.assertEquals(20, amount.precision()); + Assertions.assertEquals(2, amount.scale()); } @Test diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergNestedColumnEvolutionTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergNestedColumnEvolutionTest.java index 0f74d82f7358db..c2029243c3a709 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergNestedColumnEvolutionTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergNestedColumnEvolutionTest.java @@ -713,6 +713,26 @@ public void testModifyNestedPrimitivePromotionAllowed() { Assertions.assertEquals(Type.TypeID.LONG, metric.type().typeId()); } + @Test + public void testModifyNestedStructDecimalPrecisionPromotionAllowed() { + // Iceberg permits DECIMAL precision widening when the scale stays fixed. This is the same complex MODIFY + // issued by the Trino product test for STRUCT -> STRUCT. + Schema schema = new Schema(Types.NestedField.optional(1, "info", Types.StructType.of( + Types.NestedField.optional(2, "amount", Types.DecimalType.of(5, 3))))); + createTable("m_decimal_promote", schema); + + ops.modifyColumn("db1", "m_decimal_promote", + structModify("info", Collections.singletonList("amount"), + Collections.singletonList(ConnectorType.of("DECIMALV3", 10, 3)), + Collections.singletonList(""), Collections.singletonList(false)), + false, null); + + Types.DecimalType amount = (Types.DecimalType) reload("m_decimal_promote").findField("info") + .type().asStructType().field("amount").type(); + Assertions.assertEquals(10, amount.precision()); + Assertions.assertEquals(3, amount.scale()); + } + @Test public void testModifyNestedPrimitivePromotionDisallowedFailsLoud() { // BIGINT -> INT is not an iceberg-representable promotion. From 3cb1726293219f2652367a569e4459e5f2ef581e Mon Sep 17 00:00:00 2001 From: Dylan Date: Wed, 23 Sep 2026 12:16:12 +0800 Subject: [PATCH 2/2] [test](regression) Add nested decimal evolution coverage --- ...st_iceberg_nested_schema_evolution_ddl.out | 6 ++++ ...iceberg_nested_schema_evolution_ddl.groovy | 30 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.out index 93826ea80b7925..485f9d0b540347 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.out @@ -10,3 +10,9 @@ m_scalar map -- !query_rows -- 1 \N 10 \N \N 100 \N \N 1000 \N \N 7 70 2 first 20 after_a c2 200 202 201 2000 2002 2001 8 80 + +-- !nested_decimal_schema -- +struct + +-- !nested_decimal_old_row -- +1 12.345 diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy index 012bc88e3d9faf..7ca5b4b875cdfb 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy @@ -172,4 +172,34 @@ suite("test_iceberg_nested_schema_evolution_ddl", "p0,external,doris,external_do FROM ${tableName} ORDER BY id """ + + // Trino compatibility: an existing nested decimal may widen its precision without changing its scale. + sql """DROP TABLE IF EXISTS iceberg_nested_decimal_evolution""" + sql """ + CREATE TABLE iceberg_nested_decimal_evolution ( + id INT, + info STRUCT + ) + """ + sql """ + INSERT INTO iceberg_nested_decimal_evolution VALUES + (1, STRUCT(CAST(12.345 AS DECIMAL(5,3)))) + """ + sql """ + ALTER TABLE iceberg_nested_decimal_evolution + MODIFY COLUMN info STRUCT + """ + + qt_nested_decimal_schema """ + SELECT COLUMN_TYPE + FROM ${catalogName}.information_schema.columns + WHERE TABLE_SCHEMA = '${dbName}' AND TABLE_NAME = 'iceberg_nested_decimal_evolution' + AND COLUMN_NAME = 'info' + """ + + order_qt_nested_decimal_old_row """ + SELECT id, element_at(info, 'amount') + FROM iceberg_nested_decimal_evolution + ORDER BY id + """ }