Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@

import org.apache.iceberg.UpdateSchema;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.TypeUtil;
import org.apache.iceberg.types.Types;

import java.util.HashSet;
Expand Down Expand Up @@ -50,7 +51,7 @@
* {@code UpdateSchema.commit()}, so any guard throwing aborts the whole change atomically.</p>
*
* <p><b>Supported shape changes (legacy parity):</b> widen an existing nested field's primitive type (only the
* iceberg-representable safe promotions int&rarr;long, float&rarr;double, or an exact match), change a nested
* iceberg-representable safe promotions, including same-scale decimal precision widening), change a nested
* field's comment, widen a NOT NULL nested field to nullable, and append new (nullable) STRUCT fields. The
* category of every nested level must stay the same (struct/array/map); struct fields may not be renamed,
* reordered, dropped, or narrowed to NOT NULL; a MAP key type may not change.</p>
Expand Down Expand Up @@ -310,21 +311,13 @@ private static void requireValueNotNarrowed(Types.MapType oldMap, Types.MapType

/**
* Whether changing a nested primitive {@code oldType} to {@code newType} is a legal promotion, mirroring
* legacy {@code ColumnType.checkSupportSchemaChangeForNestedPrimitive} restricted to the iceberg-representable
* cases: an exact match (covers VARCHAR length growth, which both map to iceberg STRING), INT&rarr;BIGINT
* (iceberg INTEGER&rarr;LONG), and FLOAT&rarr;DOUBLE. Everything else (e.g. a nested DECIMAL precision change,
* any narrowing, a category change) is rejected — matching legacy's restrictive nested rule.
* Iceberg's primitive-promotion rules. Besides INTEGER&rarr;LONG and FLOAT&rarr;DOUBLE, Iceberg allows a
* DECIMAL precision increase when scale is unchanged. Delegating to Iceberg keeps the connector's validation
* aligned with the UpdateSchema operation it is about to commit.
*/
private static boolean isLegalNestedPrimitivePromotion(Type oldType, Type newType) {
if (oldType.equals(newType)) {
return true;
}
Type.TypeID oldId = oldType.typeId();
Type.TypeID newId = newType.typeId();
if (oldId == Type.TypeID.INTEGER && newId == Type.TypeID.LONG) {
return true;
}
return oldId == Type.TypeID.FLOAT && newId == Type.TypeID.DOUBLE;
return newType.isPrimitiveType()
&& TypeUtil.isPromotionAllowed(oldType, newType.asPrimitiveType());
}

/** The iceberg type category (struct/list/map) of {@code newType} must equal {@code oldType}'s. */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -606,17 +606,18 @@ public void testModifyStructNewFieldNotNullableFailsLoud() {
}

@Test
public void testModifyNestedDecimalPrecisionFailsLoud() {
// Legacy parity: a nested primitive change is restricted to int->long / float->double / exact; a
// DECIMAL precision change inside a struct is rejected (checkSupportSchemaChangeForNestedPrimitive).
public void testModifyNestedDecimalPrecisionWidens() {
// Iceberg permits a nested DECIMAL precision increase when the scale stays fixed.
createTable("s_dec", new ConnectorColumn("st",
structType(Arrays.asList("a"), Arrays.asList(ConnectorType.of("DECIMALV3", 10, 2)),
Arrays.asList(true), Arrays.asList((String) null)), "", true, null, false));
DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class,
() -> modifyComplex("s_dec", "st",
structType(Arrays.asList("a"), Arrays.asList(ConnectorType.of("DECIMALV3", 20, 2)),
Arrays.asList(true), Arrays.asList((String) null)), true));
Assertions.assertTrue(ex.getMessage().contains("nested"));
modifyComplex("s_dec", "st",
structType(Arrays.asList("a"), Arrays.asList(ConnectorType.of("DECIMALV3", 20, 2)),
Arrays.asList(true), Arrays.asList((String) null)), true);
Types.DecimalType amount = (Types.DecimalType) reload("s_dec").findField("st")
.type().asStructType().field("a").type();
Assertions.assertEquals(20, amount.precision());
Assertions.assertEquals(2, amount.scale());
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -713,6 +713,26 @@ public void testModifyNestedPrimitivePromotionAllowed() {
Assertions.assertEquals(Type.TypeID.LONG, metric.type().typeId());
}

@Test
public void testModifyNestedStructDecimalPrecisionPromotionAllowed() {
// Iceberg permits DECIMAL precision widening when the scale stays fixed. This is the same complex MODIFY
// issued by the Trino product test for STRUCT<field:DECIMAL(5,3)> -> STRUCT<field:DECIMAL(10,3)>.
Schema schema = new Schema(Types.NestedField.optional(1, "info", Types.StructType.of(
Types.NestedField.optional(2, "amount", Types.DecimalType.of(5, 3)))));
createTable("m_decimal_promote", schema);

ops.modifyColumn("db1", "m_decimal_promote",
structModify("info", Collections.singletonList("amount"),
Collections.singletonList(ConnectorType.of("DECIMALV3", 10, 3)),
Collections.singletonList(""), Collections.singletonList(false)),
false, null);

Types.DecimalType amount = (Types.DecimalType) reload("m_decimal_promote").findField("info")
.type().asStructType().field("amount").type();
Assertions.assertEquals(10, amount.precision());
Assertions.assertEquals(3, amount.scale());
}

@Test
public void testModifyNestedPrimitivePromotionDisallowedFailsLoud() {
// BIGINT -> INT is not an iceberg-representable promotion.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,9 @@ m_scalar map<string,bigint(20)>
-- !query_rows --
1 \N 10 \N \N 100 \N \N 1000 \N \N 7 70
2 first 20 after_a c2 200 202 201 2000 2002 2001 8 80

-- !nested_decimal_schema --
struct<decimalv3(10, 3)>

-- !nested_decimal_old_row --
1 12.345
Original file line number Diff line number Diff line change
Expand Up @@ -172,4 +172,34 @@ suite("test_iceberg_nested_schema_evolution_ddl", "p0,external,doris,external_do
FROM ${tableName}
ORDER BY id
"""

// Trino compatibility: an existing nested decimal may widen its precision without changing its scale.
sql """DROP TABLE IF EXISTS iceberg_nested_decimal_evolution"""
sql """
CREATE TABLE iceberg_nested_decimal_evolution (
id INT,
info STRUCT<amount:DECIMAL(5,3)>
)
"""
sql """
INSERT INTO iceberg_nested_decimal_evolution VALUES
(1, STRUCT(CAST(12.345 AS DECIMAL(5,3))))
"""
sql """
ALTER TABLE iceberg_nested_decimal_evolution
MODIFY COLUMN info STRUCT<amount:DECIMAL(10,3)>
"""

qt_nested_decimal_schema """
SELECT COLUMN_TYPE
FROM ${catalogName}.information_schema.columns
WHERE TABLE_SCHEMA = '${dbName}' AND TABLE_NAME = 'iceberg_nested_decimal_evolution'
AND COLUMN_NAME = 'info'
"""

order_qt_nested_decimal_old_row """
SELECT id, element_at(info, 'amount')
FROM iceberg_nested_decimal_evolution
ORDER BY id
"""
}
Loading