From 0b728996a0fb9d8d22ffdafb8269caea8af20f79 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Tue, 25 Aug 2026 21:12:57 +0800 Subject: [PATCH] init --- .../utils/clickhouse/ClickHouseTestSettings.scala | 3 +++ .../gluten/utils/velox/VeloxTestSettings.scala | 4 ++++ .../sql/catalyst/expressions/GlutenCastSuite.scala | 12 ++++++++++++ .../utils/clickhouse/ClickHouseTestSettings.scala | 11 +++++++++++ .../gluten/utils/velox/VeloxTestSettings.scala | 4 ++++ .../sql/catalyst/expressions/GlutenCastSuite.scala | 12 ++++++++++++ .../gluten/utils/velox/VeloxTestSettings.scala | 4 ++++ .../expressions/GlutenCastWithAnsiOffSuite.scala | 5 ++--- .../gluten/utils/velox/VeloxTestSettings.scala | 4 ++++ .../expressions/GlutenCastWithAnsiOffSuite.scala | 5 ++--- 10 files changed, 58 insertions(+), 6 deletions(-) diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 214d677cdde..3493681aad5 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -551,6 +551,9 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("cast from boolean") .exclude("data type casting") .excludeGlutenTest("data type casting") + // The Gluten rewrite of "cast from timestamp II" is not vetted on ClickHouse; + // the vanilla case is excluded separately in this block. + .excludeGlutenTest("cast from timestamp II") .exclude("cast between string and interval") .exclude("SPARK-27671: cast from nested null type in struct") .exclude("Process Infinity, -Infinity, NaN in case insensitive manner") diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 635ff6cbc57..c4bc847af52 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -100,6 +100,8 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("data type casting") // Revised by setting timezone through config and commented unsupported cases. .exclude("cast string to timestamp") + // Excluded in favour of the GlutenCastSuite rewrite, which drops the Long.MinValue + // assertion: collect() -> toJavaTimestamp -> rebaseGregorianToJulianMicros overflows. .exclude("cast from timestamp II") .exclude("SPARK-36286: invalid string cast to timestamp") .exclude("SPARK-39749: cast Decimal to string") @@ -629,6 +631,8 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenInsertSuite] // the native write staing dir is differnt with vanilla Spark for coustom partition paths .exclude("SPARK-35106: Throw exception when rename custom partition paths returns false") + // The case expects a SparkException; Gluten surfaces the raw + // FileAlreadyExistsException instead. .exclude("Stop task set if FileAlreadyExistsException was thrown") // Rewrite: Additional support for file scan with default values has been added in Spark-3.4. // It appends the default value in record if it is not present while scanning. diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala index 0e1af4a4e2c..0be74fbda77 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala @@ -165,6 +165,18 @@ class GlutenCastSuite extends CastWithAnsiOffSuite with GlutenTestsTrait { checkEvaluation(cast(false, TimestampType), tsFalse) } + // Gluten's glutenCheckExpression uses collect(), which triggers + // toJavaTimestamp -> rebaseGregorianToJulianMicros. Long.MinValue micros (~292000 BC) overflows + // during rebase, so the vanilla case's Long.MinValue assertion is dropped here. + testGluten("cast from timestamp II") { + checkEvaluation(cast(Double.NaN, TimestampType), null) + checkEvaluation(cast(1.0 / 0.0, TimestampType), null) + checkEvaluation(cast(Float.NaN, TimestampType), null) + checkEvaluation(cast(1.0f / 0.0f, TimestampType), null) + checkEvaluation(cast(Literal(Long.MaxValue), TimestampType), Long.MaxValue) + // Long.MinValue is not asserted; see the comment above the test. + } + testGluten("cast string to timestamp") { DebuggableThreadUtils.parmap( ALL_TIMEZONES diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 920554f91e8..49d637681b5 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -374,6 +374,9 @@ class ClickHouseTestSettings extends BackendTestSettings { .excludeCH("SPARK-33291: Cast struct with null elements to string") .excludeCH("SPARK-35111: Cast string to year-month interval") .excludeCH("Gluten - data type casting") + // The Gluten rewrite of "cast from timestamp II" is not vetted on ClickHouse; + // the vanilla case is excluded separately in this block. + .excludeGlutenTest("cast from timestamp II") .exclude("cast string to date #2") .exclude("casting to fixed-precision decimals") .exclude("SPARK-28470: Cast should honor nullOnOverflow property") @@ -1180,6 +1183,8 @@ class ClickHouseTestSettings extends BackendTestSettings { enableSuite[GlutenMathExpressionsSuite] // Spark round UT for round(3.1415,3) is not correct. .exclude("round/bround/floor/ceil") + // TANH(-0.1) returns -0.0996695958408681 on ClickHouse; the case expects + // -0.09966799462495582. .excludeCH("tanh") .excludeCH("unhex") .excludeCH("atan2") @@ -2195,8 +2200,14 @@ class ClickHouseTestSettings extends BackendTestSettings { .excludeCH("cast from timestamp II") .excludeCH("cast a timestamp before the epoch 1970-01-01 00:00:00Z II") .excludeCH("cast a timestamp before the epoch 1970-01-01 00:00:00Z") + // Casting the string array ("123", "true", "f") to array should yield + // [null, true, false] under try_cast; ClickHouse throws instead. .excludeCH("cast from array II") + // TRY-mode overflow inside a complex type wraps instead of yielding null on + // ClickHouse: try_cast([2.147483648E9] as array) returns [-2147483648]. .excludeCH("cast from array III") + // Same as "cast from array III": try_cast([2.147483648E9] as struct) + // returns [-2147483648] on ClickHouse. .excludeCH("cast from struct III") .excludeCH("ANSI mode: cast string to timestamp with parse error") .excludeCH("ANSI mode: cast string to date with parse error") diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 69ab3bc0d9f..538ca284176 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -104,6 +104,8 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("data type casting") // Revised by setting timezone through config and commented unsupported cases. .exclude("cast string to timestamp") + // Excluded in favour of the GlutenCastSuite rewrite, which drops the Long.MinValue + // assertion: collect() -> toJavaTimestamp -> rebaseGregorianToJulianMicros overflows. .exclude("cast from timestamp II") .exclude("SPARK-36286: invalid string cast to timestamp") .exclude("SPARK-39749: cast Decimal to string") @@ -589,6 +591,8 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenInsertSuite] // the native write staing dir is differnt with vanilla Spark for coustom partition paths .exclude("SPARK-35106: Throw exception when rename custom partition paths returns false") + // The case expects a SparkException; Gluten surfaces the raw + // FileAlreadyExistsException instead. .exclude("Stop task set if FileAlreadyExistsException was thrown") // Rewrite: Additional support for file scan with default values has been added in Spark-3.4. // It appends the default value in record if it is not present while scanning. diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala index ef06a985397..2e9990fe338 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala @@ -168,6 +168,18 @@ class GlutenCastSuite extends CastWithAnsiOffSuite with GlutenTestsTrait { checkEvaluation(cast(false, TimestampType), tsFalse) } + // Gluten's glutenCheckExpression uses collect(), which triggers + // toJavaTimestamp -> rebaseGregorianToJulianMicros. Long.MinValue micros (~292000 BC) overflows + // during rebase, so the vanilla case's Long.MinValue assertion is dropped here. + testGluten("cast from timestamp II") { + checkEvaluation(cast(Double.NaN, TimestampType), null) + checkEvaluation(cast(1.0 / 0.0, TimestampType), null) + checkEvaluation(cast(Float.NaN, TimestampType), null) + checkEvaluation(cast(1.0f / 0.0f, TimestampType), null) + checkEvaluation(cast(Literal(Long.MaxValue), TimestampType), Long.MaxValue) + // Long.MinValue is not asserted; see the comment above the test. + } + testGluten("cast string to timestamp") { DebuggableThreadUtils.parmap( ALL_TIMEZONES diff --git a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 93a7f9d1ef2..756c1b70b28 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -114,6 +114,8 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("data type casting") // Revised by setting timezone through config and commented unsupported cases. .exclude("cast string to timestamp") + // Excluded in favour of the GlutenCastWithAnsiOffSuite rewrite, which drops the Long.MinValue + // assertion: collect() -> toJavaTimestamp -> rebaseGregorianToJulianMicros overflows. .exclude("cast from timestamp II") .exclude("SPARK-36286: invalid string cast to timestamp") .exclude("SPARK-39749: cast Decimal to string") @@ -859,6 +861,8 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("SPARK-24583 Wrong schema type in InsertIntoDataSourceCommand") // the native write staing dir is differnt with vanilla Spark for coustom partition paths .exclude("SPARK-35106: Throw exception when rename custom partition paths returns false") + // The case expects a SparkException; Gluten surfaces the raw + // FileAlreadyExistsException instead. .exclude("Stop task set if FileAlreadyExistsException was thrown") // Rewrite: Additional support for file scan with default values has been added in Spark-3.4. // It appends the default value in record if it is not present while scanning. diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala index 3e195542656..9a8f5567d0b 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala @@ -134,15 +134,14 @@ class GlutenCastWithAnsiOffSuite extends CastWithAnsiOffSuite with GlutenExpress // Gluten's glutenCheckExpression uses collect(), which triggers // toJavaTimestamp -> rebaseGregorianToJulianMicros. Long.MinValue micros (~292000 BC) overflows - // during rebase. Velox computes correctly; only the collect path fails. Skip Long.MinValue. + // during rebase, so the vanilla case's Long.MinValue assertion is dropped here. testGluten("cast from timestamp II") { checkEvaluation(cast(Double.NaN, TimestampType), null) checkEvaluation(cast(1.0 / 0.0, TimestampType), null) checkEvaluation(cast(Float.NaN, TimestampType), null) checkEvaluation(cast(1.0f / 0.0f, TimestampType), null) checkEvaluation(cast(Literal(Long.MaxValue), TimestampType), Long.MaxValue) - // Skip Long.MinValue: Velox result is correct but collect() path overflows in - // rebaseGregorianToJulianMicros when converting extreme timestamp to java.sql.Timestamp. + // Long.MinValue is not asserted; see the comment above the test. } // Sync session timezone with per-expression timezone and run single-threaded. diff --git a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 9d8e5cdafef..1abdd67f9b5 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -117,6 +117,8 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("data type casting") // Revised by setting timezone through config and commented unsupported cases. .exclude("cast string to timestamp") + // Excluded in favour of the GlutenCastWithAnsiOffSuite rewrite, which drops the Long.MinValue + // assertion: collect() -> toJavaTimestamp -> rebaseGregorianToJulianMicros overflows. .exclude("cast from timestamp II") .exclude("SPARK-36286: invalid string cast to timestamp") .exclude("SPARK-39749: cast Decimal to string") @@ -840,6 +842,8 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("SPARK-24583 Wrong schema type in InsertIntoDataSourceCommand") // the native write staing dir is differnt with vanilla Spark for coustom partition paths .exclude("SPARK-35106: Throw exception when rename custom partition paths returns false") + // The case expects a SparkException; Gluten surfaces the raw + // FileAlreadyExistsException instead. .exclude("Stop task set if FileAlreadyExistsException was thrown") // Rewrite: Additional support for file scan with default values has been added in Spark-3.4. // It appends the default value in record if it is not present while scanning. diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala index a0d758aaeac..446fa995aa4 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala @@ -132,15 +132,14 @@ class GlutenCastWithAnsiOffSuite // Gluten's glutenCheckExpression uses collect(), which triggers // toJavaTimestamp -> rebaseGregorianToJulianMicros. Long.MinValue micros (~292000 BC) overflows - // during rebase. Velox computes correctly; only the collect path fails. Skip Long.MinValue. + // during rebase, so the vanilla case's Long.MinValue assertion is dropped here. testGluten("cast from timestamp II") { checkEvaluation(cast(Double.NaN, TimestampType), null) checkEvaluation(cast(1.0 / 0.0, TimestampType), null) checkEvaluation(cast(Float.NaN, TimestampType), null) checkEvaluation(cast(1.0f / 0.0f, TimestampType), null) checkEvaluation(cast(Literal(Long.MaxValue), TimestampType), Long.MaxValue) - // Skip Long.MinValue: Velox result is correct but collect() path overflows in - // rebaseGregorianToJulianMicros when converting extreme timestamp to java.sql.Timestamp. + // Long.MinValue is not asserted; see the comment above the test. } // Sync session timezone with per-expression timezone and run single-threaded.