Skip to content

[SPARK-57257][SQL] Support nanosecond-precision timestamps in Hive results #56320

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
MaxGekk wants to merge 2 commits into from
+672 −4
Closed

[SPARK-57257][SQL] Support nanosecond-precision timestamps in Hive results #56320

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e281132
[SPARK-57257][SQL] Support nanosecond-precision timestamps in Hive re…
MaxGekk Jun 4, 2026
d578fbb
[SPARK-57257][SQL][TESTS] Add pre-1970 and NULL coverage for nanos ti…
MaxGekk Jun 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,16 @@ object HiveResult extends SQLConfHelper {
case (t: Timestamp, TimestampType) => formatters.timestamp.format(t)
case (i: Instant, TimestampType) => formatters.timestamp.format(i)
case (l: LocalDateTime, TimestampNTZType) => formatters.timestamp.format(l)
// Nanosecond-precision timestamps. The external values are `Instant` (LTZ) and
// `LocalDateTime` (NTZ); convert to the physical `TimestampNanosVal` at the column precision
// and render via the same formatter methods as the cast-to-string path (SPARK-57256), so the
// output stays consistent. LTZ uses the session zone; NTZ is zone-independent.
case (i: Instant, t: TimestampLTZNanosType) =>
formatters.timestamp.formatNanos(
DateTimeUtils.instantToTimestampNanos(i, t.precision), t.precision)
case (l: LocalDateTime, t: TimestampNTZNanosType) =>
formatters.timestamp.formatWithoutTimeZoneNanos(
DateTimeUtils.localDateTimeToTimestampNanos(l, t.precision), t.precision)
case (bin: Array[Byte], BinaryType) => binaryFormatter(bin)
case (decimal: java.math.BigDecimal, DecimalType()) => decimal.toPlainString
case (n, _: NumericType) => n.toString
Expand Down
118 changes: 118 additions & 0 deletions sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
-- Automatically generated by SQLQueryTestSuite
-- !query
SELECT CAST('2020-01-01 00:00:00.123456789' AS timestamp_ltz(7))
-- !query analysis
Project [cast(2020-01-01 00:00:00.123456789 as timestamp_ltz(7)) AS CAST(2020-01-01 00:00:00.123456789 AS TIMESTAMP_LTZ(7))#x]
+- OneRowRelation


-- !query
SELECT CAST('2020-01-01 00:00:00.123456789' AS timestamp_ltz(8))
-- !query analysis
Project [cast(2020-01-01 00:00:00.123456789 as timestamp_ltz(8)) AS CAST(2020-01-01 00:00:00.123456789 AS TIMESTAMP_LTZ(8))#x]
+- OneRowRelation


-- !query
SELECT CAST('2020-01-01 00:00:00.123456789' AS timestamp_ltz(9))
-- !query analysis
Project [cast(2020-01-01 00:00:00.123456789 as timestamp_ltz(9)) AS CAST(2020-01-01 00:00:00.123456789 AS TIMESTAMP_LTZ(9))#x]
+- OneRowRelation


-- !query
SELECT CAST('2020-01-01 00:00:00.999999000' AS timestamp_ltz(9))
-- !query analysis
Project [cast(2020-01-01 00:00:00.999999000 as timestamp_ltz(9)) AS CAST(2020-01-01 00:00:00.999999000 AS TIMESTAMP_LTZ(9))#x]
+- OneRowRelation


-- !query
SELECT CAST('2020-01-01 00:00:00.000000999' AS timestamp_ltz(9))
-- !query analysis
Project [cast(2020-01-01 00:00:00.000000999 as timestamp_ltz(9)) AS CAST(2020-01-01 00:00:00.000000999 AS TIMESTAMP_LTZ(9))#x]
+- OneRowRelation


-- !query
SELECT CAST('2020-01-01 00:00:00.000000001' AS timestamp_ltz(9))
-- !query analysis
Project [cast(2020-01-01 00:00:00.000000001 as timestamp_ltz(9)) AS CAST(2020-01-01 00:00:00.000000001 AS TIMESTAMP_LTZ(9))#x]
+- OneRowRelation


-- !query
SELECT CAST('2020-01-01 00:00:00.000000001' AS timestamp_ltz(8))
-- !query analysis
Project [cast(2020-01-01 00:00:00.000000001 as timestamp_ltz(8)) AS CAST(2020-01-01 00:00:00.000000001 AS TIMESTAMP_LTZ(8))#x]
+- OneRowRelation


-- !query
SELECT CAST('2020-01-01 00:00:00.000000001' AS timestamp_ltz(7))
-- !query analysis
Project [cast(2020-01-01 00:00:00.000000001 as timestamp_ltz(7)) AS CAST(2020-01-01 00:00:00.000000001 AS TIMESTAMP_LTZ(7))#x]
+- OneRowRelation


-- !query
SELECT CAST('1960-01-01 00:00:00.000000001' AS timestamp_ltz(9))
-- !query analysis
Project [cast(1960-01-01 00:00:00.000000001 as timestamp_ltz(9)) AS CAST(1960-01-01 00:00:00.000000001 AS TIMESTAMP_LTZ(9))#x]
+- OneRowRelation


-- !query
SELECT CAST('1960-01-01 00:00:00.123456789' AS timestamp_ltz(7))
-- !query analysis
Project [cast(1960-01-01 00:00:00.123456789 as timestamp_ltz(7)) AS CAST(1960-01-01 00:00:00.123456789 AS TIMESTAMP_LTZ(7))#x]
+- OneRowRelation


-- !query
SELECT array(CAST('2020-01-01 00:00:00.123456789' AS timestamp_ltz(9)))
-- !query analysis
Project [array(cast(2020-01-01 00:00:00.123456789 as timestamp_ltz(9))) AS array(CAST(2020-01-01 00:00:00.123456789 AS TIMESTAMP_LTZ(9)))#x]
+- OneRowRelation


-- !query
SELECT map('k', CAST('2020-01-01 00:00:00.123456789' AS timestamp_ltz(9)))
-- !query analysis
Project [map(k, cast(2020-01-01 00:00:00.123456789 as timestamp_ltz(9))) AS map(k, CAST(2020-01-01 00:00:00.123456789 AS TIMESTAMP_LTZ(9)))#x]
+- OneRowRelation


-- !query
SELECT named_struct('f', CAST('2020-01-01 00:00:00.123456789' AS timestamp_ltz(9)))
-- !query analysis
Project [named_struct(f, cast(2020-01-01 00:00:00.123456789 as timestamp_ltz(9))) AS named_struct(f, CAST(2020-01-01 00:00:00.123456789 AS TIMESTAMP_LTZ(9)))#x]
+- OneRowRelation


-- !query
SELECT CAST(NULL AS timestamp_ltz(9))
-- !query analysis
Project [cast(null as timestamp_ltz(9)) AS CAST(NULL AS TIMESTAMP_LTZ(9))#x]
+- OneRowRelation


-- !query
SELECT array(CAST(NULL AS timestamp_ltz(9)))
-- !query analysis
Project [array(cast(null as timestamp_ltz(9))) AS array(CAST(NULL AS TIMESTAMP_LTZ(9)))#x]
+- OneRowRelation


-- !query
SELECT map('k', CAST(NULL AS timestamp_ltz(9)))
-- !query analysis
Project [map(k, cast(null as timestamp_ltz(9))) AS map(k, CAST(NULL AS TIMESTAMP_LTZ(9)))#x]
+- OneRowRelation


-- !query
SELECT named_struct('f', CAST(NULL AS timestamp_ltz(9)))
-- !query analysis
Project [named_struct(f, cast(null as timestamp_ltz(9))) AS named_struct(f, CAST(NULL AS TIMESTAMP_LTZ(9)))#x]
+- OneRowRelation
118 changes: 118 additions & 0 deletions sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
-- Automatically generated by SQLQueryTestSuite
-- !query
SELECT CAST('2020-01-01 00:00:00.123456789' AS timestamp_ntz(7))
-- !query analysis
Project [cast(2020-01-01 00:00:00.123456789 as timestamp_ntz(7)) AS CAST(2020-01-01 00:00:00.123456789 AS TIMESTAMP_NTZ(7))#x]
+- OneRowRelation


-- !query
SELECT CAST('2020-01-01 00:00:00.123456789' AS timestamp_ntz(8))
-- !query analysis
Project [cast(2020-01-01 00:00:00.123456789 as timestamp_ntz(8)) AS CAST(2020-01-01 00:00:00.123456789 AS TIMESTAMP_NTZ(8))#x]
+- OneRowRelation


-- !query
SELECT CAST('2020-01-01 00:00:00.123456789' AS timestamp_ntz(9))
-- !query analysis
Project [cast(2020-01-01 00:00:00.123456789 as timestamp_ntz(9)) AS CAST(2020-01-01 00:00:00.123456789 AS TIMESTAMP_NTZ(9))#x]
+- OneRowRelation


-- !query
SELECT CAST('2020-01-01 00:00:00.999999000' AS timestamp_ntz(9))
-- !query analysis
Project [cast(2020-01-01 00:00:00.999999000 as timestamp_ntz(9)) AS CAST(2020-01-01 00:00:00.999999000 AS TIMESTAMP_NTZ(9))#x]
+- OneRowRelation


-- !query
SELECT CAST('2020-01-01 00:00:00.000000999' AS timestamp_ntz(9))
-- !query analysis
Project [cast(2020-01-01 00:00:00.000000999 as timestamp_ntz(9)) AS CAST(2020-01-01 00:00:00.000000999 AS TIMESTAMP_NTZ(9))#x]
+- OneRowRelation


-- !query
SELECT CAST('2020-01-01 00:00:00.000000001' AS timestamp_ntz(9))
-- !query analysis
Project [cast(2020-01-01 00:00:00.000000001 as timestamp_ntz(9)) AS CAST(2020-01-01 00:00:00.000000001 AS TIMESTAMP_NTZ(9))#x]
+- OneRowRelation


-- !query
SELECT CAST('2020-01-01 00:00:00.000000001' AS timestamp_ntz(8))
-- !query analysis
Project [cast(2020-01-01 00:00:00.000000001 as timestamp_ntz(8)) AS CAST(2020-01-01 00:00:00.000000001 AS TIMESTAMP_NTZ(8))#x]
+- OneRowRelation


-- !query
SELECT CAST('2020-01-01 00:00:00.000000001' AS timestamp_ntz(7))
-- !query analysis
Project [cast(2020-01-01 00:00:00.000000001 as timestamp_ntz(7)) AS CAST(2020-01-01 00:00:00.000000001 AS TIMESTAMP_NTZ(7))#x]
+- OneRowRelation


-- !query
SELECT CAST('1960-01-01 00:00:00.000000001' AS timestamp_ntz(9))
-- !query analysis
Project [cast(1960-01-01 00:00:00.000000001 as timestamp_ntz(9)) AS CAST(1960-01-01 00:00:00.000000001 AS TIMESTAMP_NTZ(9))#x]
+- OneRowRelation


-- !query
SELECT CAST('1960-01-01 00:00:00.123456789' AS timestamp_ntz(7))
-- !query analysis
Project [cast(1960-01-01 00:00:00.123456789 as timestamp_ntz(7)) AS CAST(1960-01-01 00:00:00.123456789 AS TIMESTAMP_NTZ(7))#x]
+- OneRowRelation


-- !query
SELECT array(CAST('2020-01-01 00:00:00.123456789' AS timestamp_ntz(9)))
-- !query analysis
Project [array(cast(2020-01-01 00:00:00.123456789 as timestamp_ntz(9))) AS array(CAST(2020-01-01 00:00:00.123456789 AS TIMESTAMP_NTZ(9)))#x]
+- OneRowRelation


-- !query
SELECT map('k', CAST('2020-01-01 00:00:00.123456789' AS timestamp_ntz(9)))
-- !query analysis
Project [map(k, cast(2020-01-01 00:00:00.123456789 as timestamp_ntz(9))) AS map(k, CAST(2020-01-01 00:00:00.123456789 AS TIMESTAMP_NTZ(9)))#x]
+- OneRowRelation


-- !query
SELECT named_struct('f', CAST('2020-01-01 00:00:00.123456789' AS timestamp_ntz(9)))
-- !query analysis
Project [named_struct(f, cast(2020-01-01 00:00:00.123456789 as timestamp_ntz(9))) AS named_struct(f, CAST(2020-01-01 00:00:00.123456789 AS TIMESTAMP_NTZ(9)))#x]
+- OneRowRelation


-- !query
SELECT CAST(NULL AS timestamp_ntz(9))
-- !query analysis
Project [cast(null as timestamp_ntz(9)) AS CAST(NULL AS TIMESTAMP_NTZ(9))#x]
+- OneRowRelation


-- !query
SELECT array(CAST(NULL AS timestamp_ntz(9)))
-- !query analysis
Project [array(cast(null as timestamp_ntz(9))) AS array(CAST(NULL AS TIMESTAMP_NTZ(9)))#x]
+- OneRowRelation


-- !query
SELECT map('k', CAST(NULL AS timestamp_ntz(9)))
-- !query analysis
Project [map(k, cast(null as timestamp_ntz(9))) AS map(k, CAST(NULL AS TIMESTAMP_NTZ(9)))#x]
+- OneRowRelation


-- !query
SELECT named_struct('f', CAST(NULL AS timestamp_ntz(9)))
-- !query analysis
Project [named_struct(f, cast(null as timestamp_ntz(9))) AS named_struct(f, CAST(NULL AS TIMESTAMP_NTZ(9)))#x]
+- OneRowRelation
32 changes: 32 additions & 0 deletions sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql
View file
Open in desktop
Copy link
Copy Markdown
Member

@uros-b uros-b Jun 5, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be nice to add at least some test cases with null values (both top-level and nested/complex).

Copy link
Copy Markdown
Member Author

@MaxGekk MaxGekk Jun 5, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added. Note: NULLs for these columns are handled by the generic (null, _) branch in HiveResult.toHiveString, which runs before the new TimestampLTZNanosType / TimestampNTZNanosType cases, so the path is type-agnostic -- but it's worth locking in. Added top-level (NULL) and nested array/map/struct NULL cases to both golden files (and the suite). Done in d578fbb.

Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
-- Nanosecond-precision TIMESTAMP_LTZ(p) (p in [7, 9]) in Hive results (SPARK-57257).
-- LTZ values are rendered in the session time zone.

--SET spark.sql.timestampNanosTypes.enabled=true
--SET spark.sql.session.timeZone=America/Los_Angeles

-- Precision-driven fraction width: sub-p digits are floored.
SELECT CAST('2020-01-01 00:00:00.123456789' AS timestamp_ltz(7));
SELECT CAST('2020-01-01 00:00:00.123456789' AS timestamp_ltz(8));
SELECT CAST('2020-01-01 00:00:00.123456789' AS timestamp_ltz(9));

-- Trailing-zero trimming: an all-zero fraction renders as no fraction at all.
SELECT CAST('2020-01-01 00:00:00.999999000' AS timestamp_ltz(9));
SELECT CAST('2020-01-01 00:00:00.000000999' AS timestamp_ltz(9));
SELECT CAST('2020-01-01 00:00:00.000000001' AS timestamp_ltz(9));
SELECT CAST('2020-01-01 00:00:00.000000001' AS timestamp_ltz(8));
SELECT CAST('2020-01-01 00:00:00.000000001' AS timestamp_ltz(7));

-- Pre-1970 values exercise the negative-epoch path.
SELECT CAST('1960-01-01 00:00:00.000000001' AS timestamp_ltz(9));
SELECT CAST('1960-01-01 00:00:00.123456789' AS timestamp_ltz(7));

-- Nested values (array / map / struct).
SELECT array(CAST('2020-01-01 00:00:00.123456789' AS timestamp_ltz(9)));
SELECT map('k', CAST('2020-01-01 00:00:00.123456789' AS timestamp_ltz(9)));
SELECT named_struct('f', CAST('2020-01-01 00:00:00.123456789' AS timestamp_ltz(9)));

-- NULL values (top-level and nested).
SELECT CAST(NULL AS timestamp_ltz(9));
SELECT array(CAST(NULL AS timestamp_ltz(9)));
SELECT map('k', CAST(NULL AS timestamp_ltz(9)));
SELECT named_struct('f', CAST(NULL AS timestamp_ltz(9)));
32 changes: 32 additions & 0 deletions sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
-- Nanosecond-precision TIMESTAMP_NTZ(p) (p in [7, 9]) in Hive results (SPARK-57257).
-- NTZ values are zone-independent.

--SET spark.sql.timestampNanosTypes.enabled=true
--SET spark.sql.session.timeZone=America/Los_Angeles

-- Precision-driven fraction width: sub-p digits are floored.
SELECT CAST('2020-01-01 00:00:00.123456789' AS timestamp_ntz(7));
SELECT CAST('2020-01-01 00:00:00.123456789' AS timestamp_ntz(8));
SELECT CAST('2020-01-01 00:00:00.123456789' AS timestamp_ntz(9));

-- Trailing-zero trimming: an all-zero fraction renders as no fraction at all.
SELECT CAST('2020-01-01 00:00:00.999999000' AS timestamp_ntz(9));
SELECT CAST('2020-01-01 00:00:00.000000999' AS timestamp_ntz(9));
SELECT CAST('2020-01-01 00:00:00.000000001' AS timestamp_ntz(9));
SELECT CAST('2020-01-01 00:00:00.000000001' AS timestamp_ntz(8));
SELECT CAST('2020-01-01 00:00:00.000000001' AS timestamp_ntz(7));

-- Pre-1970 values exercise the negative-epoch path.
SELECT CAST('1960-01-01 00:00:00.000000001' AS timestamp_ntz(9));
SELECT CAST('1960-01-01 00:00:00.123456789' AS timestamp_ntz(7));

-- Nested values (array / map / struct).
SELECT array(CAST('2020-01-01 00:00:00.123456789' AS timestamp_ntz(9)));
SELECT map('k', CAST('2020-01-01 00:00:00.123456789' AS timestamp_ntz(9)));
SELECT named_struct('f', CAST('2020-01-01 00:00:00.123456789' AS timestamp_ntz(9)));

-- NULL values (top-level and nested).
SELECT CAST(NULL AS timestamp_ntz(9));
SELECT array(CAST(NULL AS timestamp_ntz(9)));
SELECT map('k', CAST(NULL AS timestamp_ntz(9)));
SELECT named_struct('f', CAST(NULL AS timestamp_ntz(9)));
Loading