diff --git a/be/src/core/call_on_type_index.h b/be/src/core/call_on_type_index.h index 98aad077413a56..7056abb6d721ce 100644 --- a/be/src/core/call_on_type_index.h +++ b/be/src/core/call_on_type_index.h @@ -37,6 +37,7 @@ struct TypePair { class DataTypeDate; class DataTypeDateV2; class DataTypeDateTimeV2; +class DataTypeTimeStampNs; class DataTypeDateTime; class DataTypeIPv4; class DataTypeIPv6; @@ -83,6 +84,8 @@ bool call_on_index_and_data_type(PrimitiveType number, F&& f) { return f(TypePair()); case PrimitiveType::TYPE_DATETIMEV2: return f(TypePair()); + case PrimitiveType::TYPE_TIMESTAMP_NS: + return f(TypePair()); case PrimitiveType::TYPE_DATETIME: return f(TypePair()); case PrimitiveType::TYPE_TIMEV2: @@ -236,6 +239,8 @@ bool dispatch_type_base(PrimitiveType number, F&& f) { return f(DispatchDataType()); case PrimitiveType::TYPE_DATETIMEV2: return f(DispatchDataType()); + case PrimitiveType::TYPE_TIMESTAMP_NS: + return f(DispatchDataType()); case PrimitiveType::TYPE_DATETIME: return f(DispatchDataType()); case PrimitiveType::TYPE_TIMEV2: diff --git a/be/src/core/column/column_array.cpp b/be/src/core/column/column_array.cpp index 604c46948539b1..dad91af31c01cf 100644 --- a/be/src/core/column/column_array.cpp +++ b/be/src/core/column/column_array.cpp @@ -874,6 +874,9 @@ ColumnArrayDataOffsets filter_return_new_dispatch(const Filter& filt, ssize_t re return filter_number_return_new(filt, result_size_hint, data, offsets); if (check_and_get_column(data.get())) return filter_number_return_new(filt, result_size_hint, data, offsets); + if (check_and_get_column(data.get())) { + return filter_number_return_new(filt, result_size_hint, data, offsets); + } if (check_and_get_column(data.get())) return filter_number_return_new(filt, result_size_hint, data, offsets); if (check_and_get_column(data.get())) @@ -940,6 +943,9 @@ size_t filter_inplace_dispatch(const Filter& filter, IColumn& src_data, return filter_number_inplace(filter, src_data, src_offsets); if (check_and_get_column(&src_data)) return filter_number_inplace(filter, src_data, src_offsets); + if (check_and_get_column(&src_data)) { + return filter_number_inplace(filter, src_data, src_offsets); + } if (check_and_get_column(&src_data)) return filter_number_inplace(filter, src_data, src_offsets); if (check_and_get_column(&src_data)) diff --git a/be/src/core/column/column_vector.cpp b/be/src/core/column/column_vector.cpp index 31e4862187fb79..8d43364e5a898d 100644 --- a/be/src/core/column/column_vector.cpp +++ b/be/src/core/column/column_vector.cpp @@ -559,6 +559,7 @@ template class ColumnVector; template class ColumnVector; template class ColumnVector; template class ColumnVector; +template class ColumnVector; template class ColumnVector; template class ColumnVector; template class ColumnVector; diff --git a/be/src/core/column/column_vector.h b/be/src/core/column/column_vector.h index 4f3a7a6c46f2b4..8105ade3d02916 100644 --- a/be/src/core/column/column_vector.h +++ b/be/src/core/column/column_vector.h @@ -412,7 +412,8 @@ class ColumnVector final : public COWHelper> { // when run function which need_replace_null_data_to_default, use the value far from 0 to avoid // raise errors for null cell. static value_type default_value() { - if constexpr (is_date_type(T) || T == PrimitiveType::TYPE_TIMESTAMPTZ) { + if constexpr ((is_date_type(T) && T != PrimitiveType::TYPE_TIMESTAMP_NS) || + T == PrimitiveType::TYPE_TIMESTAMPTZ) { return PrimitiveTypeTraits::CppType::DEFAULT_VALUE; } else { return value_type(); diff --git a/be/src/core/column/columns_common.cpp b/be/src/core/column/columns_common.cpp index bfaee36a8e0058..1129032dc07c11 100644 --- a/be/src/core/column/columns_common.cpp +++ b/be/src/core/column/columns_common.cpp @@ -359,6 +359,8 @@ INSTANTIATE(DateV2Value, IColumn::Offset) INSTANTIATE(DateV2Value, ColumnArray::Offset64) INSTANTIATE(DateV2Value, IColumn::Offset) INSTANTIATE(DateV2Value, ColumnArray::Offset64) +INSTANTIATE(TimeStampNsValue, IColumn::Offset) +INSTANTIATE(TimeStampNsValue, ColumnArray::Offset64) INSTANTIATE(TimestampTzValue, IColumn::Offset) INSTANTIATE(TimestampTzValue, ColumnArray::Offset64) INSTANTIATE(UInt32, IColumn::Offset) diff --git a/be/src/core/data_type/convert_field_to_type.cpp b/be/src/core/data_type/convert_field_to_type.cpp index 8873c09346b2c0..c5582d8bca1d0d 100644 --- a/be/src/core/data_type/convert_field_to_type.cpp +++ b/be/src/core/data_type/convert_field_to_type.cpp @@ -58,6 +58,9 @@ void dispatch(F&& f, const Field& field) { case PrimitiveType::TYPE_DATETIMEV2: f(field.template get()); return; + case PrimitiveType::TYPE_TIMESTAMP_NS: + f(field.template get()); + return; case PrimitiveType::TYPE_TIMESTAMPTZ: f(field.template get()); return; @@ -180,6 +183,9 @@ class FieldVisitorToJsonb : public StaticVisitor { void operator()(const DateV2Value& x, JsonbWriter* writer) const { writer->writeInt64(*(UInt64*)&x); } + void operator()(const TimeStampNsValue& x, JsonbWriter* writer) const { + writer->writeInt64(x.epoch_nanos()); + } void operator()(const TimestampTzValue& x, JsonbWriter* writer) const { writer->writeInt64(*(UInt64*)&x); } @@ -828,4 +834,4 @@ void convert_field_to_type(const Field& from_value, const IDataType& to_type, Fi return convert_field_to_typeImpl(from_value, to_type, from_type_hint, to); } } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/core/data_type/data_type.cpp b/be/src/core/data_type/data_type.cpp index 941a0377b05e66..d5980d4a8ecd80 100644 --- a/be/src/core/data_type/data_type.cpp +++ b/be/src/core/data_type/data_type.cpp @@ -126,6 +126,8 @@ PGenericType_TypeId IDataType::get_pdata_type(const IDataType* data_type) { return PGenericType::VARIANT; case PrimitiveType::TYPE_DATETIMEV2: return PGenericType::DATETIMEV2; + case PrimitiveType::TYPE_TIMESTAMP_NS: + return PGenericType::TIMESTAMP_NS; case PrimitiveType::TYPE_TIMESTAMPTZ: return PGenericType::TIMESTAMPTZ; case PrimitiveType::TYPE_BITMAP: diff --git a/be/src/core/data_type/data_type_array.cpp b/be/src/core/data_type/data_type_array.cpp index 1cd003e42893aa..369689a90b40b0 100644 --- a/be/src/core/data_type/data_type_array.cpp +++ b/be/src/core/data_type/data_type_array.cpp @@ -162,7 +162,8 @@ FieldWithDataType DataTypeArray::get_field_with_data_type(const IColumn& column, if (is_decimal(nested_type_id)) { precision = nested_type->get_precision(); scale = nested_type->get_scale(); - } else if (nested_type_id == TYPE_DATETIMEV2 || nested_type_id == TYPE_TIMESTAMPTZ) { + } else if (nested_type_id == TYPE_DATETIMEV2 || nested_type_id == TYPE_TIMESTAMP_NS || + nested_type_id == TYPE_TIMESTAMPTZ) { scale = nested_type->get_scale(); } else if (nested_type_id == TYPE_JSONB) { // Array should return JsonbField as element diff --git a/be/src/core/data_type/data_type_factory.cpp b/be/src/core/data_type/data_type_factory.cpp index 463f79f352a29b..9e2ef01633a19d 100644 --- a/be/src/core/data_type/data_type_factory.cpp +++ b/be/src/core/data_type/data_type_factory.cpp @@ -59,6 +59,7 @@ #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "core/data_type/data_type_time.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/data_type_timestamptz.h" #include "core/data_type/data_type_varbinary.h" #include "core/data_type/data_type_variant.h" @@ -152,6 +153,9 @@ DataTypePtr DataTypeFactory::_create_primitive_data_type(const FieldType& type, case FieldType::OLAP_FIELD_TYPE_DATETIMEV2: result = create_datetimev2(scale); break; + case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS: + result = std::make_shared(); + break; case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ: result = std::make_shared(scale); break; @@ -258,6 +262,9 @@ DataTypePtr DataTypeFactory::create_data_type(const PColumnMeta& pcolumn) { case PGenericType::DATETIMEV2: nested = std::make_shared(pcolumn.decimal_param().scale()); break; + case PGenericType::TIMESTAMP_NS: + nested = std::make_shared(); + break; case PGenericType::DATETIME: nested = std::make_shared(); break; @@ -436,6 +443,9 @@ DataTypePtr DataTypeFactory::create_data_type(const PrimitiveType primitive_type case TYPE_DATETIMEV2: nested = create_datetimev2(scale); break; + case TYPE_TIMESTAMP_NS: + nested = std::make_shared(); + break; case TYPE_DATETIME: nested = std::make_shared(); break; diff --git a/be/src/core/data_type/data_type_number_base.cpp b/be/src/core/data_type/data_type_number_base.cpp index dcd167bce1df0d..15bc934c2594f0 100644 --- a/be/src/core/data_type/data_type_number_base.cpp +++ b/be/src/core/data_type/data_type_number_base.cpp @@ -203,6 +203,7 @@ template class DataTypeNumberBase; template class DataTypeNumberBase; template class DataTypeNumberBase; template class DataTypeNumberBase; +template class DataTypeNumberBase; template class DataTypeNumberBase; template class DataTypeNumberBase; template class DataTypeNumberBase; diff --git a/be/src/core/data_type/data_type_time.h b/be/src/core/data_type/data_type_time.h index afcc97fe8e1853..7ff188c9b2457b 100644 --- a/be/src/core/data_type/data_type_time.h +++ b/be/src/core/data_type/data_type_time.h @@ -26,6 +26,7 @@ #include #include #include +#include #include "core/data_type/data_type.h" #include "core/data_type/data_type_number_base.h" @@ -40,8 +41,10 @@ class IColumn; class DataTypeTimeV2 final : public DataTypeNumberBase { public: + static constexpr UInt32 MAX_SCALE = 6; + DataTypeTimeV2(int scale = 0) : _scale(scale) { - if (UNLIKELY(scale > 6)) { + if (UNLIKELY(std::cmp_greater(scale, MAX_SCALE))) { throw doris::Exception(ErrorCode::INTERNAL_ERROR, "Scale {} is out of bounds", scale); } if (scale == -1) { diff --git a/be/src/core/data_type/data_type_timestamp_ns.cpp b/be/src/core/data_type/data_type_timestamp_ns.cpp new file mode 100644 index 00000000000000..80d5651fc0c7ba --- /dev/null +++ b/be/src/core/data_type/data_type_timestamp_ns.cpp @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/data_type/data_type_timestamp_ns.h" + +#include + +#include +#include +#include + +#include "common/exception.h" +#include "core/assert_cast.h" +#include "core/column/column_vector.h" +#include "core/string_ref.h" +#include "core/value/timestamp_ns_value.h" + +namespace doris { + +Field DataTypeTimeStampNs::get_field(const TExprNode& node) const { + int64_t value = 0; + const StringRef string_value(node.date_literal.value.data(), node.date_literal.value.size()); + const auto status = parse_timestamp_ns(string_value, &value); + if (!status.ok()) { + throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT, + "Invalid value: {} for type TimeStampNs: {}", + node.date_literal.value, status.to_string()); + } + return Field::create_field(TimeStampNsValue(value)); +} + +bool DataTypeTimeStampNs::equals(const IDataType& rhs) const { + return typeid(rhs) == typeid(*this); +} + +void DataTypeTimeStampNs::to_pb_column_meta(PColumnMeta* col_meta) const { + IDataType::to_pb_column_meta(col_meta); +} + +FieldWithDataType DataTypeTimeStampNs::get_field_with_data_type(const IColumn& column, + size_t row_num) const { + const auto& column_data = + assert_cast(column); + Field field; + column_data.get(row_num, field); + return FieldWithDataType {.field = std::move(field), + .base_scalar_type_id = get_primitive_type(), + .precision = -1, + .scale = static_cast(get_scale())}; +} + +} // namespace doris diff --git a/be/src/core/data_type/data_type_timestamp_ns.h b/be/src/core/data_type/data_type_timestamp_ns.h new file mode 100644 index 00000000000000..38c23a60d3ee09 --- /dev/null +++ b/be/src/core/data_type/data_type_timestamp_ns.h @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "core/data_type/data_type_number_base.h" +#include "core/data_type_serde/data_type_timestamp_ns_serde.h" + +namespace doris { + +class DataTypeTimeStampNs final : public DataTypeNumberBase { +public: + static constexpr PrimitiveType PType = TYPE_TIMESTAMP_NS; + static constexpr bool is_parametric = false; + static constexpr UInt32 SCALE = 9; + + PrimitiveType get_primitive_type() const override { return TYPE_TIMESTAMP_NS; } + doris::FieldType get_storage_field_type() const override { + return doris::FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS; + } + const std::string get_family_name() const override { return "TimeStampNs"; } + std::string do_get_name() const override { return "TimeStampNs"; } + + bool equals(const IDataType& rhs) const override; + bool equals_ignore_precision(const IDataType& rhs) const override { + return rhs.get_primitive_type() == TYPE_TIMESTAMP_NS; + } + + using SerDeType = DataTypeTimeStampNsSerDe; + DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override { + return std::make_shared(nesting_level); + } + + Field get_field(const TExprNode& node) const override; + UInt32 get_scale() const override { return SCALE; } + void to_pb_column_meta(PColumnMeta* col_meta) const override; + FieldWithDataType get_field_with_data_type(const IColumn& column, + size_t row_num) const override; +}; + +} // namespace doris diff --git a/be/src/core/data_type/define_primitive_type.h b/be/src/core/data_type/define_primitive_type.h index 852b68a96b5358..6606242cf1d257 100644 --- a/be/src/core/data_type/define_primitive_type.h +++ b/be/src/core/data_type/define_primitive_type.h @@ -73,7 +73,8 @@ enum PrimitiveType : PrimitiveNative { TYPE_UINT64, /* 39, used as offset */ TYPE_FIXED_LENGTH_OBJECT, /* 40, represent fixed-length object on BE */ TYPE_VARBINARY, /* 41, varbinary */ - TYPE_TIMESTAMPTZ /* 42, timestamptz */ + TYPE_TIMESTAMPTZ, /* 42, timestamptz */ + TYPE_TIMESTAMP_NS /* 43, signed Int64 epoch nanoseconds */ }; } // namespace doris diff --git a/be/src/core/data_type/primitive_type.cpp b/be/src/core/data_type/primitive_type.cpp index 746fc655323a22..208e9e9f411caa 100644 --- a/be/src/core/data_type/primitive_type.cpp +++ b/be/src/core/data_type/primitive_type.cpp @@ -69,6 +69,9 @@ PrimitiveType thrift_to_type(TPrimitiveType::type ttype) { case TPrimitiveType::DATETIMEV2: return TYPE_DATETIMEV2; + case TPrimitiveType::TIMESTAMP_NS: + return TYPE_TIMESTAMP_NS; + case TPrimitiveType::TIMEV2: return TYPE_TIMEV2; @@ -191,6 +194,9 @@ TPrimitiveType::type to_thrift(PrimitiveType ptype) { case TYPE_DATETIMEV2: return TPrimitiveType::DATETIMEV2; + case TYPE_TIMESTAMP_NS: + return TPrimitiveType::TIMESTAMP_NS; + case TYPE_TIMEV2: return TPrimitiveType::TIMEV2; @@ -305,6 +311,9 @@ std::string type_to_string(PrimitiveType t) { case TYPE_DATETIMEV2: return "DATETIMEV2"; + case TYPE_TIMESTAMP_NS: + return "TIMESTAMP_NS"; + case TYPE_TIMEV2: return "TIMEV2"; diff --git a/be/src/core/data_type/primitive_type.h b/be/src/core/data_type/primitive_type.h index cd4c55b19b121f..1d3ae0bdec88bc 100644 --- a/be/src/core/data_type/primitive_type.h +++ b/be/src/core/data_type/primitive_type.h @@ -31,6 +31,7 @@ #include "core/string_view.h" #include "core/types.h" #include "core/uint24.h" +#include "core/value/timestamp_ns_value.h" #include "core/value/timestamptz_value.h" #include "core/value/vdatetime_value.h" #include "exec/common/template_helpers.hpp" @@ -75,6 +76,7 @@ class DataTypeTimeV2; class DataTypeDateTime; class DataTypeDate; class DataTypeDateTimeV2; +class DataTypeTimeStampNs; class DataTypeDateV2; class DataTypeTimeStampTz; template @@ -112,6 +114,7 @@ using ColumnDateTime = ColumnVector; using ColumnDateV2 = ColumnVector; using ColumnTimeStampTz = ColumnVector; using ColumnDateTimeV2 = ColumnVector; +using ColumnTimeStampNs = ColumnVector; using ColumnFloat32 = ColumnVector; using ColumnFloat64 = ColumnVector; using ColumnIPv4 = ColumnVector; @@ -140,6 +143,7 @@ constexpr bool is_enumeration_type(PrimitiveType type) { case TYPE_STRING: case TYPE_DATETIME: case TYPE_DATETIMEV2: + case TYPE_TIMESTAMP_NS: case TYPE_TIMESTAMPTZ: case TYPE_TIMEV2: case TYPE_DECIMALV2: @@ -175,7 +179,11 @@ constexpr bool is_enumeration_type(PrimitiveType type) { constexpr bool is_date_type(PrimitiveType type) { return type == TYPE_DATETIME || type == TYPE_DATE || type == TYPE_DATETIMEV2 || - type == TYPE_DATEV2; + type == TYPE_TIMESTAMP_NS || type == TYPE_DATEV2; +} + +constexpr bool is_timestamp_ns_type(PrimitiveType type) { + return type == TYPE_TIMESTAMP_NS; } constexpr bool is_time_type(PrimitiveType type) { @@ -194,6 +202,10 @@ constexpr bool is_date_v2_or_datetime_v2(PrimitiveType type) { return type == TYPE_DATETIMEV2 || type == TYPE_DATEV2; } +constexpr bool is_datetime_v2(PrimitiveType type) { + return type == TYPE_DATETIMEV2; +} + constexpr bool is_ip(PrimitiveType type) { return type == TYPE_IPV4 || type == TYPE_IPV6; } @@ -366,6 +378,13 @@ struct PrimitiveTypeTraits { using ColumnType = ColumnDateTimeV2; }; template <> +struct PrimitiveTypeTraits { + using CppType = TimeStampNsValue; + using StorageFieldType = int64_t; + using DataType = DataTypeTimeStampNs; + using ColumnType = ColumnTimeStampNs; +}; +template <> struct PrimitiveTypeTraits { using CppType = DateV2Value; using StorageFieldType = uint32_t; diff --git a/be/src/core/data_type/storage_field_type.cpp b/be/src/core/data_type/storage_field_type.cpp index 269015401dcab7..ec10e25ee68e8a 100644 --- a/be/src/core/data_type/storage_field_type.cpp +++ b/be/src/core/data_type/storage_field_type.cpp @@ -73,6 +73,8 @@ FieldType primitive_type_to_storage_field_type(PrimitiveType type) { return FieldType::OLAP_FIELD_TYPE_DATEV2; case PrimitiveType::TYPE_DATETIMEV2: return FieldType::OLAP_FIELD_TYPE_DATETIMEV2; + case PrimitiveType::TYPE_TIMESTAMP_NS: + return FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS; case PrimitiveType::TYPE_TIMEV2: return FieldType::OLAP_FIELD_TYPE_TIMEV2; case PrimitiveType::TYPE_DECIMAL32: @@ -167,6 +169,8 @@ PrimitiveType storage_field_type_to_primitive_type(FieldType type) { return PrimitiveType::TYPE_DATEV2; case FieldType::OLAP_FIELD_TYPE_DATETIMEV2: return PrimitiveType::TYPE_DATETIMEV2; + case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS: + return PrimitiveType::TYPE_TIMESTAMP_NS; case FieldType::OLAP_FIELD_TYPE_TIMEV2: return PrimitiveType::TYPE_TIMEV2; case FieldType::OLAP_FIELD_TYPE_DECIMAL32: diff --git a/be/src/core/data_type_serde/data_type_number_serde.cpp b/be/src/core/data_type_serde/data_type_number_serde.cpp index a1a4a3df65df19..e35ffc7d495b3b 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.cpp +++ b/be/src/core/data_type_serde/data_type_number_serde.cpp @@ -737,6 +737,13 @@ Status DataTypeNumberSerDe::write_column_to_arrow(const IColumn& column, cons return Status::OK(); } +template <> +Status DataTypeNumberSerDe::write_column_to_arrow( + const IColumn& column, const NullMap* null_map, arrow::ArrayBuilder* array_builder, + int64_t start, int64_t end, const cctz::time_zone& ctz) const { + return Status::NotSupported("DataTypeNumberSerDe::write_column_to_arrow"); +} + template Status DataTypeNumberSerDe::read_column_from_decoded_values( IColumn& column, const DecodedColumnView& view) const { @@ -1117,8 +1124,8 @@ template constexpr bool can_write_to_jsonb_from_number() { return T == TYPE_BOOLEAN || T == TYPE_TINYINT || T == TYPE_SMALLINT || T == TYPE_INT || T == TYPE_BIGINT || T == TYPE_LARGEINT || T == TYPE_FLOAT || T == TYPE_DOUBLE || - T == TYPE_DATEV2 || T == TYPE_DATETIMEV2 || T == TYPE_TIMESTAMPTZ || T == TYPE_IPV4 || - T == TYPE_IPV6 || T == TYPE_TIMEV2; + T == TYPE_DATEV2 || T == TYPE_DATETIMEV2 || T == TYPE_TIMESTAMP_NS || + T == TYPE_TIMESTAMPTZ || T == TYPE_IPV4 || T == TYPE_IPV6 || T == TYPE_TIMEV2; } template @@ -1156,6 +1163,8 @@ bool write_to_jsonb_from_number(auto& data, JsonbWriter& writer, int scale) { return jsonb_writer_string(writer, CastToString::from_datev2(data)); } else if constexpr (T == TYPE_DATETIMEV2) { return jsonb_writer_string(writer, CastToString::from_datetimev2(data, scale)); + } else if constexpr (T == TYPE_TIMESTAMP_NS) { + return jsonb_writer_string(writer, TimeStampNsValue(data).to_string(scale)); } else if constexpr (T == TYPE_TIMESTAMPTZ) { return jsonb_writer_string(writer, CastToString::from_timestamptz(data, scale)); } else if constexpr (T == TYPE_IPV4) { @@ -1445,6 +1454,8 @@ void DataTypeNumberSerDe::read_one_cell_from_jsonb(IColumn& column, col.insert_value(binary_cast(static_cast(read_int()))); } else if constexpr (T == TYPE_BIGINT) { col.insert_value(static_cast(read_int())); + } else if constexpr (T == TYPE_TIMESTAMP_NS) { + col.insert_value(TimeStampNsValue(static_cast(read_int()))); } else if constexpr (T == TYPE_LARGEINT) { col.insert_value(static_cast<__int128_t>(read_int())); } else if constexpr (T == TYPE_FLOAT) { @@ -1489,7 +1500,7 @@ void DataTypeNumberSerDe::write_one_cell_to_jsonb(const IColumn& column, int32_t val = *reinterpret_cast(data_ref.data); result.writeInt32(val); } else if constexpr (T == TYPE_BIGINT || T == TYPE_DATE || T == TYPE_DATETIME || - T == TYPE_DATETIMEV2 || T == TYPE_TIMESTAMPTZ) { + T == TYPE_DATETIMEV2 || T == TYPE_TIMESTAMP_NS || T == TYPE_TIMESTAMPTZ) { int64_t val = *reinterpret_cast(data_ref.data); if (options.enable_row_store_compact_jsonb) { result.writeInt(val); @@ -1748,6 +1759,10 @@ const uint8_t* DataTypeNumberSerDe::deserialize_binary_to_column(const uint8_ col.insert_value(binary_cast>( unaligned_load(data))); data += sizeof(UInt64); + } else if constexpr (T == TYPE_TIMESTAMP_NS) { + data += sizeof(uint8_t); + col.insert_value(TimeStampNsValue(unaligned_load(data))); + data += sizeof(Int64); } else { throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, "deserialize_binary_to_column with type '{}'", type_to_string(T)); @@ -1810,6 +1825,14 @@ const uint8_t* DataTypeNumberSerDe::deserialize_binary_to_field(const uint8_t info.scale = static_cast(scale); field = Field::create_field(*(typename PrimitiveTypeTraits::CppType*)&v); data += sizeof(UInt64); + } else if constexpr (T == TYPE_TIMESTAMP_NS) { + const uint8_t scale = *data; + data += sizeof(uint8_t); + info.precision = -1; + info.scale = static_cast(scale); + field = Field::create_field( + TimeStampNsValue(unaligned_load(data))); + data += sizeof(Int64); } else { throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, "deserialize_binary_to_column with type '{}'", type_to_string(T)); @@ -1828,6 +1851,9 @@ void value_to_string(const typename PrimitiveTypeTraits::CppType value, Buffe CastToString::push_datev2(value, bw); } else if constexpr (T == TYPE_DATETIMEV2) { CastToString::push_datetimev2(value, scale, bw); + } else if constexpr (T == TYPE_TIMESTAMP_NS) { + const auto string_value = TimeStampNsValue(value).to_string(scale); + bw.write(string_value.data(), string_value.size()); } else if constexpr (T == TYPE_TIMESTAMPTZ) { CastToString::push_timestamptz(value, scale, bw, options); } else if constexpr (T == TYPE_TIMEV2) { @@ -1954,6 +1980,7 @@ template class DataTypeNumberSerDe; template class DataTypeNumberSerDe; template class DataTypeNumberSerDe; template class DataTypeNumberSerDe; +template class DataTypeNumberSerDe; template class DataTypeNumberSerDe; template class DataTypeNumberSerDe; template class DataTypeNumberSerDe; diff --git a/be/src/core/data_type_serde/data_type_number_serde.h b/be/src/core/data_type_serde/data_type_number_serde.h index 9c0de8cb0c0e8a..c69540ff462df4 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.h +++ b/be/src/core/data_type_serde/data_type_number_serde.h @@ -247,6 +247,12 @@ Status DataTypeNumberSerDe::read_column_from_pb(IColumn& column, const PValue for (int i = 0; i < arg.int64_value_size(); ++i) { data[old_column_size + i] = arg.int64_value(i); } + } else if constexpr (T == TYPE_TIMESTAMP_NS) { + column.resize(old_column_size + arg.int64_value_size()); + auto& data = reinterpret_cast(column).get_data(); + for (int i = 0; i < arg.int64_value_size(); ++i) { + data[old_column_size + i] = TimeStampNsValue(arg.int64_value(i)); + } } else if constexpr (T == TYPE_FLOAT) { column.resize(old_column_size + arg.float_value_size()); auto& data = reinterpret_cast(column).get_data(); @@ -332,6 +338,12 @@ Status DataTypeNumberSerDe::write_column_to_pb(const IColumn& column, PValues auto* values = result.mutable_int64_value(); values->Reserve(row_count); values->Add(data.begin() + start, data.begin() + end); + } else if constexpr (T == TYPE_TIMESTAMP_NS) { + ptype->set_id(PGenericType::INT64); + auto* values = result.mutable_int64_value(); + values->Reserve(row_count); + values->Add(reinterpret_cast(data.begin()) + start, + reinterpret_cast(data.begin()) + end); } else if constexpr (T == TYPE_FLOAT) { ptype->set_id(PGenericType::FLOAT); auto* values = result.mutable_float_value(); diff --git a/be/src/core/data_type_serde/data_type_serde.cpp b/be/src/core/data_type_serde/data_type_serde.cpp index ca5f2c206ada16..b8dc7ccb09e5da 100644 --- a/be/src/core/data_type_serde/data_type_serde.cpp +++ b/be/src/core/data_type_serde/data_type_serde.cpp @@ -288,6 +288,7 @@ const uint8_t* DataTypeSerDe::deserialize_binary_to_column(const uint8_t* data, HANDLE_T_NUM_SERDE(OLAP_FIELD_TYPE_IPV6, TYPE_IPV6) HANDLE_T_NUM_SERDE(OLAP_FIELD_TYPE_DATEV2, TYPE_DATEV2) HANDLE_T_NUM_SERDE(OLAP_FIELD_TYPE_DATETIMEV2, TYPE_DATETIMEV2) + HANDLE_T_NUM_SERDE(OLAP_FIELD_TYPE_TIMESTAMP_NS, TYPE_TIMESTAMP_NS) HANDLE_T_DEC_SERDE(OLAP_FIELD_TYPE_DECIMAL32, TYPE_DECIMAL32) HANDLE_T_DEC_SERDE(OLAP_FIELD_TYPE_DECIMAL64, TYPE_DECIMAL64) HANDLE_T_DEC_SERDE(OLAP_FIELD_TYPE_DECIMAL128I, TYPE_DECIMAL128I) @@ -350,6 +351,7 @@ const uint8_t* DataTypeSerDe::deserialize_binary_to_field(const uint8_t* data, F HANDLE_T_NUM_SERDE(OLAP_FIELD_TYPE_IPV6, TYPE_IPV6) HANDLE_T_NUM_SERDE(OLAP_FIELD_TYPE_DATEV2, TYPE_DATEV2) HANDLE_T_NUM_SERDE(OLAP_FIELD_TYPE_DATETIMEV2, TYPE_DATETIMEV2) + HANDLE_T_NUM_SERDE(OLAP_FIELD_TYPE_TIMESTAMP_NS, TYPE_TIMESTAMP_NS) HANDLE_T_DEC_SERDE(OLAP_FIELD_TYPE_DECIMAL32, TYPE_DECIMAL32) HANDLE_T_DEC_SERDE(OLAP_FIELD_TYPE_DECIMAL64, TYPE_DECIMAL64) HANDLE_T_DEC_SERDE(OLAP_FIELD_TYPE_DECIMAL128I, TYPE_DECIMAL128I) diff --git a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp new file mode 100644 index 00000000000000..b1bdb90323a704 --- /dev/null +++ b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp @@ -0,0 +1,339 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/data_type_serde/data_type_timestamp_ns_serde.h" + +#include + +#include +#include +#include +#include + +#include "common/exception.h" +#include "core/assert_cast.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/data_type_serde/decoded_column_view.h" +#include "core/value/vdatetime_value.h" +#include "exprs/function/cast/cast_to_datetimev2_impl.hpp" +#include "util/mysql_row_buffer.h" +#include "util/unaligned.h" + +namespace doris { +namespace { + +constexpr int64_t NANOS_PER_MILLISECOND = 1000000; +constexpr int64_t NANOS_PER_MICROSECOND = 1000; +constexpr size_t NANOSECOND_SCALE = 9; + +bool checked_scale_to_nanos(int64_t value, int64_t multiplier, int64_t* result) { + return !__builtin_mul_overflow(value, multiplier, result); +} + +Status utc_epoch_nanos_to_local_epoch_nanos(int64_t source, const cctz::time_zone& timezone, + int64_t* result) { + const TimeStampNsValue source_value(source); + DateV2Value local_value; + local_value.from_unixtime(source_value.epoch_seconds(), timezone); + local_value.set_microsecond(source_value.microsecond()); + TimeStampNsValue target_value; + if (!target_value.from_datetime(local_value, source_value.nanosecond_remainder())) { + return Status::DataQualityError("Timestamp {} is outside TIMESTAMP_NS range", source); + } + *result = target_value.epoch_nanos(); + return Status::OK(); +} + +} // namespace + +Status parse_timestamp_ns(StringRef str, int64_t* epoch_nanos, + const cctz::time_zone* local_time_zone) { + std::string input(str.data, str.size); + const size_t dot = input.rfind('.'); + size_t fraction_begin = std::string::npos; + size_t fraction_end = std::string::npos; + if (dot != std::string::npos && dot + 1 < input.size() && + std::isdigit(static_cast(input[dot + 1]))) { + fraction_begin = dot + 1; + fraction_end = fraction_begin; + while (fraction_end < input.size() && + std::isdigit(static_cast(input[fraction_end]))) { + ++fraction_end; + } + } + + uint32_t nanos = 0; + size_t fraction_length = 0; + if (fraction_begin != std::string::npos) { + fraction_length = fraction_end - fraction_begin; + const size_t copied_digits = std::min(fraction_length, 9); + for (size_t i = 0; i < copied_digits; ++i) { + nanos = nanos * 10 + static_cast(input[fraction_begin + i] - '0'); + } + for (size_t i = copied_digits; i < 9; ++i) { + nanos *= 10; + } + } + + if (fraction_length > NANOSECOND_SCALE && input[fraction_begin + NANOSECOND_SCALE] >= '5') { + ++nanos; + } + + // Keep the fractional token in place so that the legacy parser validates its position and all + // trailing syntax. Zeroing the digits prevents its scale-0 rounding from changing the second; + // the nanosecond rounding above remains the only source of fractional rounding. + if (fraction_begin != std::string::npos) { + std::fill(input.begin() + fraction_begin, input.begin() + fraction_end, '0'); + } + const StringRef input_ref(input.data(), input.size()); + DateV2Value datetime; + CastParameters params {.status = Status::OK(), .is_strict = true}; + CastToDatetimeV2::from_string_strict_mode( + input_ref, datetime, local_time_zone, 0, params); + if (!params.status.ok()) { + if (dot != std::string::npos) { + return Status::InvalidArgument("Invalid TIMESTAMP_NS value '{}'", + std::string(str.data, str.size)); + } + return params.status; + } + + if (nanos == TimeStampNsValue::NANOS_PER_SECOND) { + if (!datetime.date_add_interval( + TimeInterval {TimeUnit::SECOND, 1, false})) { + return Status::InvalidArgument("TIMESTAMP_NS value overflows while rounding '{}'", + std::string(str.data, str.size)); + } + nanos = 0; + } + datetime.set_microsecond(nanos / NANOS_PER_MICROSECOND); + TimeStampNsValue value; + if (!value.from_datetime(datetime, static_cast(nanos % NANOS_PER_MICROSECOND))) { + return Status::InvalidArgument( + "TIMESTAMP_NS value '{}' is outside [{}, {}]", std::string(str.data, str.size), + TimeStampNsValue(std::numeric_limits::min()).to_string(), + TimeStampNsValue(std::numeric_limits::max()).to_string()); + } + *epoch_nanos = value.epoch_nanos(); + return Status::OK(); +} + +Status DataTypeTimeStampNsSerDe::from_string_batch(const ColumnString& strings, + ColumnNullable& result, + const FormatOptions& options) const { + auto& data = assert_cast(result.get_nested_column()).get_data(); + auto& null_map = result.get_null_map_column().get_data(); + result.resize(strings.size()); + for (size_t i = 0; i < strings.size(); ++i) { + int64_t value = 0; + const auto status = parse_timestamp_ns(strings.get_data_at(i), &value, options.timezone); + null_map[i] = !status.ok(); + data[i] = TimeStampNsValue(value); + } + return Status::OK(); +} + +Status DataTypeTimeStampNsSerDe::from_string_strict_mode_batch( + const ColumnString& strings, IColumn& result, const FormatOptions& options, + const NullMap::value_type* null_map) const { + auto& data = assert_cast(result).get_data(); + result.resize(strings.size()); + for (size_t i = 0; i < strings.size(); ++i) { + if (null_map != nullptr && null_map[i]) { + continue; + } + int64_t value = 0; + RETURN_IF_ERROR(parse_timestamp_ns(strings.get_data_at(i), &value, options.timezone)); + data[i] = TimeStampNsValue(value); + } + return Status::OK(); +} + +Status DataTypeTimeStampNsSerDe::from_string(StringRef& str, IColumn& column, + const FormatOptions& options) const { + int64_t value = 0; + RETURN_IF_ERROR(parse_timestamp_ns(str, &value, options.timezone)); + assert_cast(column).insert_value(TimeStampNsValue(value)); + return Status::OK(); +} + +Status DataTypeTimeStampNsSerDe::from_string_strict_mode(StringRef& str, IColumn& column, + const FormatOptions& options) const { + return from_string(str, column, options); +} + +Status DataTypeTimeStampNsSerDe::serialize_column_to_json(const IColumn& column, int64_t start_idx, + int64_t end_idx, BufferWritable& bw, + FormatOptions& options) const { + SERIALIZE_COLUMN_TO_JSON(); +} + +Status DataTypeTimeStampNsSerDe::serialize_one_cell_to_json(const IColumn& column, int64_t row_num, + BufferWritable& bw, + FormatOptions& options) const { + auto [column_ptr, index] = check_column_const_set_readability(column, row_num); + if (_nesting_level > 1) { + bw.write('"'); + } + const auto value = + assert_cast(*column_ptr) + .get_element(index); + const std::string result = value.to_string(9); + bw.write(result.data(), result.size()); + if (_nesting_level > 1) { + bw.write('"'); + } + return Status::OK(); +} + +Status DataTypeTimeStampNsSerDe::deserialize_column_from_json_vector( + IColumn& column, std::vector& slices, uint64_t* num_deserialized, + const FormatOptions& options) const { + DESERIALIZE_COLUMN_FROM_JSON_VECTOR(); + return Status::OK(); +} + +Status DataTypeTimeStampNsSerDe::deserialize_one_cell_from_json( + IColumn& column, Slice& slice, const FormatOptions& options) const { + if (_nesting_level > 1) { + slice.trim_quote(); + } + StringRef str(slice.data, slice.size); + return from_string(str, column, options); +} + +Status DataTypeTimeStampNsSerDe::deserialize_column_from_jsonb(IColumn& column, + const JsonbValue* jsonb_value, + CastParameters& cast_params) const { + DORIS_CHECK(jsonb_value->isString()); + return parse_column_from_jsonb_string(column, jsonb_value, cast_params); +} + +Status DataTypeTimeStampNsSerDe::deserialize_column_from_jsonb_vector( + ColumnNullable& column_to, const ColumnString& column_from, + CastParameters& cast_params) const { + return DataTypeSerDe::deserialize_column_from_jsonb_vector(column_to, column_from, cast_params); +} + +Status DataTypeTimeStampNsSerDe::write_column_to_arrow(const IColumn& column, + const NullMap* null_map, + arrow::ArrayBuilder* array_builder, + int64_t start, int64_t end, + const cctz::time_zone& ctz) const { + return Status::NotSupported("DataTypeTimeStampNsSerDe::write_column_to_arrow"); +} + +Status DataTypeTimeStampNsSerDe::read_column_from_arrow(IColumn& column, + const arrow::Array* arrow_array, + int64_t start, int64_t end, + const cctz::time_zone& ctz) const { + return Status::NotSupported("DataTypeTimeStampNsSerDe::read_column_from_arrow"); +} + +Status DataTypeTimeStampNsSerDe::read_column_from_decoded_values( + IColumn& column, const DecodedColumnView& view) const { + if (view.value_kind != DecodedValueKind::INT64) { + return decoded_column_view_handle_conversion_failure( + column, view, + Status::NotSupported("TIMESTAMP_NS decoded reader expects INT64 source")); + } + auto& data = assert_cast(column).get_data(); + const auto* values = reinterpret_cast(view.values); + static const auto utc = cctz::utc_time_zone(); + const auto& timezone = view.timezone == nullptr ? utc : *view.timezone; + for (int64_t row = 0; row < view.row_count; ++row) { + if (decoded_column_view_row_is_null(view, row)) { + data.push_back(TimeStampNsValue(0)); + continue; + } + int64_t nanos = 0; + int64_t multiplier = 1; + switch (view.time_unit) { + case DecodedTimeUnit::MILLIS: + multiplier = NANOS_PER_MILLISECOND; + break; + case DecodedTimeUnit::MICROS: + multiplier = NANOS_PER_MICROSECOND; + break; + case DecodedTimeUnit::NANOS: + break; + case DecodedTimeUnit::UNKNOWN: + return decoded_column_view_handle_conversion_failure( + column, view, + Status::NotSupported("TIMESTAMP_NS decoded reader requires a time unit")); + } + if (!checked_scale_to_nanos(values[row], multiplier, &nanos)) { + return decoded_column_view_handle_conversion_failure( + column, view, + Status::DataQualityError("Timestamp {} overflows nanoseconds", values[row])); + } + if (view.timestamp_is_adjusted_to_utc) { + RETURN_IF_ERROR(utc_epoch_nanos_to_local_epoch_nanos(nanos, timezone, &nanos)); + } + data.push_back(TimeStampNsValue(nanos)); + } + return Status::OK(); +} + +Status DataTypeTimeStampNsSerDe::write_column_to_mysql_binary(const IColumn& column, + MysqlRowBinaryBuffer& row_buffer, + int64_t row_idx, bool col_const, + const FormatOptions& options) const { + const auto& data = assert_cast(column).get_data(); + const auto index = index_check_const(row_idx, col_const); + const auto value = TimeStampNsValue(data[index]).to_string(9); + if (row_buffer.push_string(value.data(), value.size()) != 0) { + return Status::InternalError("pack MySQL TIMESTAMP_NS buffer failed"); + } + return Status::OK(); +} + +Status DataTypeTimeStampNsSerDe::write_column_to_orc(const std::string& timezone, + const IColumn& column, const NullMap* null_map, + orc::ColumnVectorBatch* orc_col_batch, + int64_t start, int64_t end, Arena& arena, + const FormatOptions& options) const { + return Status::NotSupported("DataTypeTimeStampNsSerDe::write_column_to_orc"); +} + +void DataTypeTimeStampNsSerDe::write_one_cell_to_binary(const IColumn& src_column, + ColumnString::Chars& chars, + int64_t row_num) const { + const auto type = static_cast(FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS); + constexpr auto scale = static_cast(9); + const auto value = assert_cast(src_column).get_element(row_num); + const size_t old_size = chars.size(); + chars.resize(old_size + sizeof(type) + sizeof(scale) + sizeof(value)); + memcpy(chars.data() + old_size, &type, sizeof(type)); + memcpy(chars.data() + old_size + sizeof(type), &scale, sizeof(scale)); + memcpy(chars.data() + old_size + sizeof(type) + sizeof(scale), &value, sizeof(value)); +} + +std::string DataTypeTimeStampNsSerDe::to_olap_string(const Field& field) const { + return field.get().to_string(9); +} + +Status DataTypeTimeStampNsSerDe::from_olap_string(const std::string& str, Field& field, + const FormatOptions& options) const { + int64_t value = 0; + RETURN_IF_ERROR(parse_timestamp_ns(StringRef(str.data(), str.size()), &value)); + field = Field::create_field(TimeStampNsValue(value)); + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.h b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.h new file mode 100644 index 00000000000000..5489b34eeb8c11 --- /dev/null +++ b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.h @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "core/data_type_serde/data_type_number_serde.h" + +namespace doris { + +Status parse_timestamp_ns(StringRef str, int64_t* epoch_nanos, + const cctz::time_zone* local_time_zone = nullptr); + +class DataTypeTimeStampNsSerDe final + : public DataTypeNumberSerDe { +public: + explicit DataTypeTimeStampNsSerDe(int nesting_level = 1) + : DataTypeNumberSerDe(nesting_level) {} + + Status from_string_batch(const ColumnString& str, ColumnNullable& column, + const FormatOptions& options) const override; + Status from_string_strict_mode_batch( + const ColumnString& str, IColumn& column, const FormatOptions& options, + const NullMap::value_type* null_map = nullptr) const override; + Status from_string(StringRef& str, IColumn& column, + const FormatOptions& options) const override; + Status from_string_strict_mode(StringRef& str, IColumn& column, + const FormatOptions& options) const override; + + Status serialize_one_cell_to_json(const IColumn& column, int64_t row_num, BufferWritable& bw, + FormatOptions& options) const override; + Status serialize_column_to_json(const IColumn& column, int64_t start_idx, int64_t end_idx, + BufferWritable& bw, FormatOptions& options) const override; + Status deserialize_one_cell_from_json(IColumn& column, Slice& slice, + const FormatOptions& options) const override; + Status deserialize_column_from_json_vector(IColumn& column, std::vector& slices, + uint64_t* num_deserialized, + const FormatOptions& options) const override; + Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value, + CastParameters& cast_params) const override; + Status deserialize_column_from_jsonb_vector(ColumnNullable& column_to, + const ColumnString& column_from, + CastParameters& cast_params) const override; + + Status write_column_to_arrow(const IColumn& column, const NullMap* null_map, + arrow::ArrayBuilder* array_builder, int64_t start, int64_t end, + const cctz::time_zone& ctz) const override; + Status read_column_from_arrow(IColumn& column, const arrow::Array* arrow_array, int64_t start, + int64_t end, const cctz::time_zone& ctz) const override; + Status read_column_from_decoded_values(IColumn& column, + const DecodedColumnView& view) const override; + Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& row_buffer, + int64_t row_idx, bool col_const, + const FormatOptions& options) const override; + Status write_column_to_orc(const std::string& timezone, const IColumn& column, + const NullMap* null_map, orc::ColumnVectorBatch* orc_col_batch, + int64_t start, int64_t end, Arena& arena, + const FormatOptions& options) const override; + + void write_one_cell_to_binary(const IColumn& src_column, ColumnString::Chars& chars, + int64_t row_num) const override; + int get_scale() const override { return 9; } + std::string to_olap_string(const Field& field) const override; + +protected: + Status from_olap_string(const std::string& str, Field& field, + const FormatOptions& options) const override; +}; + +} // namespace doris diff --git a/be/src/core/field.cpp b/be/src/core/field.cpp index f63aaf8227a7c6..1466a0d44d2d8a 100644 --- a/be/src/core/field.cpp +++ b/be/src/core/field.cpp @@ -124,6 +124,9 @@ void Field::create(Field&& field) { case PrimitiveType::TYPE_DATETIMEV2: create_concrete(std::move(field.template get())); return; + case PrimitiveType::TYPE_TIMESTAMP_NS: + create_concrete(std::move(field.template get())); + return; case PrimitiveType::TYPE_DATEV2: create_concrete(std::move(field.template get())); return; @@ -253,6 +256,9 @@ void Field::create(const Field& field) { case PrimitiveType::TYPE_DATETIMEV2: create_concrete(field.template get()); return; + case PrimitiveType::TYPE_TIMESTAMP_NS: + create_concrete(field.template get()); + return; case PrimitiveType::TYPE_DATEV2: create_concrete(field.template get()); return; @@ -414,6 +420,9 @@ void Field::assign(Field&& field) { case PrimitiveType::TYPE_DATETIMEV2: assign_concrete(std::move(field.template get())); return; + case PrimitiveType::TYPE_TIMESTAMP_NS: + assign_concrete(std::move(field.template get())); + return; case PrimitiveType::TYPE_DATETIME: assign_concrete(std::move(field.template get())); return; @@ -523,6 +532,9 @@ void Field::assign(const Field& field) { case PrimitiveType::TYPE_DATETIMEV2: assign_concrete(field.template get()); return; + case PrimitiveType::TYPE_TIMESTAMP_NS: + assign_concrete(field.template get()); + return; case PrimitiveType::TYPE_DATETIME: assign_concrete(field.template get()); return; @@ -708,6 +720,9 @@ std::strong_ordering Field::operator<=>(const Field& rhs) const { case PrimitiveType::TYPE_DATETIMEV2: return get().to_date_int_val() <=> rhs.get().to_date_int_val(); + case PrimitiveType::TYPE_TIMESTAMP_NS: + return get() <=> + rhs.get(); case PrimitiveType::TYPE_DATEV2: return get().to_date_int_val() <=> rhs.get().to_date_int_val(); @@ -826,6 +841,7 @@ std::string_view Field::as_string_view() const { // MATCH_PRIMITIVE_TYPE(TYPE_QUANTILE_STATE); MATCH_PRIMITIVE_TYPE(TYPE_DATEV2); MATCH_PRIMITIVE_TYPE(TYPE_DATETIMEV2); + MATCH_PRIMITIVE_TYPE(TYPE_TIMESTAMP_NS); MATCH_PRIMITIVE_TYPE(TYPE_TIMEV2); MATCH_PRIMITIVE_TYPE(TYPE_DECIMAL32); MATCH_PRIMITIVE_TYPE(TYPE_DECIMAL64); @@ -883,6 +899,8 @@ std::string Field::to_debug_string(int scale) const { return CastToString::from_datev2(get()); case PrimitiveType::TYPE_DATETIMEV2: return CastToString::from_datetimev2(get(), scale); + case PrimitiveType::TYPE_TIMESTAMP_NS: + return get().to_string(scale); case PrimitiveType::TYPE_TIMESTAMPTZ: return CastToString::from_timestamptz(get(), scale); case PrimitiveType::TYPE_DECIMALV2: @@ -927,6 +945,8 @@ std::string Field::to_debug_string(int scale) const { typename PrimitiveTypeTraits::CppType && rhs); \ template void Field::FUNC_NAME( \ typename PrimitiveTypeTraits::CppType && rhs); \ + template void Field::FUNC_NAME( \ + typename PrimitiveTypeTraits::CppType && rhs); \ template void Field::FUNC_NAME( \ typename PrimitiveTypeTraits::CppType && rhs); \ template void Field::FUNC_NAME( \ @@ -973,6 +993,8 @@ std::string Field::to_debug_string(int scale) const { const typename PrimitiveTypeTraits::CppType& rhs); \ template void Field::FUNC_NAME( \ const typename PrimitiveTypeTraits::CppType& rhs); \ + template void Field::FUNC_NAME( \ + const typename PrimitiveTypeTraits::CppType& rhs); \ template void Field::FUNC_NAME( \ const typename PrimitiveTypeTraits::CppType& rhs); \ template void Field::FUNC_NAME( \ @@ -1070,6 +1092,7 @@ DECLARE_FUNCTION(TYPE_DATE) DECLARE_FUNCTION(TYPE_DATETIME) DECLARE_FUNCTION(TYPE_DATEV2) DECLARE_FUNCTION(TYPE_DATETIMEV2) +DECLARE_FUNCTION(TYPE_TIMESTAMP_NS) DECLARE_FUNCTION(TYPE_TIMESTAMPTZ) DECLARE_FUNCTION(TYPE_DECIMAL32) DECLARE_FUNCTION(TYPE_DECIMAL64) diff --git a/be/src/core/type_limit.h b/be/src/core/type_limit.h index 824433a03191dc..7a286470f5a25d 100644 --- a/be/src/core/type_limit.h +++ b/be/src/core/type_limit.h @@ -20,6 +20,7 @@ #include "core/extended_types.h" #include "core/string_ref.h" #include "core/value/decimalv2_value.h" +#include "core/value/timestamp_ns_value.h" #include "core/value/timestamptz_value.h" namespace doris { @@ -118,6 +119,12 @@ struct type_limit> { } }; +template <> +struct type_limit { + static TimeStampNsValue min() { return TimeStampNsValue(std::numeric_limits::min()); } + static TimeStampNsValue max() { return TimeStampNsValue(std::numeric_limits::max()); } +}; + template <> struct type_limit { static TimestampTzValue min() { diff --git a/be/src/core/value/timestamp_ns_value.cpp b/be/src/core/value/timestamp_ns_value.cpp new file mode 100644 index 00000000000000..965e613a3d251e --- /dev/null +++ b/be/src/core/value/timestamp_ns_value.cpp @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/value/timestamp_ns_value.h" + +#include + +#include "exec/common/int_exp.h" + +namespace doris { + +DateV2Value TimeStampNsValue::to_datetime() const { + // epoch_seconds() is floor-divided, so set_microsecond() always receives the first six digits + // of a non-negative fractional second even for timestamps before the epoch. + DateV2Value value; + value.from_unixtime(epoch_seconds(), cctz::utc_time_zone()); + value.set_microsecond(microsecond()); + return value; +} + +bool TimeStampNsValue::from_datetime(const DateV2Value& value, + uint16_t nanosecond_remainder) { + DORIS_CHECK_LE(nanosecond_remainder, 999); + int64_t seconds = 0; + value.unix_timestamp(&seconds, cctz::utc_time_zone()); + // The civil adapter and remainder form one exact epoch-nanosecond value. Int128 is required: + // valid boundary dates overflow Int64 during the intermediate seconds-to-nanoseconds product. + const __int128 epoch_nanos = static_cast<__int128>(seconds) * NANOS_PER_SECOND + + static_cast<__int128>(value.microsecond()) * 1000 + + nanosecond_remainder; + if (epoch_nanos < std::numeric_limits::min() || + epoch_nanos > std::numeric_limits::max()) { + return false; + } + _epoch_nanos = static_cast(epoch_nanos); + return true; +} + +int32_t TimeStampNsValue::to_buffer(char* buffer, int scale) const { + DORIS_CHECK_GE(scale, 0); + DORIS_CHECK_LE(scale, 9); + const auto value = to_datetime(); + const int32_t base_length = value.to_buffer(buffer, 0); + if (scale == 0) { + return base_length; + } + buffer[base_length] = '.'; + uint32_t nanos = nanosecond(); + // Consume the fraction from most significant to least significant digit. Deliberately do not + // round here: type normalization happens while parsing/casting, before a value is stored. + for (int i = 0; i < scale; ++i) { + buffer[base_length + 1 + i] = + static_cast('0' + nanos / static_cast(int_exp10(8 - i))); + nanos %= static_cast(int_exp10(8 - i)); + } + return base_length + 1 + scale; +} + +} // namespace doris diff --git a/be/src/core/value/timestamp_ns_value.h b/be/src/core/value/timestamp_ns_value.h new file mode 100644 index 00000000000000..7e676fb6f3df15 --- /dev/null +++ b/be/src/core/value/timestamp_ns_value.h @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "core/value/vdatetime_value.h" + +namespace doris { + +// TIMESTAMP_NS is represented by a signed Int64 count of nanoseconds from the Unix epoch. The +// complete Int64 domain is valid, giving the exact range +// [1677-09-21 00:12:43.145224192, 2262-04-11 23:47:16.854775807]. The SQL type is timezone-naive: +// UTC below is only a deterministic bridge between the integer and civil calendar fields, and +// must not apply the session time zone. +// +// Keep this class an Int64-sized trivially-copyable value. Columns, storage encodings, hashing, +// and SIMD paths rely on being able to move its bytes exactly like an Int64. +class TimeStampNsValue { +public: + static constexpr int64_t NANOS_PER_SECOND = 1000000000; + static constexpr int64_t NANOS_PER_MILLISECOND = 1000000; + static constexpr int64_t NANOS_PER_MICROSECOND = 1000; + + constexpr TimeStampNsValue() = default; + explicit constexpr TimeStampNsValue(int64_t epoch_nanos) : _epoch_nanos(epoch_nanos) {} + + // This is the raw physical key, not the packed civil layout used by DATETIMEV2. + constexpr int64_t to_date_int_val() const { return _epoch_nanos; } + constexpr int64_t epoch_nanos() const { return _epoch_nanos; } + + // Split epoch nanoseconds using floor division so the fractional component is always + // non-negative. For example, -1ns becomes second -1 plus nanosecond 999,999,999 rather than + // second 0 plus a negative fraction. + int64_t epoch_seconds() const { + int64_t seconds = _epoch_nanos / NANOS_PER_SECOND; + if (_epoch_nanos % NANOS_PER_SECOND < 0) { + --seconds; + } + return seconds; + } + + uint32_t nanosecond() const { + int64_t nanos = _epoch_nanos % NANOS_PER_SECOND; + if (nanos < 0) { + nanos += NANOS_PER_SECOND; + } + return static_cast(nanos); + } + + uint32_t microsecond() const { return nanosecond() / 1000; } + uint16_t nanosecond_remainder() const { return static_cast(nanosecond() % 1000); } + + // DATETIMEV2 is used only as a civil-calendar adapter. It carries the first six fractional + // digits; callers must preserve nanosecond_remainder() separately when the operation should + // not discard the final three digits. + DateV2Value to_datetime() const; + bool from_datetime(const DateV2Value& value, + uint16_t nanosecond_remainder = 0); + + // sequence_match still uses the common date-value concept in BE. These two integral helpers + // are the only civil accessors required by that execution path. + int64_t daynr() const { return to_datetime().daynr(); } + int64_t time_part_to_seconds() const { return to_datetime().time_part_to_seconds(); } + + // This intentionally compares integral civil seconds and ignores fractional seconds. It is + // the contract used by sequence_match's second-based pattern conditions. + template + int64_t datetime_diff_in_seconds(const RHS& rhs) const { + return (daynr() - rhs.daynr()) * SECOND_PER_HOUR * HOUR_PER_DAY + time_part_to_seconds() - + rhs.time_part_to_seconds(); + } + + bool is_valid_date() const { return true; } + + // Formatting emits the requested leading fractional digits without rounding. The SQL type + // normally requests scale 9; smaller scales are retained for generic date-format callers. + int32_t to_buffer(char* buffer, int scale = 9) const; + char* to_string(char* buffer, int scale = 9) const { + const int32_t length = to_buffer(buffer, scale); + buffer[length] = '\0'; + return buffer + length + 1; + } + std::string to_string(int scale = 9) const { + char buffer[40]; + const int32_t length = to_buffer(buffer, scale); + return {buffer, static_cast(length)}; + } + + auto operator<=>(const TimeStampNsValue&) const = default; + + uint32_t hash(int seed) const { + return HashUtil::hash(&_epoch_nanos, sizeof(_epoch_nanos), seed); + } + +private: + int64_t _epoch_nanos = 0; +}; + +static_assert(sizeof(TimeStampNsValue) == sizeof(int64_t)); +static_assert(std::is_trivially_copyable_v); + +} // namespace doris + +template <> +struct std::hash { + size_t operator()(const doris::TimeStampNsValue& value) const { + return std::hash {}(value.epoch_nanos()); + } +}; diff --git a/be/src/core/value/vdatetime_value.cpp b/be/src/core/value/vdatetime_value.cpp index 2bceb41b809537..6dfd7062f75d78 100644 --- a/be/src/core/value/vdatetime_value.cpp +++ b/be/src/core/value/vdatetime_value.cpp @@ -47,6 +47,7 @@ #include "util/timezone_utils.h" namespace doris { + #include "common/compile_check_avoid_begin.h" static const char* s_ab_month_name[] = {"", "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", nullptr}; diff --git a/be/src/exec/common/field_visitors.h b/be/src/exec/common/field_visitors.h index 8b14b6bbd17626..25ca811dd889ae 100644 --- a/be/src/exec/common/field_visitors.h +++ b/be/src/exec/common/field_visitors.h @@ -47,6 +47,9 @@ typename std::decay_t::ResultType apply_visitor(Visitor&& visitor, F&& case PrimitiveType::TYPE_DATETIMEV2: return visitor.template apply( field.template get()); + case PrimitiveType::TYPE_TIMESTAMP_NS: + return visitor.template apply( + field.template get()); case PrimitiveType::TYPE_TIMESTAMPTZ: return visitor.template apply( field.template get()); diff --git a/be/src/exec/common/hash_table/hash.h b/be/src/exec/common/hash_table/hash.h index e8487bf4c8db96..ea598fd7fc383a 100644 --- a/be/src/exec/common/hash_table/hash.h +++ b/be/src/exec/common/hash_table/hash.h @@ -26,6 +26,7 @@ #include "core/string_ref.h" #include "core/types.h" #include "core/uint128.h" +#include "core/value/timestamp_ns_value.h" #include "parallel_hashmap/phmap_utils.h" // Here is an empirical value. @@ -111,6 +112,11 @@ struct DefaultHash> { } }; +template <> +struct DefaultHash { + size_t operator()(doris::TimeStampNsValue key) const { return int_hash64(key.epoch_nanos()); } +}; + template <> struct DefaultHash { size_t operator()(doris::TimestampTzValue key) const { @@ -168,6 +174,11 @@ inline size_t hash_crc32(doris::DateV2Value u) { return hash_crc32(u.to_date_int_val()); } +template <> +inline size_t hash_crc32(doris::TimeStampNsValue u) { + return hash_crc32(u.epoch_nanos()); +} + template <> inline size_t hash_crc32(doris::TimestampTzValue u) { return hash_crc32(u.to_date_int_val()); @@ -196,6 +207,7 @@ DEFINE_HASH(doris::Float64) DEFINE_HASH(doris::VecDateTimeValue) DEFINE_HASH(doris::DateV2Value) DEFINE_HASH(doris::DateV2Value) +DEFINE_HASH(doris::TimeStampNsValue) DEFINE_HASH(doris::TimestampTzValue) DEFINE_HASH(unsigned __int128) diff --git a/be/src/exec/operator/scan_operator.cpp b/be/src/exec/operator/scan_operator.cpp index e1aff774e7a39d..1f31ad0a630b2e 100644 --- a/be/src/exec/operator/scan_operator.cpp +++ b/be/src/exec/operator/scan_operator.cpp @@ -933,12 +933,13 @@ Status ScanLocalStateBase::_change_value_range(bool is_equal_op, func(temp_range, to_olap_filter_type(fn_name), tmp_value); } } else if constexpr ((PrimitiveType == TYPE_DECIMALV2) || (PrimitiveType == TYPE_DATETIMEV2) || - (PrimitiveType == TYPE_TINYINT) || (PrimitiveType == TYPE_SMALLINT) || - (PrimitiveType == TYPE_INT) || (PrimitiveType == TYPE_BIGINT) || - (PrimitiveType == TYPE_LARGEINT) || (PrimitiveType == TYPE_FLOAT) || - (PrimitiveType == TYPE_DOUBLE) || (PrimitiveType == TYPE_IPV4) || - (PrimitiveType == TYPE_IPV6) || (PrimitiveType == TYPE_DECIMAL32) || - (PrimitiveType == TYPE_DECIMAL64) || (PrimitiveType == TYPE_DECIMAL128I) || + (PrimitiveType == TYPE_TIMESTAMP_NS) || (PrimitiveType == TYPE_TINYINT) || + (PrimitiveType == TYPE_SMALLINT) || (PrimitiveType == TYPE_INT) || + (PrimitiveType == TYPE_BIGINT) || (PrimitiveType == TYPE_LARGEINT) || + (PrimitiveType == TYPE_FLOAT) || (PrimitiveType == TYPE_DOUBLE) || + (PrimitiveType == TYPE_IPV4) || (PrimitiveType == TYPE_IPV6) || + (PrimitiveType == TYPE_DECIMAL32) || (PrimitiveType == TYPE_DECIMAL64) || + (PrimitiveType == TYPE_DECIMAL128I) || (PrimitiveType == TYPE_DECIMAL256) || (PrimitiveType == TYPE_BOOLEAN) || (PrimitiveType == TYPE_DATEV2) || (PrimitiveType == TYPE_TIMESTAMPTZ) || (PrimitiveType == TYPE_DATETIME) || is_string_type(PrimitiveType)) { diff --git a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp index 109fa6f0171878..d8f108c3ec07b1 100644 --- a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp +++ b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp @@ -298,6 +298,8 @@ Status RuntimeFilterWrapper::_assign(const PInFilter& in_filter, bool contain_nu } case TYPE_TIMESTAMPTZ: [[fallthrough]]; + case TYPE_TIMESTAMP_NS: + [[fallthrough]]; case TYPE_DATETIMEV2: { batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { auto date_v2_val = column.longval(); @@ -491,6 +493,8 @@ Status RuntimeFilterWrapper::_assign(const PMinMaxFilter& minmax_filter, bool co } case TYPE_TIMESTAMPTZ: [[fallthrough]]; + case TYPE_TIMESTAMP_NS: + [[fallthrough]]; case TYPE_DATETIMEV2: { int64_t min_val = minmax_filter.min_val().longval(); int64_t max_val = minmax_filter.max_val().longval(); diff --git a/be/src/exprs/aggregate/aggregate_function_approx_count_distinct.cpp b/be/src/exprs/aggregate/aggregate_function_approx_count_distinct.cpp index 28aa90fc4420d2..2821382ebeb4e2 100644 --- a/be/src/exprs/aggregate/aggregate_function_approx_count_distinct.cpp +++ b/be/src/exprs/aggregate/aggregate_function_approx_count_distinct.cpp @@ -30,7 +30,7 @@ AggregateFunctionPtr create_aggregate_function_approx_count_distinct( TYPE_BOOLEAN, TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT, TYPE_LARGEINT, TYPE_FLOAT, TYPE_DOUBLE, TYPE_DECIMALV2, TYPE_DECIMAL32, TYPE_DECIMAL64, TYPE_DECIMAL128I, TYPE_DECIMAL256, TYPE_VARCHAR, TYPE_DATEV2, TYPE_DATETIMEV2, - TYPE_IPV4, TYPE_IPV6, + TYPE_TIMESTAMP_NS, TYPE_IPV4, TYPE_IPV6, TYPE_TIMESTAMPTZ>::create(argument_types, result_is_nullable, attr); diff --git a/be/src/exprs/aggregate/aggregate_function_histogram.cpp b/be/src/exprs/aggregate/aggregate_function_histogram.cpp index 622359b3da2199..17cc1cc96ca77d 100644 --- a/be/src/exprs/aggregate/aggregate_function_histogram.cpp +++ b/be/src/exprs/aggregate/aggregate_function_histogram.cpp @@ -39,11 +39,11 @@ AggregateFunctionPtr create_aggregate_function_histogram(const std::string& name const bool result_is_nullable, const AggregateFunctionAttr& attr) { assert_arity_range(name, argument_types, 1, 2); - using creator = - creator_with_type_list; + using creator = creator_with_type_list; if (argument_types.size() == 2) { return creator::create( argument_types, result_is_nullable, attr); @@ -58,4 +58,4 @@ void register_aggregate_function_histogram(AggregateFunctionSimpleFactory& facto factory.register_alias("histogram", "hist"); } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/exprs/aggregate/aggregate_function_map_v2.cpp b/be/src/exprs/aggregate/aggregate_function_map_v2.cpp index eaaec268914488..28f6f020ab0afd 100644 --- a/be/src/exprs/aggregate/aggregate_function_map_v2.cpp +++ b/be/src/exprs/aggregate/aggregate_function_map_v2.cpp @@ -53,6 +53,7 @@ AggregateFunctionPtr create_aggregate_function_map_agg_v2(const std::string& nam case PrimitiveType::TYPE_VARCHAR: case PrimitiveType::TYPE_DATEV2: case PrimitiveType::TYPE_DATETIMEV2: + case PrimitiveType::TYPE_TIMESTAMP_NS: case PrimitiveType::TYPE_TIMEV2: case PrimitiveType::TYPE_TIMESTAMPTZ: return create_agg_function_map_agg_v2(argument_types, result_is_nullable, attr); @@ -67,4 +68,4 @@ void register_aggregate_function_map_agg_v2(AggregateFunctionSimpleFactory& fact factory.register_function_both("map_agg_v2", create_aggregate_function_map_agg_v2); } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/exprs/aggregate/aggregate_function_min_max_impl.h b/be/src/exprs/aggregate/aggregate_function_min_max_impl.h index 75a03f8893b53a..9717cc0461c7c1 100644 --- a/be/src/exprs/aggregate/aggregate_function_min_max_impl.h +++ b/be/src/exprs/aggregate/aggregate_function_min_max_impl.h @@ -59,6 +59,10 @@ AggregateFunctionPtr create_aggregate_function_single_value(const String& name, return creator_without_type::create_unary_arguments< AggregateFunctionsSingleValue>>>( argument_types, result_is_nullable, attr); + case PrimitiveType::TYPE_TIMESTAMP_NS: + return creator_without_type::create_unary_arguments< + AggregateFunctionsSingleValue>>>( + argument_types, result_is_nullable, attr); case PrimitiveType::TYPE_TIMESTAMPTZ: return creator_without_type::create_unary_arguments< AggregateFunctionsSingleValue>>>( diff --git a/be/src/exprs/aggregate/aggregate_function_sequence_match.cpp b/be/src/exprs/aggregate/aggregate_function_sequence_match.cpp index 7e55c11e2219c3..3de5d44fcc091b 100644 --- a/be/src/exprs/aggregate/aggregate_function_sequence_match.cpp +++ b/be/src/exprs/aggregate/aggregate_function_sequence_match.cpp @@ -48,6 +48,9 @@ AggregateFunctionPtr create_aggregate_function_sequence_base(const std::string& case TYPE_DATETIMEV2: return creator_without_type::create>( argument_types, result_is_nullable, attr); + case TYPE_TIMESTAMP_NS: + return creator_without_type::create>( + argument_types, result_is_nullable, attr); case TYPE_DATEV2: return creator_without_type::create>( argument_types, result_is_nullable, attr); diff --git a/be/src/exprs/aggregate/aggregate_function_topn_array.cpp b/be/src/exprs/aggregate/aggregate_function_topn_array.cpp index 8c8fee3f4fde9d..7f1ebf7b4c2e5d 100644 --- a/be/src/exprs/aggregate/aggregate_function_topn_array.cpp +++ b/be/src/exprs/aggregate/aggregate_function_topn_array.cpp @@ -27,11 +27,10 @@ using ImplArray = AggregateFunctionTopNImplArray; template using ImplArrayWithDefault = AggregateFunctionTopNImplArray; -using topn_array_creator = - creator_with_type_list; +using topn_array_creator = creator_with_type_list< + TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT, TYPE_LARGEINT, TYPE_FLOAT, TYPE_DOUBLE, + TYPE_DECIMAL32, TYPE_DECIMAL64, TYPE_DECIMAL128I, TYPE_DECIMAL256, TYPE_VARCHAR, + TYPE_DATEV2, TYPE_DATETIMEV2, TYPE_TIMESTAMP_NS, TYPE_TIMESTAMPTZ, TYPE_IPV4, TYPE_IPV6>; AggregateFunctionPtr create_aggregate_function_topn_array(const std::string& name, const DataTypes& argument_types, diff --git a/be/src/exprs/aggregate/aggregate_function_topn_weighted.cpp b/be/src/exprs/aggregate/aggregate_function_topn_weighted.cpp index d1c8102458443c..dc955320d32ff5 100644 --- a/be/src/exprs/aggregate/aggregate_function_topn_weighted.cpp +++ b/be/src/exprs/aggregate/aggregate_function_topn_weighted.cpp @@ -27,11 +27,10 @@ using ImplWeight = AggregateFunctionTopNImplWeight; template using ImplWeightWithDefault = AggregateFunctionTopNImplWeight; -using topn_weighted_creator = - creator_with_type_list; +using topn_weighted_creator = creator_with_type_list< + TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT, TYPE_LARGEINT, TYPE_FLOAT, TYPE_DOUBLE, + TYPE_DECIMAL32, TYPE_DECIMAL64, TYPE_DECIMAL128I, TYPE_DECIMAL256, TYPE_VARCHAR, + TYPE_DATEV2, TYPE_DATETIMEV2, TYPE_TIMESTAMP_NS, TYPE_TIMESTAMPTZ, TYPE_IPV4, TYPE_IPV6>; AggregateFunctionPtr create_aggregate_function_topn_weighted(const std::string& name, const DataTypes& argument_types, diff --git a/be/src/exprs/aggregate/aggregate_function_uniq.cpp b/be/src/exprs/aggregate/aggregate_function_uniq.cpp index 9bfcc38855c46e..f9f5d75de774bc 100644 --- a/be/src/exprs/aggregate/aggregate_function_uniq.cpp +++ b/be/src/exprs/aggregate/aggregate_function_uniq.cpp @@ -52,12 +52,13 @@ AggregateFunctionPtr create_aggregate_function_uniq(const std::string& name, argument_types, result_is_nullable, attr); } } - return creator_with_type_list< - TYPE_BOOLEAN, TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT, TYPE_LARGEINT, - TYPE_DECIMAL32, TYPE_DECIMAL64, TYPE_DECIMAL128I, TYPE_DECIMAL256, TYPE_VARCHAR, - TYPE_ARRAY, TYPE_FLOAT, TYPE_DOUBLE, TYPE_DATEV2, TYPE_DATETIMEV2, TYPE_TIMESTAMPTZ, - TYPE_VARBINARY>::create(argument_types, result_is_nullable, - attr); + return creator_with_type_list::create( + argument_types, result_is_nullable, attr); } void register_aggregate_function_uniq(AggregateFunctionSimpleFactory& factory) { diff --git a/be/src/exprs/create_predicate_function.h b/be/src/exprs/create_predicate_function.h index b20c6613199d52..49b87750046032 100644 --- a/be/src/exprs/create_predicate_function.h +++ b/be/src/exprs/create_predicate_function.h @@ -92,6 +92,7 @@ class PredicateFunctionCreator { M(TYPE_DATETIME) \ M(TYPE_DATEV2) \ M(TYPE_DATETIMEV2) \ + M(TYPE_TIMESTAMP_NS) \ M(TYPE_TIMESTAMPTZ) \ M(TYPE_TIMEV2) \ M(TYPE_CHAR) \ diff --git a/be/src/exprs/function/cast/cast_to_date.h b/be/src/exprs/function/cast/cast_to_date.h index 009739b0b0bbbf..a4eb08df07699b 100644 --- a/be/src/exprs/function/cast/cast_to_date.h +++ b/be/src/exprs/function/cast/cast_to_date.h @@ -32,6 +32,7 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_time.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/primitive_type.h" #include "core/data_type_serde/data_type_serde.h" #include "core/types.h" @@ -43,7 +44,8 @@ namespace doris { template - requires(IsStringType && IsDatelikeTypes) + requires(IsStringType && + (IsDatelikeTypes || std::is_same_v)) class CastToImpl : public CastToBase { public: Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, diff --git a/be/src/exprs/function/cast/function_cast.cpp b/be/src/exprs/function/cast/function_cast.cpp index 5561a3a76de4a6..cabbba3d125e03 100644 --- a/be/src/exprs/function/cast/function_cast.cpp +++ b/be/src/exprs/function/cast/function_cast.cpp @@ -21,6 +21,7 @@ #include "core/data_type/data_type_decimal.h" #include "core/data_type/data_type_number.h" // IWYU pragma: keep #include "core/data_type/data_type_quantilestate.h" +#include "core/data_type/data_type_timestamp_ns.h" // IWYU pragma: keep #include "core/data_type/data_type_variant.h" #include "core/data_type/data_type_variant_v2.h" #include "core/data_type/primitive_type.h" @@ -312,6 +313,7 @@ WrapperType prepare_impl(FunctionContext* context, const DataTypePtr& origin_fro case PrimitiveType::TYPE_DATETIME: case PrimitiveType::TYPE_DATEV2: case PrimitiveType::TYPE_DATETIMEV2: + case PrimitiveType::TYPE_TIMESTAMP_NS: case PrimitiveType::TYPE_TIMEV2: return create_datelike_wrapper(context, from_type, to_type->get_primitive_type()); case PrimitiveType::TYPE_TIMESTAMPTZ: diff --git a/be/src/exprs/function/cast/function_cast_date.cpp b/be/src/exprs/function/cast/function_cast_date.cpp index 88013ab5e3a04b..5005e329f8082e 100644 --- a/be/src/exprs/function/cast/function_cast_date.cpp +++ b/be/src/exprs/function/cast/function_cast_date.cpp @@ -19,6 +19,7 @@ #include "core/data_type/data_type_date_or_datetime_v2.h" #include "core/data_type/data_type_date_time.h" #include "core/data_type/data_type_time.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "exprs/function/cast/cast_to_date.h" namespace doris::CastWrapper { @@ -30,9 +31,15 @@ WrapperType create_datelike_wrapper(FunctionContext* context, const DataTypePtr& auto make_datelike_wrapper = [&](const auto& types) -> bool { using Types = std::decay_t; using FromDataType = typename Types::LeftType; - if constexpr (CastUtil::IsPureDigitType || IsDatelikeTypes || - IsStringType || - std::is_same_v) { + constexpr bool is_timestamp_ns_source = std::is_same_v; + constexpr bool is_timestamp_ns_target = std::is_same_v; + constexpr bool is_supported_timestamp_ns_cast = + is_timestamp_ns_target && IsStringType; + constexpr bool is_supported_legacy_cast = + !is_timestamp_ns_source && !is_timestamp_ns_target && + (CastUtil::IsPureDigitType || IsDatelikeTypes || + IsStringType || std::is_same_v); + if constexpr (is_supported_timestamp_ns_cast || is_supported_legacy_cast) { if (context->enable_strict_mode()) { cast_to_datelike = std::make_shared< CastToImpl>(); @@ -72,6 +79,8 @@ WrapperType create_datelike_wrapper(FunctionContext* context, const DataTypePtr& return create_datelike_wrapper(context, from_type); case TYPE_DATETIMEV2: return create_datelike_wrapper(context, from_type); + case TYPE_TIMESTAMP_NS: + return create_datelike_wrapper(context, from_type); case TYPE_TIMEV2: return create_datelike_wrapper(context, from_type); default: diff --git a/be/src/exprs/function/functions_comparison.h b/be/src/exprs/function/functions_comparison.h index a0789c259c790b..5bd7f9fc8c2444 100644 --- a/be/src/exprs/function/functions_comparison.h +++ b/be/src/exprs/function/functions_comparison.h @@ -798,6 +798,8 @@ class FunctionComparison : public IFunction { return execute_num_type(block, result, col_left_ptr, col_right_ptr); case TYPE_DATETIMEV2: return execute_num_type(block, result, col_left_ptr, col_right_ptr); + case TYPE_TIMESTAMP_NS: + return execute_num_type(block, result, col_left_ptr, col_right_ptr); case TYPE_TIMESTAMPTZ: return execute_num_type(block, result, col_left_ptr, col_right_ptr); case TYPE_TINYINT: diff --git a/be/src/exprs/lambda_function/varray_sort_function.cpp b/be/src/exprs/lambda_function/varray_sort_function.cpp index dfd2ed1a8ae58e..2d0484c64f07a7 100644 --- a/be/src/exprs/lambda_function/varray_sort_function.cpp +++ b/be/src/exprs/lambda_function/varray_sort_function.cpp @@ -52,7 +52,7 @@ using ConstColumnVariant = const ColumnDecimal32*, const ColumnDecimal64*, const ColumnDecimal128V2*, const ColumnDecimal128V3*, const ColumnDecimal256*, const ColumnDate*, const ColumnDateTime*, const ColumnDateV2*, const ColumnDateTimeV2*, - const ColumnTimeV2*>; + const ColumnTimeStampNs*, const ColumnTimeV2*>; template struct is_column_vector : std::false_type {}; @@ -422,6 +422,7 @@ class ArraySortFunction : public LambdaFunction { DISPATCH_PRIMITIVE_TYPE(TYPE_DATETIME, ColumnDateTime) DISPATCH_PRIMITIVE_TYPE(TYPE_DATEV2, ColumnDateV2) DISPATCH_PRIMITIVE_TYPE(TYPE_DATETIMEV2, ColumnDateTimeV2) + DISPATCH_PRIMITIVE_TYPE(TYPE_TIMESTAMP_NS, ColumnTimeStampNs) DISPATCH_PRIMITIVE_TYPE(TYPE_TIMEV2, ColumnTimeV2) default: return Status::InternalError("Unsupported type in array_sort"); diff --git a/be/src/exprs/vcase_expr.h b/be/src/exprs/vcase_expr.h index 07793b969e019f..d1312cb4983c3a 100644 --- a/be/src/exprs/vcase_expr.h +++ b/be/src/exprs/vcase_expr.h @@ -143,6 +143,7 @@ class VCaseExpr final : public VExpr { CASE_TYPE(TYPE_DATETIME, ColumnDateTime) CASE_TYPE(TYPE_DATEV2, ColumnDateV2) CASE_TYPE(TYPE_DATETIMEV2, ColumnDateTimeV2) + CASE_TYPE(TYPE_TIMESTAMP_NS, ColumnTimeStampNs) CASE_TYPE(TYPE_TIMESTAMPTZ, ColumnTimeStampTz) CASE_TYPE(TYPE_IPV6, ColumnIPv6) CASE_TYPE(TYPE_IPV4, ColumnIPv4) @@ -231,6 +232,7 @@ class VCaseExpr final : public VExpr { std::is_same_v || std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v) { result_raw_data[i] = ColumnType::default_value(); } else { @@ -251,6 +253,7 @@ class VCaseExpr final : public VExpr { std::is_same_v || std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v) { for (int row_idx = 0; row_idx < rows_count; row_idx++) { result_raw_data[row_idx] = (then_idx[row_idx] == i) ? column_raw_data[row_idx] diff --git a/be/src/exprs/vcondition_expr.cpp b/be/src/exprs/vcondition_expr.cpp index 0f9d8e6a7bd427..e2293faf3dcd93 100644 --- a/be/src/exprs/vcondition_expr.cpp +++ b/be/src/exprs/vcondition_expr.cpp @@ -577,6 +577,11 @@ void insert_result_data(MutableColumnPtr& result_column, ColumnPtr& argument_col result_raw_data[row].to_date_int_val() + column_raw_data[row].to_date_int_val() * uint64_t(!(null_map_data[row] | filled_flag[row]))); + } else if constexpr (std::is_same_v) { + result_raw_data[row] = + TimeStampNsValue(result_raw_data[row].epoch_nanos() + + column_raw_data[row].epoch_nanos() * + int64_t(!(null_map_data[row] | filled_flag[row]))); } else if constexpr (std::is_same_v) { result_raw_data[row] = binary_cast( result_raw_data[row].to_date_int_val() + @@ -635,7 +640,6 @@ Status filled_result_column(const DataTypePtr& data_type, MutableColumnPtr& resu return insert_result_data_bitmap(result_column, argument_column, null_map_data, filled_flag, input_rows_count); } - auto call = [&](const auto& type) -> bool { using DispatchType = std::decay_t; insert_result_data( diff --git a/be/src/exprs/vdirect_in_predicate.h b/be/src/exprs/vdirect_in_predicate.h index 693571d25bf777..5a71294ddb42a2 100644 --- a/be/src/exprs/vdirect_in_predicate.h +++ b/be/src/exprs/vdirect_in_predicate.h @@ -264,6 +264,7 @@ class VDirectInPredicate final : public VExpr { RETURN_RAW_FIXED_SIZE(TYPE_DATETIME); RETURN_RAW_FIXED_SIZE(TYPE_DATEV2); RETURN_RAW_FIXED_SIZE(TYPE_DATETIMEV2); + RETURN_RAW_FIXED_SIZE(TYPE_TIMESTAMP_NS); RETURN_RAW_FIXED_SIZE(TYPE_TIMESTAMPTZ); RETURN_RAW_FIXED_SIZE(TYPE_TIMEV2); RETURN_RAW_FIXED_SIZE(TYPE_DECIMAL32); diff --git a/be/src/exprs/vectorized_fn_call.cpp b/be/src/exprs/vectorized_fn_call.cpp index 6e9f25be48d864..d88ac5a587a3c3 100644 --- a/be/src/exprs/vectorized_fn_call.cpp +++ b/be/src/exprs/vectorized_fn_call.cpp @@ -197,6 +197,7 @@ size_t raw_comparison_value_size(PrimitiveType primitive_type) { RETURN_RAW_COMPARISON_SIZE(TYPE_DATETIME); RETURN_RAW_COMPARISON_SIZE(TYPE_DATEV2); RETURN_RAW_COMPARISON_SIZE(TYPE_DATETIMEV2); + RETURN_RAW_COMPARISON_SIZE(TYPE_TIMESTAMP_NS); RETURN_RAW_COMPARISON_SIZE(TYPE_TIMESTAMPTZ); RETURN_RAW_COMPARISON_SIZE(TYPE_TIMEV2); RETURN_RAW_COMPARISON_SIZE(TYPE_DECIMAL32); @@ -542,6 +543,7 @@ Status VectorizedFnCall::execute_on_raw_fixed_values(const uint8_t* values, size EXECUTE_RAW_SCALAR_COMPARISON(TYPE_DATETIME); EXECUTE_RAW_SCALAR_COMPARISON(TYPE_DATEV2); EXECUTE_RAW_SCALAR_COMPARISON(TYPE_DATETIMEV2); + EXECUTE_RAW_SCALAR_COMPARISON(TYPE_TIMESTAMP_NS); EXECUTE_RAW_SCALAR_COMPARISON(TYPE_TIMESTAMPTZ); EXECUTE_RAW_SCALAR_COMPARISON(TYPE_TIMEV2); EXECUTE_RAW_SCALAR_COMPARISON(TYPE_DECIMAL32); diff --git a/be/src/exprs/vexpr.cpp b/be/src/exprs/vexpr.cpp index 5b1f835ace5a26..7ee3c772829863 100644 --- a/be/src/exprs/vexpr.cpp +++ b/be/src/exprs/vexpr.cpp @@ -138,6 +138,10 @@ TExprNode create_texpr_node_from(const void* data, const PrimitiveType& type, in THROW_IF_ERROR(create_texpr_literal_node(data, &node, precision, scale)); break; } + case TYPE_TIMESTAMP_NS: { + THROW_IF_ERROR(create_texpr_literal_node(data, &node, precision, scale)); + break; + } case TYPE_DATE: { THROW_IF_ERROR(create_texpr_literal_node(data, &node)); break; @@ -261,6 +265,12 @@ TExprNode create_texpr_node_from(const Field& field, const PrimitiveType& type, create_texpr_literal_node(&storage, &node, precision, scale)); break; } + case TYPE_TIMESTAMP_NS: { + const auto& storage = field.get(); + THROW_IF_ERROR( + create_texpr_literal_node(&storage, &node, precision, scale)); + break; + } case TYPE_TIMESTAMPTZ: { const auto& storage = field.get(); diff --git a/be/src/exprs/vexpr.h b/be/src/exprs/vexpr.h index c0b99ab7a8f4dc..52eb9987050e66 100644 --- a/be/src/exprs/vexpr.h +++ b/be/src/exprs/vexpr.h @@ -612,6 +612,15 @@ Status create_texpr_literal_node(const void* data, TExprNode* node, int precisio (*node).__set_date_literal(date_literal); (*node).__set_node_type(TExprNodeType::DATE_LITERAL); (*node).__set_type(create_type_desc(PrimitiveType::TYPE_DATETIMEV2, precision, scale)); + } else if constexpr (T == TYPE_TIMESTAMP_NS) { + const auto* origin_value = reinterpret_cast(data); + TDateLiteral date_literal; + char convert_buffer[30]; + origin_value->to_string(convert_buffer, scale); + date_literal.__set_value(convert_buffer); + (*node).__set_date_literal(date_literal); + (*node).__set_node_type(TExprNodeType::DATE_LITERAL); + (*node).__set_type(create_type_desc(PrimitiveType::TYPE_TIMESTAMP_NS)); } else if constexpr (T == TYPE_TIMESTAMPTZ) { const auto* origin_value = reinterpret_cast(data); TDateLiteral date_literal; diff --git a/be/src/exprs/vin_predicate.cpp b/be/src/exprs/vin_predicate.cpp index 0dbfb3f7e9a289..61bd18f2f8b1c8 100644 --- a/be/src/exprs/vin_predicate.cpp +++ b/be/src/exprs/vin_predicate.cpp @@ -64,6 +64,7 @@ size_t raw_in_value_size(PrimitiveType primitive_type) { RETURN_RAW_IN_SIZE(TYPE_DATETIME); RETURN_RAW_IN_SIZE(TYPE_DATEV2); RETURN_RAW_IN_SIZE(TYPE_DATETIMEV2); + RETURN_RAW_IN_SIZE(TYPE_TIMESTAMP_NS); RETURN_RAW_IN_SIZE(TYPE_TIMESTAMPTZ); RETURN_RAW_IN_SIZE(TYPE_TIMEV2); RETURN_RAW_IN_SIZE(TYPE_DECIMAL32); diff --git a/be/src/exprs/vtopn_pred.cpp b/be/src/exprs/vtopn_pred.cpp index cc26b6679a1342..07211dc1916a89 100644 --- a/be/src/exprs/vtopn_pred.cpp +++ b/be/src/exprs/vtopn_pred.cpp @@ -45,6 +45,7 @@ size_t topn_raw_value_size(PrimitiveType type) { RETURN_TOPN_RAW_SIZE(TYPE_DATETIME); RETURN_TOPN_RAW_SIZE(TYPE_DATEV2); RETURN_TOPN_RAW_SIZE(TYPE_DATETIMEV2); + RETURN_TOPN_RAW_SIZE(TYPE_TIMESTAMP_NS); RETURN_TOPN_RAW_SIZE(TYPE_TIMESTAMPTZ); // Master no longer defines a C++ carrier for legacy TYPE_TIME; Parquet time values use // TYPE_TIMEV2, so advertising the deprecated tag would make direct filtering unusable. @@ -160,6 +161,7 @@ Status VTopNPred::execute_on_raw_fixed_values(const uint8_t* values, size_t num_ EXECUTE_TOPN_RAW_SCALAR(TYPE_DATETIME); EXECUTE_TOPN_RAW_SCALAR(TYPE_DATEV2); EXECUTE_TOPN_RAW_SCALAR(TYPE_DATETIMEV2); + EXECUTE_TOPN_RAW_SCALAR(TYPE_TIMESTAMP_NS); EXECUTE_TOPN_RAW_SCALAR(TYPE_TIMESTAMPTZ); // Keep dispatch aligned with topn_raw_value_size(): legacy TYPE_TIME has no master carrier. EXECUTE_TOPN_RAW_SCALAR(TYPE_TIMEV2); diff --git a/be/src/information_schema/schema_columns_scanner.cpp b/be/src/information_schema/schema_columns_scanner.cpp index e84acc790e0b2b..158e13db97ec2b 100644 --- a/be/src/information_schema/schema_columns_scanner.cpp +++ b/be/src/information_schema/schema_columns_scanner.cpp @@ -134,6 +134,8 @@ std::string SchemaColumnsScanner::_to_mysql_data_type_string(TColumnDesc& desc) case TPrimitiveType::DATETIME: case TPrimitiveType::DATETIMEV2: return "datetime"; + case TPrimitiveType::TIMESTAMP_NS: + return "timestamp_ns"; case TPrimitiveType::TIMESTAMPTZ: return "timestamp"; case TPrimitiveType::DECIMAL32: diff --git a/be/src/runtime/fold_constant_executor.cpp b/be/src/runtime/fold_constant_executor.cpp index 97efd2d72deb5b..dcfcf656cc0e38 100644 --- a/be/src/runtime/fold_constant_executor.cpp +++ b/be/src/runtime/fold_constant_executor.cpp @@ -265,6 +265,11 @@ Status FoldConstantExecutor::_get_result(void* src, size_t size, const DataTypeP result = CastToString::from_datetimev2(value, type->get_scale()); break; } + case TYPE_TIMESTAMP_NS: { + const auto value = TimeStampNsValue(*reinterpret_cast(src)); + result = value.to_string(type->get_scale()); + break; + } case TYPE_TIMESTAMPTZ: { auto value = binary_cast(*(int64_t*)src); result = CastToString::from_timestamptz(value, type->get_scale(), options.timezone); @@ -298,4 +303,4 @@ Status FoldConstantExecutor::_get_result(void* src, size_t size, const DataTypeP return Status::OK(); } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/storage/delete/delete_handler.cpp b/be/src/storage/delete/delete_handler.cpp index 1781c7734de7d0..132a0cb12eb3e0 100644 --- a/be/src/storage/delete/delete_handler.cpp +++ b/be/src/storage/delete/delete_handler.cpp @@ -28,6 +28,7 @@ #include "common/logging.h" #include "common/status.h" #include "core/data_type_serde/data_type_serde.h" +#include "core/data_type_serde/data_type_timestamp_ns_serde.h" #include "storage/olap_common.h" #include "storage/predicate/block_column_predicate.h" #include "storage/predicate/null_predicate.h" @@ -88,6 +89,7 @@ Status convert(const DataTypePtr& data_type, const std::list& str, FROM_FE_STRING_CASE(TYPE_DATETIME); FROM_FE_STRING_CASE(TYPE_DATEV2); FROM_FE_STRING_CASE(TYPE_DATETIMEV2); + FROM_FE_STRING_CASE(TYPE_TIMESTAMP_NS); FROM_FE_STRING_CASE(TYPE_TIMESTAMPTZ); FROM_FE_STRING_CASE(TYPE_BOOLEAN); FROM_FE_STRING_CASE(TYPE_IPV4); @@ -359,6 +361,10 @@ bool DeleteHandler::is_condition_value_valid(const TabletColumn& column, case FieldType::OLAP_FIELD_TYPE_DATETIMEV2: case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ: return valid_datetime(value_str, column.frac()); + case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS: { + int64_t epoch_nanos = 0; + return parse_timestamp_ns(StringRef(value_str), &epoch_nanos).ok(); + } case FieldType::OLAP_FIELD_TYPE_BOOL: return valid_bool(value_str); case FieldType::OLAP_FIELD_TYPE_IPV4: diff --git a/be/src/storage/index/bloom_filter/bloom_filter_index_writer.cpp b/be/src/storage/index/bloom_filter/bloom_filter_index_writer.cpp index 785e889a831eee..8d6a7aec6015fb 100644 --- a/be/src/storage/index/bloom_filter/bloom_filter_index_writer.cpp +++ b/be/src/storage/index/bloom_filter/bloom_filter_index_writer.cpp @@ -330,6 +330,7 @@ Status BloomFilterIndexWriter::create(const BloomFilterOptions& bf_options, Fiel M(FieldType::OLAP_FIELD_TYPE_DECIMAL) M(FieldType::OLAP_FIELD_TYPE_DATEV2) M(FieldType::OLAP_FIELD_TYPE_DATETIMEV2) + M(FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS) M(FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ) M(FieldType::OLAP_FIELD_TYPE_DECIMAL32) M(FieldType::OLAP_FIELD_TYPE_DECIMAL64) diff --git a/be/src/storage/index/index_writer.cpp b/be/src/storage/index/index_writer.cpp index 2325d280471337..785c37be49e293 100644 --- a/be/src/storage/index/index_writer.cpp +++ b/be/src/storage/index/index_writer.cpp @@ -102,6 +102,7 @@ Status IndexColumnWriter::create(const TabletColumn* column, M(FieldType::OLAP_FIELD_TYPE_DECIMAL) M(FieldType::OLAP_FIELD_TYPE_DATEV2) M(FieldType::OLAP_FIELD_TYPE_DATETIMEV2) + M(FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS) M(FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ) M(FieldType::OLAP_FIELD_TYPE_DECIMAL32) M(FieldType::OLAP_FIELD_TYPE_DECIMAL64) diff --git a/be/src/storage/index/inverted/inverted_index_writer.cpp b/be/src/storage/index/inverted/inverted_index_writer.cpp index 0f82b5225e666e..519568c28eb0bf 100644 --- a/be/src/storage/index/inverted/inverted_index_writer.cpp +++ b/be/src/storage/index/inverted/inverted_index_writer.cpp @@ -672,6 +672,7 @@ template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; +template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; @@ -683,4 +684,4 @@ template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; -} // namespace doris::segment_v2 \ No newline at end of file +} // namespace doris::segment_v2 diff --git a/be/src/storage/index/zone_map/zone_map_index.cpp b/be/src/storage/index/zone_map/zone_map_index.cpp index 23e2e5a4a0feec..cd687b4a623735 100644 --- a/be/src/storage/index/zone_map/zone_map_index.cpp +++ b/be/src/storage/index/zone_map/zone_map_index.cpp @@ -358,6 +358,7 @@ ZoneMapIndexReader::~ZoneMapIndexReader() = default; M(TYPE_DATETIME) \ M(TYPE_DATEV2) \ M(TYPE_DATETIMEV2) \ + M(TYPE_TIMESTAMP_NS) \ M(TYPE_TIMESTAMPTZ) \ M(TYPE_IPV4) \ M(TYPE_IPV6) \ diff --git a/be/src/storage/iterator/olap_data_convertor.cpp b/be/src/storage/iterator/olap_data_convertor.cpp index 47da182b68e05b..294bebd8e42ec1 100644 --- a/be/src/storage/iterator/olap_data_convertor.cpp +++ b/be/src/storage/iterator/olap_data_convertor.cpp @@ -167,6 +167,9 @@ OlapBlockDataConvertor::create_olap_column_data_convertor(const TabletColumn& co case FieldType::OLAP_FIELD_TYPE_DATETIMEV2: { return std::make_unique(); } + case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS: { + return std::make_unique>(); + } case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ: { return std::make_unique>(); } diff --git a/be/src/storage/key_coder.cpp b/be/src/storage/key_coder.cpp index d7a4d7b1457724..e683b299e9bb1d 100644 --- a/be/src/storage/key_coder.cpp +++ b/be/src/storage/key_coder.cpp @@ -77,6 +77,7 @@ class KeyCoderResolver { add_mapping(); add_mapping(); add_mapping(); + add_mapping(); add_mapping(); add_mapping(); add_mapping(); diff --git a/be/src/storage/key_coder.h b/be/src/storage/key_coder.h index 0c4bcf08d171e5..803e5c9c7326d2 100644 --- a/be/src/storage/key_coder.h +++ b/be/src/storage/key_coder.h @@ -469,6 +469,7 @@ class KeyCoderTraits M(OLAP_FIELD_TYPE_DATETIME, TYPE_DATETIME) \ M(OLAP_FIELD_TYPE_DATEV2, TYPE_DATEV2) \ M(OLAP_FIELD_TYPE_DATETIMEV2, TYPE_DATETIMEV2) \ + M(OLAP_FIELD_TYPE_TIMESTAMP_NS, TYPE_TIMESTAMP_NS) \ M(OLAP_FIELD_TYPE_TIMESTAMPTZ, TYPE_TIMESTAMPTZ) \ M(OLAP_FIELD_TYPE_IPV4, TYPE_IPV4) \ M(OLAP_FIELD_TYPE_IPV6, TYPE_IPV6) diff --git a/be/src/storage/olap_common.h b/be/src/storage/olap_common.h index 12ca1d2e6a458f..af14528bb7937a 100644 --- a/be/src/storage/olap_common.h +++ b/be/src/storage/olap_common.h @@ -168,6 +168,7 @@ enum class FieldType { OLAP_FIELD_TYPE_IPV4 = 38, OLAP_FIELD_TYPE_IPV6 = 39, OLAP_FIELD_TYPE_TIMESTAMPTZ = 40, + OLAP_FIELD_TYPE_TIMESTAMP_NS = 41, }; // Define all aggregation methods supported by TabletColumn @@ -224,6 +225,7 @@ constexpr bool field_is_numeric_type(const FieldType& field_type) { field_type == FieldType::OLAP_FIELD_TYPE_DATEV2 || field_type == FieldType::OLAP_FIELD_TYPE_DATETIME || field_type == FieldType::OLAP_FIELD_TYPE_DATETIMEV2 || + field_type == FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS || field_type == FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ || field_type == FieldType::OLAP_FIELD_TYPE_LARGEINT || field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL || diff --git a/be/src/storage/olap_scan_common.h b/be/src/storage/olap_scan_common.h index f7162f3b1f12b4..c14596f86ad4f8 100644 --- a/be/src/storage/olap_scan_common.h +++ b/be/src/storage/olap_scan_common.h @@ -233,6 +233,7 @@ class ColumnValueRange { primitive_type == PrimitiveType::TYPE_BOOLEAN || primitive_type == PrimitiveType::TYPE_DATETIME || primitive_type == PrimitiveType::TYPE_DATETIMEV2 || + primitive_type == PrimitiveType::TYPE_TIMESTAMP_NS || primitive_type == PrimitiveType::TYPE_TIMESTAMPTZ || primitive_type == PrimitiveType::TYPE_DECIMAL256; }; @@ -337,10 +338,10 @@ using ColumnValueRangeType = std::variant< ColumnValueRange, ColumnValueRange, ColumnValueRange, ColumnValueRange, ColumnValueRange, ColumnValueRange, ColumnValueRange, ColumnValueRange, - ColumnValueRange, ColumnValueRange, - ColumnValueRange, ColumnValueRange, - ColumnValueRange, ColumnValueRange, - ColumnValueRange>; + ColumnValueRange, ColumnValueRange, + ColumnValueRange, ColumnValueRange, + ColumnValueRange, ColumnValueRange, + ColumnValueRange, ColumnValueRange>; template const typename ColumnValueRange::CppType diff --git a/be/src/storage/partial_update_info.cpp b/be/src/storage/partial_update_info.cpp index 1bea6e05cb75f5..205400a2b674f8 100644 --- a/be/src/storage/partial_update_info.cpp +++ b/be/src/storage/partial_update_info.cpp @@ -273,6 +273,7 @@ void PartialUpdateInfo::_generate_default_values_for_missing_cids( if (column.has_default_value()) { std::string default_value; if (UNLIKELY((column.type() == FieldType::OLAP_FIELD_TYPE_DATETIMEV2 || + column.type() == FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS || column.type() == FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ) && to_lower(column.default_value()).find(to_lower("CURRENT_TIMESTAMP")) != std::string::npos)) { @@ -288,7 +289,19 @@ void PartialUpdateInfo::_generate_default_values_for_missing_cids( int precision = std::stoi(column.default_value().substr(pos + 1)); DateV2Value dtv; dtv.from_unixtime(timestamp_ms / 1000, nano_seconds, timezone, precision); - default_value = dtv.to_string(); + if (column.type() == FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS) { + const int32_t scale_factor = static_cast(int_exp10(9 - precision)); + const int32_t truncated_nano_seconds = + nano_seconds / scale_factor * scale_factor; + TimeStampNsValue value; + DORIS_CHECK(value.from_datetime( + dtv, static_cast( + truncated_nano_seconds % + (TimeStampNsValue::NANOS_PER_SECOND / 1000000)))); + default_value = value.to_string(precision); + } else { + default_value = dtv.to_string(); + } if (column.type() == FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ) { default_value += timezone; } diff --git a/be/src/storage/predicate/predicate_creator.cpp b/be/src/storage/predicate/predicate_creator.cpp index 485ca89eea6f49..bcfdf93ef15386 100644 --- a/be/src/storage/predicate/predicate_creator.cpp +++ b/be/src/storage/predicate/predicate_creator.cpp @@ -94,6 +94,10 @@ std::shared_ptr create_bloom_filter_predicate( return BloomFilterColumnPredicate::create_shared(cid, col_name, filter_olap); } + case TYPE_TIMESTAMP_NS: { + return BloomFilterColumnPredicate::create_shared(cid, col_name, + filter_olap); + } case TYPE_TIMESTAMPTZ: { return BloomFilterColumnPredicate::create_shared(cid, col_name, filter_olap); diff --git a/be/src/storage/predicate/predicate_creator_comparison.cpp b/be/src/storage/predicate/predicate_creator_comparison.cpp index b10a175b016592..d444a35a0845b8 100644 --- a/be/src/storage/predicate/predicate_creator_comparison.cpp +++ b/be/src/storage/predicate/predicate_creator_comparison.cpp @@ -102,6 +102,10 @@ std::shared_ptr create_comparison_predicate(const uint32_t cid, return ComparisonPredicateBase::create_shared(cid, col_name, value, opposite); } + case TYPE_TIMESTAMP_NS: { + return ComparisonPredicateBase::create_shared(cid, col_name, value, + opposite); + } case TYPE_TIMESTAMPTZ: { return ComparisonPredicateBase::create_shared(cid, col_name, value, opposite); diff --git a/be/src/storage/predicate/predicate_creator_in_list_in.cpp b/be/src/storage/predicate/predicate_creator_in_list_in.cpp index f1ad365b4f523e..9162d02506f735 100644 --- a/be/src/storage/predicate/predicate_creator_in_list_in.cpp +++ b/be/src/storage/predicate/predicate_creator_in_list_in.cpp @@ -139,6 +139,10 @@ std::shared_ptr create_in_list_predicate( cid, col_name, set, is_opposite); } + case TYPE_TIMESTAMP_NS: { + return create_in_list_predicate_impl( + cid, col_name, set, is_opposite); + } case TYPE_TIMESTAMPTZ: { return create_in_list_predicate_impl( cid, col_name, set, is_opposite); diff --git a/be/src/storage/predicate/predicate_creator_in_list_not_in.cpp b/be/src/storage/predicate/predicate_creator_in_list_not_in.cpp index e4cb4731a57095..d90d8538dffca9 100644 --- a/be/src/storage/predicate/predicate_creator_in_list_not_in.cpp +++ b/be/src/storage/predicate/predicate_creator_in_list_not_in.cpp @@ -139,6 +139,10 @@ std::shared_ptr create_in_list_predicate( cid, col_name, set, is_opposite); } + case TYPE_TIMESTAMP_NS: { + return create_in_list_predicate_impl( + cid, col_name, set, is_opposite); + } case TYPE_TIMESTAMPTZ: { return create_in_list_predicate_impl( cid, col_name, set, is_opposite); diff --git a/be/src/storage/segment/encoding_info.cpp b/be/src/storage/segment/encoding_info.cpp index d2b9d99f114e85..5c15d34de3e062 100644 --- a/be/src/storage/segment/encoding_info.cpp +++ b/be/src/storage/segment/encoding_info.cpp @@ -306,6 +306,9 @@ EncodingInfoResolver::EncodingInfoResolver() { _register_supported_encoding(); _register_supported_encoding(); _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); _register_supported_encoding(); _register_supported_encoding(); _register_supported_encoding(); @@ -364,6 +367,7 @@ EncodingInfoResolver::EncodingInfoResolver() { _set_v2_default(); _set_v2_default(); _set_v2_default(); + _set_v2_default(); _set_v2_default(); _set_v2_default(); _set_v2_default(); @@ -397,6 +401,7 @@ EncodingInfoResolver::EncodingInfoResolver() { _set_v3_default(); _set_v3_default(); _set_v3_default(); + _set_v3_default(); _set_v3_default(); _set_v3_default(); _set_v3_default(); diff --git a/be/src/storage/tablet/tablet_schema.cpp b/be/src/storage/tablet/tablet_schema.cpp index 13ea91d032dadd..6773226d2ba9cd 100644 --- a/be/src/storage/tablet/tablet_schema.cpp +++ b/be/src/storage/tablet/tablet_schema.cpp @@ -98,6 +98,8 @@ FieldType TabletColumn::get_field_type_by_string(const std::string& type_str) { type = FieldType::OLAP_FIELD_TYPE_DATEV2; } else if (0 == upper_type_str.compare("DATETIMEV2")) { type = FieldType::OLAP_FIELD_TYPE_DATETIMEV2; + } else if (0 == upper_type_str.compare("TIMESTAMP_NS")) { + type = FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS; } else if (0 == upper_type_str.compare("DATETIME")) { type = FieldType::OLAP_FIELD_TYPE_DATETIME; } else if (0 == upper_type_str.compare("TIMESTAMPTZ")) { @@ -241,6 +243,8 @@ std::string TabletColumn::get_string_by_field_type(FieldType type) { case FieldType::OLAP_FIELD_TYPE_DATETIMEV2: return "DATETIMEV2"; + case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS: + return "TIMESTAMP_NS"; case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ: return "TIMESTAMPTZ"; @@ -356,6 +360,7 @@ uint32_t TabletColumn::get_field_length_by_type(TPrimitiveType::type type, uint3 case TPrimitiveType::DATETIME: return 8; case TPrimitiveType::DATETIMEV2: + case TPrimitiveType::TIMESTAMP_NS: case TPrimitiveType::TIMESTAMPTZ: return 8; case TPrimitiveType::FLOAT: @@ -479,6 +484,9 @@ void TabletColumn::init_from_pb(const ColumnPB& column) { if (column.has_frac()) { _frac = column.frac(); } + if (_type == FieldType::OLAP_FIELD_TYPE_DATETIMEV2 && _frac > 6) { + _type = FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS; + } _length = column.length(); _index_length = column.index_length(); if (column.has_is_bf_column()) { diff --git a/be/src/storage/tablet_info.cpp b/be/src/storage/tablet_info.cpp index 9fee41082442c2..9b6a3fe6ed7699 100644 --- a/be/src/storage/tablet_info.cpp +++ b/be/src/storage/tablet_info.cpp @@ -42,6 +42,7 @@ #include "core/data_type/data_type_factory.hpp" #include "core/data_type/define_primitive_type.h" #include "core/data_type/primitive_type.h" +#include "core/data_type_serde/data_type_timestamp_ns_serde.h" #include "core/value/large_int_value.h" #include "runtime/descriptors.h" #include "runtime/memory/mem_tracker.h" @@ -647,6 +648,13 @@ static Status _create_partition_key(const TExprNode& t_expr, BlockRow* part_key, return Status::InternalError(ss.str()); } column->insert_data(reinterpret_cast(&dt), 0); + } else if (primitive_type == TYPE_TIMESTAMP_NS) { + int64_t epoch_nanos = 0; + RETURN_IF_ERROR(parse_timestamp_ns( + {t_expr.date_literal.value.data(), t_expr.date_literal.value.size()}, + &epoch_nanos)); + const TimeStampNsValue dt(epoch_nanos); + column->insert_data(reinterpret_cast(&dt), 0); } else if (primitive_type == TYPE_TIMESTAMPTZ) { TimestampTzValue res; CastParameters params {.status = Status::OK(), .is_strict = true}; diff --git a/be/src/storage/types.h b/be/src/storage/types.h index 7d66631ef16180..432715b56a01c0 100644 --- a/be/src/storage/types.h +++ b/be/src/storage/types.h @@ -173,6 +173,11 @@ struct CppTypeTraits { using UnsignedCppType = uint64_t; }; template <> +struct CppTypeTraits { + using CppType = int64_t; + using UnsignedCppType = uint64_t; +}; +template <> struct CppTypeTraits { using CppType = uint64_t; using UnsignedCppType = uint64_t; @@ -306,6 +311,9 @@ struct FieldTypeTraits template <> struct FieldTypeTraits : public BaseFieldTypeTraits {}; +template <> +struct FieldTypeTraits + : public BaseFieldTypeTraits {}; template <> struct FieldTypeTraits @@ -421,6 +429,7 @@ inline size_t field_type_size(FieldType field_type) { DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DATETIME) DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DATEV2) DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DATETIMEV2) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_TIMESTAMP_NS) DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_TIMEV2) DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_TIMESTAMPTZ) DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_IPV4) diff --git a/be/src/tools/meta_tool.cpp b/be/src/tools/meta_tool.cpp index 18bc27c1d839c2..2c95ce29d9b0f7 100644 --- a/be/src/tools/meta_tool.cpp +++ b/be/src/tools/meta_tool.cpp @@ -386,6 +386,8 @@ std::string get_field_type_string(doris::FieldType type) { return "DATEV2"; case doris::FieldType::OLAP_FIELD_TYPE_DATETIMEV2: return "DATETIMEV2"; + case doris::FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS: + return "TIMESTAMP_NS"; case doris::FieldType::OLAP_FIELD_TYPE_BOOL: return "BOOLEAN"; case doris::FieldType::OLAP_FIELD_TYPE_STRUCT: @@ -513,6 +515,14 @@ std::string format_column_value(const doris::IColumn& column, size_t row, } return ""; } + case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS: { + const StringRef& data = column.get_data_at(row); + if (data.size == sizeof(int64_t)) { + int64_t val = *reinterpret_cast(data.data); + return std::to_string(val); + } + return ""; + } case FieldType::OLAP_FIELD_TYPE_CHAR: case FieldType::OLAP_FIELD_TYPE_VARCHAR: case FieldType::OLAP_FIELD_TYPE_STRING: diff --git a/be/src/util/raw_value.h b/be/src/util/raw_value.h index 4babb5c0810037..ca9914e4064d9d 100644 --- a/be/src/util/raw_value.h +++ b/be/src/util/raw_value.h @@ -85,7 +85,8 @@ inline uint32_t RawValue::zlib_crc32(const void* v, size_t len, const PrimitiveT return HashUtil::zlib_crc_hash(v, 4, seed); } - case TYPE_DATETIMEV2: { + case TYPE_DATETIMEV2: + case TYPE_TIMESTAMP_NS: { return HashUtil::zlib_crc_hash(v, 8, seed); } diff --git a/be/test/core/data_type/data_type_datetime_v2_test.cpp b/be/test/core/data_type/data_type_datetime_v2_test.cpp index e1a58ae49a543e..b3aeec27da2e8e 100644 --- a/be/test/core/data_type/data_type_datetime_v2_test.cpp +++ b/be/test/core/data_type/data_type_datetime_v2_test.cpp @@ -120,6 +120,9 @@ TEST_F(DataTypeDateTimeV2Test, simple_func_test) { test_func(dt_time_v2_6); EXPECT_THROW(DataTypeDateTimeV2(7), Exception); EXPECT_THROW(create_datetimev2(7), Exception); + EXPECT_THROW(create_datetimev2(8), Exception); + EXPECT_THROW(create_datetimev2(9), Exception); + EXPECT_THROW(create_datetimev2(10), Exception); EXPECT_THROW(DataTypeTimeV2(7), Exception); } @@ -869,4 +872,4 @@ TEST_F(DataTypeDateTimeV2Test, GetFieldWithDataTypeTest) { field_datetime_v2); } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/test/core/data_type/data_type_timestamp_ns_test.cpp b/be/test/core/data_type/data_type_timestamp_ns_test.cpp new file mode 100644 index 00000000000000..2cd70689272ce7 --- /dev/null +++ b/be/test/core/data_type/data_type_timestamp_ns_test.cpp @@ -0,0 +1,686 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/data_type/data_type_timestamp_ns.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "core/assert_cast.h" +#include "core/column/column_const.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type_serde/data_type_serde.h" +#include "core/data_type_serde/data_type_timestamp_ns_serde.h" +#include "core/string_buffer.hpp" +#include "core/value/vdatetime_value.h" +#include "exprs/function/cast/cast_parameters.h" +#include "util/jsonb_utils.h" +#include "util/jsonb_writer.h" +#include "util/mysql_row_buffer.h" +#include "util/slice.h" +#include "util/timezone_utils.h" + +namespace doris { + +TEST(DataTypeTimeStampNsTest, TypeFamilyClassifiersKeepTimestampNsIndependent) { + EXPECT_TRUE(is_date_type(TYPE_TIMESTAMP_NS)); + EXPECT_TRUE(is_timestamp_ns_type(TYPE_TIMESTAMP_NS)); + EXPECT_FALSE(is_timestamp_ns_type(TYPE_DATETIMEV2)); + + EXPECT_TRUE(is_date_v2_or_datetime_v2(TYPE_DATEV2)); + EXPECT_TRUE(is_date_v2_or_datetime_v2(TYPE_DATETIMEV2)); + EXPECT_FALSE(is_date_v2_or_datetime_v2(TYPE_TIMESTAMP_NS)); + + EXPECT_TRUE(is_datetime_v2(TYPE_DATETIMEV2)); + EXPECT_FALSE(is_datetime_v2(TYPE_TIMESTAMP_NS)); + + EXPECT_TRUE(IsDataTypeDateTimeV2); + EXPECT_FALSE(IsDataTypeDateTimeV2); +} + +TEST(DataTypeTimeStampNsTest, Int64EpochRangeAndOrdering) { + const TimeStampNsValue epoch(0); + const TimeStampNsValue before_epoch(-1); + const TimeStampNsValue minimum(std::numeric_limits::min()); + const TimeStampNsValue maximum(std::numeric_limits::max()); + + EXPECT_EQ(epoch.to_string(9), "1970-01-01 00:00:00.000000000"); + EXPECT_EQ(before_epoch.to_string(9), "1969-12-31 23:59:59.999999999"); + EXPECT_EQ(minimum.to_string(9), "1677-09-21 00:12:43.145224192"); + EXPECT_EQ(maximum.to_string(9), "2262-04-11 23:47:16.854775807"); + EXPECT_LT(minimum, before_epoch); + EXPECT_LT(before_epoch, epoch); + EXPECT_LT(epoch, maximum); +} + +TEST(DataTypeTimeStampNsTest, NegativeEpochUsesFloorSecondAndNormalizedFraction) { + struct TestCase { + int64_t epoch_nanos; + int64_t epoch_seconds; + uint32_t nanosecond; + }; + const std::vector cases = { + {-1000000001, -2, 999999999}, + {-1000000000, -1, 0}, + {-999999999, -1, 1}, + {-1, -1, 999999999}, + {0, 0, 0}, + {1, 0, 1}, + }; + + for (const auto& test_case : cases) { + const TimeStampNsValue value(test_case.epoch_nanos); + EXPECT_EQ(value.epoch_seconds(), test_case.epoch_seconds); + EXPECT_EQ(value.nanosecond(), test_case.nanosecond); + EXPECT_EQ( + static_cast<__int128>(value.epoch_seconds()) * TimeStampNsValue::NANOS_PER_SECOND + + value.nanosecond(), + test_case.epoch_nanos); + } +} + +TEST(DataTypeTimeStampNsTest, ParseAtFixedNanosecondPrecision) { + int64_t value = 0; + + ASSERT_TRUE(parse_timestamp_ns(StringRef("1970-01-01 00:00:00.12345675"), &value).ok()); + EXPECT_EQ(TimeStampNsValue(value).to_string(), "1970-01-01 00:00:00.123456750"); + + ASSERT_TRUE(parse_timestamp_ns(StringRef("1969-12-31 23:59:59.999999999"), &value).ok()); + EXPECT_EQ(value, -1); + + ASSERT_TRUE(parse_timestamp_ns(StringRef("1970-01-01 00:00:00.999999995"), &value).ok()); + EXPECT_EQ(TimeStampNsValue(value).to_string(), "1970-01-01 00:00:00.999999995"); + + ASSERT_TRUE(parse_timestamp_ns(StringRef("1970-01-01 00:00:00.9999999995"), &value).ok()); + EXPECT_EQ(TimeStampNsValue(value).to_string(), "1970-01-01 00:00:01.000000000"); +} + +TEST(DataTypeTimeStampNsTest, ParseTimezoneSuffixInSessionTimezone) { + TimezoneUtils::load_timezones_to_cache(); + cctz::time_zone shanghai; + ASSERT_TRUE(cctz::load_time_zone("Asia/Shanghai", &shanghai)); + + int64_t value = 0; + auto status = + parse_timestamp_ns(StringRef("2023-08-17T01:41:18.123456789Z"), &value, &shanghai); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(TimeStampNsValue(value).to_string(9), "2023-08-17 09:41:18.123456789"); + + ASSERT_TRUE(parse_timestamp_ns(StringRef("2023-08-17T01:41:18.123456789America/Los_Angeles"), + &value, &shanghai) + .ok()); + EXPECT_EQ(TimeStampNsValue(value).to_string(9), "2023-08-17 16:41:18.123456789"); + + EXPECT_FALSE( + parse_timestamp_ns(StringRef("1677-09-21T00:12:43.145224192+14:00"), &value, &shanghai) + .ok()); + EXPECT_FALSE( + parse_timestamp_ns(StringRef("2262-04-11T23:47:16.854775807-01:00"), &value, &shanghai) + .ok()); +} + +TEST(DataTypeTimeStampNsTest, ParseAcceptsFractionalWidthsAndRejectsMalformedValues) { + struct ValidCase { + const char* input; + const char* expected; + }; + const std::vector valid_cases = { + {"2024-02-29 12:34:56.1234567", "2024-02-29 12:34:56.123456700"}, + {"2024-02-29 12:34:56.12345678", "2024-02-29 12:34:56.123456780"}, + {"2024-02-29 12:34:56.123456789", "2024-02-29 12:34:56.123456789"}, + {"2024-02-29 12:34:56.1234567894", "2024-02-29 12:34:56.123456789"}, + {"2024-02-29 12:34:56.1234567895", "2024-02-29 12:34:56.123456790"}, + {"2024-02-29 12:34:56", "2024-02-29 12:34:56.000000000"}, + }; + + for (const auto& test_case : valid_cases) { + int64_t value = 0; + ASSERT_TRUE(parse_timestamp_ns(StringRef(test_case.input), &value).ok()) << test_case.input; + EXPECT_EQ(TimeStampNsValue(value).to_string(), test_case.expected); + } + + const std::vector invalid_values = { + "", + "not-a-date", + "2023-02-29 00:00:00.000000000", + "2024-13-01", + "2024-01-01 24:00:00", + "2024-01-01 00:00:00.trailing", + "2024-01-01 00:00:00.123.456", + "2024-01-01.123 00:00:00", + "2024.01-01 00:00:00", + }; + for (const char* input : invalid_values) { + int64_t value = 0; + EXPECT_FALSE(parse_timestamp_ns(StringRef(input), &value).ok()) << input; + } +} + +TEST(DataTypeTimeStampNsTest, RejectValuesOutsideEpochRange) { + int64_t value = 0; + EXPECT_FALSE(parse_timestamp_ns(StringRef("0000-01-01 00:00:00.000000000"), &value).ok()); + EXPECT_FALSE(parse_timestamp_ns(StringRef("1677-09-21 00:12:43.145224191"), &value).ok()); + EXPECT_FALSE(parse_timestamp_ns(StringRef("2262-04-11 23:47:16.854775808"), &value).ok()); + EXPECT_FALSE(parse_timestamp_ns(StringRef("9999-12-31 23:59:59.999999999"), &value).ok()); +} + +TEST(DataTypeTimeStampNsTest, CivilRoundTripPreservesSubMicrosecondDigits) { + DateV2Value civil; + civil.unchecked_set_time(2024, 2, 29, 23, 59, 58, 123456); + + TimeStampNsValue value; + ASSERT_TRUE(value.from_datetime(civil, 789)); + EXPECT_EQ(value.to_string(9), "2024-02-29 23:59:58.123456789"); + const auto round_trip = value.to_datetime(); + EXPECT_EQ(round_trip.year(), 2024); + EXPECT_EQ(round_trip.month(), 2); + EXPECT_EQ(round_trip.day(), 29); + EXPECT_EQ(round_trip.hour(), 23); + EXPECT_EQ(round_trip.minute(), 59); + EXPECT_EQ(round_trip.second(), 58); + EXPECT_EQ(value.microsecond(), 123456); + EXPECT_EQ(value.nanosecond_remainder(), 789); + EXPECT_EQ(round_trip.to_date_int_val(), civil.to_date_int_val()); +} + +TEST(DataTypeTimeStampNsTest, FactoryKeepsTimestampNsSeparateFromDateTimeV2) { + const auto microseconds = create_datetimev2(6); + const auto timestamp_ns = std::make_shared(); + + EXPECT_EQ(microseconds->get_primitive_type(), TYPE_DATETIMEV2); + EXPECT_THROW(create_datetimev2(7), Exception); + EXPECT_THROW(create_datetimev2(8), Exception); + EXPECT_THROW(create_datetimev2(9), Exception); + EXPECT_EQ(timestamp_ns->get_primitive_type(), TYPE_TIMESTAMP_NS); + EXPECT_EQ(timestamp_ns->get_storage_field_type(), FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS); + EXPECT_EQ(timestamp_ns->get_scale(), 9); + EXPECT_EQ(microseconds->get_family_name(), "DateTimeV2"); + EXPECT_EQ(timestamp_ns->get_family_name(), "TimeStampNs"); +} + +TEST(DataTypeTimeStampNsTest, SerDeRoundTripsTextProtobufAndBinary) { + const auto type = std::make_shared(); + const auto serde = type->get_serde(); + auto source = type->create_column(); + DataTypeSerDe::FormatOptions options; + const std::vector inputs = { + "1677-09-21 00:12:43.145224192", "1969-12-31 23:59:59.999999999", + "1970-01-01 00:00:00.000000000", "2024-02-29 12:34:56.123456789", + "2262-04-11 23:47:16.854775807", + }; + for (const auto& input : inputs) { + StringRef ref(input); + ASSERT_TRUE(serde->from_string(ref, *source, options).ok()) << input; + } + const auto& source_data = assert_cast(*source).get_data(); + + PValues protobuf_values; + ASSERT_TRUE(serde->write_column_to_pb(*source, protobuf_values, 0, source->size()).ok()); + auto protobuf_result = type->create_column(); + ASSERT_TRUE(serde->read_column_from_pb(*protobuf_result, protobuf_values).ok()); + const auto& protobuf_data = assert_cast(*protobuf_result).get_data(); + EXPECT_EQ(protobuf_data, source_data); + + ColumnString::Chars binary; + std::vector offsets = {0}; + for (size_t row = 0; row < source->size(); ++row) { + serde->write_one_cell_to_binary(*source, binary, row); + offsets.push_back(binary.size()); + } + constexpr size_t bytes_per_row = sizeof(uint8_t) + sizeof(uint8_t) + sizeof(int64_t); + ASSERT_EQ(binary.size(), source->size() * bytes_per_row); + auto binary_result = ColumnNullable::create(type->create_column(), ColumnUInt8::create()); + for (size_t row = 0; row < source->size(); ++row) { + const uint8_t* begin = binary.data() + offsets[row]; + const uint8_t* end = DataTypeSerDe::deserialize_binary_to_column(begin, *binary_result); + EXPECT_EQ(end - begin, bytes_per_row); + } + const auto& binary_data = + assert_cast(binary_result->get_nested_column()).get_data(); + EXPECT_EQ(binary_data, source_data); +} + +/* +TEST(DataTypeTimeStampNsTest, SerDeRoundTripsArrowNanosecondsBeforeEpoch) { + const auto type = std::make_shared(); + const auto serde = type->get_serde(); + auto source = type->create_column(); + auto& source_data = assert_cast(*source).get_data(); + source_data.push_back(TimeStampNsValue(-1)); + source_data.push_back(TimeStampNsValue(0)); + source_data.push_back(TimeStampNsValue(1234567890)); + + arrow::TimestampBuilder builder(arrow::timestamp(arrow::TimeUnit::NANO), + arrow::default_memory_pool()); + ASSERT_TRUE(serde->write_column_to_arrow(*source, nullptr, &builder, 0, source->size(), + cctz::utc_time_zone()) + .ok()); + std::shared_ptr array; + ASSERT_TRUE(builder.Finish(&array).ok()); + const auto* timestamps = assert_cast(array.get()); + EXPECT_EQ(timestamps->Value(0), -1); + EXPECT_EQ(timestamps->Value(1), 0); + EXPECT_EQ(timestamps->Value(2), 1234567890); + + auto result = type->create_column(); + ASSERT_TRUE(serde->read_column_from_arrow(*result, array.get(), 0, array->length(), + cctz::utc_time_zone()) + .ok()); + EXPECT_EQ(assert_cast(*result).get_data(), source_data); +} +*/ + +TEST(DataTypeTimeStampNsTest, DecodedTimestampUnitsAndValidation) { + const auto type = std::make_shared(); + const auto serde = type->get_serde(); + const int64_t values[] = {-1, 0, 1}; + + const auto check_unit = [&](DecodedTimeUnit unit, int64_t multiplier) { + auto column = type->create_column(); + DecodedColumnView view; + view.value_kind = DecodedValueKind::INT64; + view.time_unit = unit; + view.row_count = 3; + view.values = reinterpret_cast(values); + ASSERT_TRUE(serde->read_column_from_decoded_values(*column, view).ok()); + const auto& data = assert_cast(*column).get_data(); + ASSERT_EQ(data.size(), 3); + EXPECT_EQ(data[0].epoch_nanos(), -multiplier); + EXPECT_EQ(data[1].epoch_nanos(), 0); + EXPECT_EQ(data[2].epoch_nanos(), multiplier); + }; + + check_unit(DecodedTimeUnit::MILLIS, TimeStampNsValue::NANOS_PER_MILLISECOND); + check_unit(DecodedTimeUnit::MICROS, TimeStampNsValue::NANOS_PER_MICROSECOND); + check_unit(DecodedTimeUnit::NANOS, 1); + + { + auto column = type->create_column(); + DecodedColumnView view; + view.value_kind = DecodedValueKind::INT64; + view.time_unit = DecodedTimeUnit::UNKNOWN; + view.row_count = 3; + view.values = reinterpret_cast(values); + EXPECT_FALSE(serde->read_column_from_decoded_values(*column, view).ok()); + } + + { + const int64_t overflow = std::numeric_limits::max(); + auto column = type->create_column(); + DecodedColumnView view; + view.value_kind = DecodedValueKind::INT64; + view.time_unit = DecodedTimeUnit::MILLIS; + view.row_count = 1; + view.values = reinterpret_cast(&overflow); + EXPECT_FALSE(serde->read_column_from_decoded_values(*column, view).ok()); + } +} + +TEST(DataTypeTimeStampNsTest, DataTypeLiteralField) { + const DataTypeTimeStampNs type; + const DataTypeDateTimeV2 legacy6(6); + + EXPECT_TRUE(type.equals(DataTypeTimeStampNs {})); + EXPECT_FALSE(type.equals(legacy6)); + EXPECT_FALSE(type.equals_ignore_precision(legacy6)); + EXPECT_FALSE(legacy6.equals_ignore_precision(type)); + + TExprNode node; + node.date_literal.value = "2024-02-29 12:34:56.123456789"; + const Field field = type.get_field(node); + EXPECT_EQ(field.get().to_string(9), "2024-02-29 12:34:56.123456789"); + node.date_literal.value = "not-a-datetime"; + EXPECT_THROW(type.get_field(node), Exception); + + auto column = type.create_column(); + column->insert(field); + const auto field_with_type = type.get_field_with_data_type(*column, 0); + EXPECT_EQ(field_with_type.field, field); + EXPECT_EQ(field_with_type.base_scalar_type_id, TYPE_TIMESTAMP_NS); + EXPECT_EQ(field_with_type.precision, -1); + EXPECT_EQ(field_with_type.scale, 9); +} + +TEST(DataTypeTimeStampNsTest, FormattingAndHash) { + int64_t raw = 0; + ASSERT_TRUE(parse_timestamp_ns(StringRef("2024-02-29 12:34:56.123456789"), &raw).ok()); + TimeStampNsValue value(raw); + + EXPECT_TRUE(value.is_valid_date()); + + char text[40] = {}; + const char* end = value.to_string(text, 9); + EXPECT_STREQ(text, "2024-02-29 12:34:56.123456789"); + EXPECT_EQ(end, text + std::strlen(text) + 1); + + EXPECT_EQ(value.hash(17), value.hash(17)); + EXPECT_EQ(std::hash {}(value), std::hash {}(value.epoch_nanos())); +} + +TEST(DataTypeTimeStampNsTest, SerDeStrictBatchJsonJsonbMysqlAndBinaryField) { + const DataTypeTimeStampNs type; + const auto serde = type.get_serde(); + DataTypeSerDe::FormatOptions options; + options.field_delim = ";"; + + auto strings = ColumnString::create(); + strings->insert_data("1970-01-01 00:00:00.000000001", 29); + strings->insert_data("ignored-invalid-value", 21); + strings->insert_data("2024-02-29 12:34:56.123456789", 29); + NullMap null_map = {0, 1, 0}; + auto strict_result = type.create_column(); + ASSERT_TRUE( + serde->from_string_strict_mode_batch(*strings, *strict_result, options, null_map.data()) + .ok()); + const auto& strict_data = assert_cast(*strict_result).get_data(); + EXPECT_EQ(strict_data[0].epoch_nanos(), 1); + EXPECT_EQ(strict_data[2].to_string(9), "2024-02-29 12:34:56.123456789"); + + auto invalid_strings = ColumnString::create(); + invalid_strings->insert_data("invalid", 7); + auto invalid_result = type.create_column(); + EXPECT_FALSE(serde->from_string_strict_mode_batch(*invalid_strings, *invalid_result, options, + nullptr) + .ok()); + + auto source = type.create_column(); + for (const std::string input : + {"1970-01-01 00:00:00.000000001", "2024-02-29 12:34:56.123456789"}) { + StringRef ref(input); + ASSERT_TRUE(serde->from_string_strict_mode(ref, *source, options).ok()); + } + + auto serialized = ColumnString::create(); + VectorBufferWriter writer(*serialized); + ASSERT_TRUE(serde->serialize_column_to_json(*source, 0, source->size(), writer, options).ok()); + writer.commit(); + EXPECT_EQ(serialized->get_data_at(0).to_string(), + "1970-01-01 00:00:00.000000001;2024-02-29 12:34:56.123456789"); + + std::vector json_values = {"1970-01-01 00:00:00.000000001", + "2024-02-29 12:34:56.123456789"}; + std::vector slices; + for (auto& json_value : json_values) { + slices.emplace_back(json_value.data(), json_value.size()); + } + auto json_result = type.create_column(); + uint64_t num_deserialized = 0; + ASSERT_TRUE(serde->deserialize_column_from_json_vector(*json_result, slices, &num_deserialized, + options) + .ok()); + EXPECT_EQ(num_deserialized, json_values.size()); + EXPECT_EQ(assert_cast(*json_result).get_data(), + assert_cast(*source).get_data()); + + const auto nested_serde = type.get_serde(2); + auto nested_json = ColumnString::create(); + VectorBufferWriter nested_writer(*nested_json); + ASSERT_TRUE(nested_serde->serialize_one_cell_to_json(*source, 1, nested_writer, options).ok()); + nested_writer.commit(); + EXPECT_EQ(nested_json->get_data_at(0).to_string(), "\"2024-02-29 12:34:56.123456789\""); + auto nested_result = type.create_column(); + std::string quoted = nested_json->get_data_at(0).to_string(); + Slice quoted_slice(quoted.data(), quoted.size()); + ASSERT_TRUE(nested_serde->deserialize_one_cell_from_json(*nested_result, quoted_slice, options) + .ok()); + EXPECT_EQ(assert_cast(*nested_result).get_element(0), + assert_cast(*source).get_element(1)); + + auto one_value = source->clone_resized(1); + auto const_column = ColumnConst::create(std::move(one_value), 2); + auto const_json = ColumnString::create(); + VectorBufferWriter const_writer(*const_json); + ASSERT_TRUE(serde->serialize_one_cell_to_json(*const_column, 1, const_writer, options).ok()); + const_writer.commit(); + EXPECT_EQ(const_json->get_data_at(0).to_string(), "1970-01-01 00:00:00.000000001"); + + JsonbWriter jsonb_writer; + ASSERT_TRUE(serde->serialize_column_to_jsonb(*source, 1, jsonb_writer).ok()); + EXPECT_EQ(JsonbToJson::jsonb_to_json_string(jsonb_writer.getOutput()->getBuffer(), + jsonb_writer.getOutput()->getSize()), + "\"2024-02-29 12:34:56.123456789\""); + CastParameters cast_params {.status = Status::OK(), .is_strict = true}; + auto jsonb_result = type.create_column(); + ASSERT_TRUE(serde->deserialize_column_from_jsonb(*jsonb_result, jsonb_writer.getValue(), + cast_params) + .ok()); + EXPECT_EQ(assert_cast(*jsonb_result).get_element(0), + assert_cast(*source).get_element(1)); + + auto jsonb_values = ColumnString::create(); + ASSERT_TRUE(serde->serialize_column_to_jsonb_vector(*source, *jsonb_values).ok()); + auto jsonb_vector_result = ColumnNullable::create(type.create_column(), ColumnUInt8::create()); + ASSERT_TRUE(serde->deserialize_column_from_jsonb_vector(*jsonb_vector_result, *jsonb_values, + cast_params) + .ok()); + EXPECT_EQ(assert_cast(jsonb_vector_result->get_nested_column()) + .get_data(), + assert_cast(*source).get_data()); + + JsonbWriter row_store_writer; + Arena row_store_arena; + row_store_writer.writeStartObject(); + serde->write_one_cell_to_jsonb(*source, row_store_writer, row_store_arena, 0, 1, options); + row_store_writer.writeEndObject(); + const JsonbDocument* row_store_document = nullptr; + ASSERT_TRUE(JsonbDocument::checkAndCreateDocument(row_store_writer.getOutput()->getBuffer(), + row_store_writer.getOutput()->getSize(), + &row_store_document) + .ok()); + auto row_store_result = type.create_column(); + serde->read_one_cell_from_jsonb(*row_store_result, (*row_store_document)->begin()->value()); + EXPECT_EQ(assert_cast(*row_store_result).get_element(0), + assert_cast(*source).get_element(1)); + + MysqlRowBinaryBuffer mysql_buffer; + ASSERT_TRUE(serde->write_column_to_mysql_binary(*source, mysql_buffer, 1, false, options).ok()); + ASSERT_EQ(static_cast(mysql_buffer.buf()[0]), 29); + EXPECT_EQ(std::string(mysql_buffer.buf() + 1, 29), "2024-02-29 12:34:56.123456789"); + + ColumnString::Chars binary; + serde->write_one_cell_to_binary(*source, binary, 1); + Field binary_field; + FieldInfo info; + const uint8_t* end = + DataTypeSerDe::deserialize_binary_to_field(binary.data(), binary_field, info); + EXPECT_EQ(end, binary.data() + binary.size()); + EXPECT_EQ(info.scalar_type_id, TYPE_TIMESTAMP_NS); + EXPECT_EQ(info.scale, 9); + EXPECT_EQ(binary_field.get().to_string(9), "2024-02-29 12:34:56.123456789"); +} + +TEST(DataTypeTimeStampNsTest, SerDeBatchRejectsMultipleFractionalSeparators) { + const DataTypeTimeStampNs type; + const auto serde = type.get_serde(); + DataTypeSerDe::FormatOptions options; + + auto strings = ColumnString::create(); + strings->insert_data("2024-01-01 00:00:00.123456789", 29); + strings->insert_data("2024-01-01 00:00:00.123.456", 27); + + auto permissive_result = ColumnNullable::create(type.create_column(), ColumnUInt8::create()); + ASSERT_TRUE(serde->from_string_batch(*strings, *permissive_result, options).ok()); + const auto& permissive_null_map = permissive_result->get_null_map_data(); + ASSERT_EQ(permissive_null_map.size(), 2); + EXPECT_EQ(permissive_null_map[0], 0); + EXPECT_EQ(permissive_null_map[1], 1); + + auto invalid_string = ColumnString::create(); + invalid_string->insert_data("2024-01-01 00:00:00.123.456", 27); + auto strict_result = type.create_column(); + EXPECT_FALSE( + serde->from_string_strict_mode_batch(*invalid_string, *strict_result, options, nullptr) + .ok()); +} + +/* +TEST(DataTypeTimeStampNsTest, SerDeArrowUnitsNullTimezoneAndErrors) { + TimezoneUtils::load_timezones_to_cache(); + cctz::time_zone shanghai; + ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("Asia/Shanghai", shanghai)); + + const DataTypeTimeStampNs type; + const auto serde = type.get_serde(); + auto source = type.create_column(); + DataTypeSerDe::FormatOptions options; + for (const std::string input : + {"1970-01-01 08:00:01.000000000", "1970-01-01 08:00:02.000000000"}) { + StringRef ref(input); + ASSERT_TRUE(serde->from_string(ref, *source, options).ok()); + } + const NullMap null_map = {0, 1}; + + const std::array, 4> units = { + std::pair {arrow::TimeUnit::SECOND, 1LL}, + std::pair {arrow::TimeUnit::MILLI, 1000LL}, + std::pair {arrow::TimeUnit::MICRO, 1000000LL}, + std::pair {arrow::TimeUnit::NANO, 1000000000LL}, + }; + for (const auto& [unit, multiplier] : units) { + arrow::TimestampBuilder builder(arrow::timestamp(unit, "Asia/Shanghai"), + arrow::default_memory_pool()); + ASSERT_TRUE(serde->write_column_to_arrow(*source, &null_map, &builder, 0, source->size(), + shanghai) + .ok()); + std::shared_ptr array; + ASSERT_TRUE(builder.Finish(&array).ok()); + const auto& timestamps = assert_cast(*array); + EXPECT_EQ(timestamps.Value(0), multiplier); + EXPECT_TRUE(timestamps.IsNull(1)); + + auto result = type.create_column(); + ASSERT_TRUE(serde->read_column_from_arrow(*result, array.get(), 0, 1, shanghai).ok()); + EXPECT_EQ(assert_cast(*result).get_element(0), + assert_cast(*source).get_element(0)); + } + + arrow::Int64Builder wrong_builder; + ASSERT_TRUE(wrong_builder.Append(1).ok()); + std::shared_ptr wrong_array; + ASSERT_TRUE(wrong_builder.Finish(&wrong_array).ok()); + auto wrong_result = type.create_column(); + EXPECT_FALSE( + serde->read_column_from_arrow(*wrong_result, wrong_array.get(), 0, 1, shanghai).ok()); + + for (const auto unit : + {arrow::TimeUnit::SECOND, arrow::TimeUnit::MILLI, arrow::TimeUnit::MICRO}) { + arrow::TimestampBuilder overflow_builder(arrow::timestamp(unit), + arrow::default_memory_pool()); + ASSERT_TRUE(overflow_builder.Append(std::numeric_limits::max()).ok()); + std::shared_ptr overflow_array; + ASSERT_TRUE(overflow_builder.Finish(&overflow_array).ok()); + auto overflow_result = type.create_column(); + EXPECT_FALSE(serde->read_column_from_arrow(*overflow_result, overflow_array.get(), 0, 1, + cctz::utc_time_zone()) + .ok()); + } + + arrow::TimestampBuilder boundary_builder( + arrow::timestamp(arrow::TimeUnit::NANO, "Asia/Shanghai"), arrow::default_memory_pool()); + ASSERT_TRUE(boundary_builder.Append(std::numeric_limits::max()).ok()); + std::shared_ptr boundary_array; + ASSERT_TRUE(boundary_builder.Finish(&boundary_array).ok()); + auto boundary_result = type.create_column(); + EXPECT_FALSE( + serde->read_column_from_arrow(*boundary_result, boundary_array.get(), 0, 1, shanghai) + .ok()); + + auto minimum = type.create_column(); + assert_cast(*minimum).insert_value( + TimeStampNsValue(std::numeric_limits::min())); + arrow::TimestampBuilder minimum_builder( + arrow::timestamp(arrow::TimeUnit::NANO, "Asia/Shanghai"), arrow::default_memory_pool()); + EXPECT_FALSE( + serde->write_column_to_arrow(*minimum, nullptr, &minimum_builder, 0, 1, shanghai).ok()); +} + +TEST(DataTypeTimeStampNsTest, SerDeDecodedNullTimezoneAndOrc) { + TimezoneUtils::load_timezones_to_cache(); + cctz::time_zone shanghai; + ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("Asia/Shanghai", shanghai)); + + const DataTypeTimeStampNs type; + const auto serde = type.get_serde(); + const int64_t values[] = {0, 1}; + const NullMap null_map = {0, 1}; + DecodedColumnView view; + view.value_kind = DecodedValueKind::INT64; + view.time_unit = DecodedTimeUnit::NANOS; + view.row_count = 2; + view.values = reinterpret_cast(values); + view.null_map = null_map.data(); + view.timestamp_is_adjusted_to_utc = true; + view.timezone = &shanghai; + auto decoded = type.create_column(); + ASSERT_TRUE(serde->read_column_from_decoded_values(*decoded, view).ok()); + const auto& decoded_data = assert_cast(*decoded).get_data(); + EXPECT_EQ(decoded_data[0].to_string(9), "1970-01-01 08:00:00.000000000"); + EXPECT_EQ(decoded_data[1].epoch_nanos(), 0); + + view.value_kind = DecodedValueKind::INT32; + auto invalid_kind = type.create_column(); + EXPECT_FALSE(serde->read_column_from_decoded_values(*invalid_kind, view).ok()); + + auto source = type.create_column(); + int64_t source_raw = 0; + ASSERT_TRUE( + parse_timestamp_ns(StringRef("1970-01-01 08:00:01.123456789"), &source_raw).ok()); + assert_cast(*source).insert_value(TimeStampNsValue(source_raw)); + assert_cast(*source).insert_value(TimeStampNsValue(0)); + + Arena arena; + DataTypeSerDe::FormatOptions options; + orc::TimestampVectorBatch batch(2, *orc::getDefaultPool()); + batch.resize(2); + batch.notNull[0] = 1; + batch.notNull[1] = 0; + batch.hasNulls = true; + ASSERT_TRUE(serde->write_column_to_orc("Asia/Shanghai", *source, nullptr, &batch, 0, 2, arena, + options) + .ok()); + EXPECT_EQ(batch.numElements, 2); + EXPECT_EQ(batch.data[0], 1); + EXPECT_EQ(batch.nanoseconds[0], 123456789); + EXPECT_EQ(batch.data[1], 0); + EXPECT_FALSE(serde->write_column_to_orc("invalid/timezone", *source, nullptr, &batch, 0, 2, + arena, options) + .ok()); + + auto minimum = type.create_column(); + assert_cast(*minimum).insert_value( + TimeStampNsValue(std::numeric_limits::min())); + orc::TimestampVectorBatch minimum_batch(1, *orc::getDefaultPool()); + minimum_batch.resize(1); + minimum_batch.notNull[0] = 1; + EXPECT_FALSE(serde->write_column_to_orc("Asia/Shanghai", *minimum, nullptr, &minimum_batch, 0, + 1, arena, options) + .ok()); +} +*/ + +} // namespace doris diff --git a/be/test/core/data_type/storage_field_type_test.cpp b/be/test/core/data_type/storage_field_type_test.cpp index af621ceb9c6abe..f2f40f2a265429 100644 --- a/be/test/core/data_type/storage_field_type_test.cpp +++ b/be/test/core/data_type/storage_field_type_test.cpp @@ -72,6 +72,7 @@ constexpr std::array supported_mappings { TypePair {PrimitiveType::TYPE_UINT32, FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT}, TypePair {PrimitiveType::TYPE_UINT64, FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT}, TypePair {PrimitiveType::TYPE_TIMESTAMPTZ, FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ}, + TypePair {PrimitiveType::TYPE_TIMESTAMP_NS, FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS}, }; TEST(StorageFieldTypeTest, SupportedMappingsRoundTrip) { @@ -90,7 +91,6 @@ TEST(StorageFieldTypeTest, UnsupportedPrimitiveTypesThrow) { static_cast(33), // TYPE_LAMBDA_FUNCTION (deprecated) PrimitiveType::TYPE_FIXED_LENGTH_OBJECT, PrimitiveType::TYPE_VARBINARY, - static_cast(43), static_cast(255), }; @@ -107,7 +107,6 @@ TEST(StorageFieldTypeTest, UnsupportedOrInvalidFieldTypesThrow) { FieldType::OLAP_FIELD_TYPE_DISCRETE_DOUBLE, static_cast(-1), static_cast(0), - static_cast(41), static_cast(255), }; @@ -159,6 +158,7 @@ TEST(StorageFieldTypeTest, PersistedFieldTypeValuesStayStable) { FieldType::OLAP_FIELD_TYPE_IPV4, FieldType::OLAP_FIELD_TYPE_IPV6, FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, + FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS, }; for (size_t i = 0; i < persisted_types.size(); ++i) { diff --git a/be/test/core/data_type_serde/data_type_serde_decoded_values_test.cpp b/be/test/core/data_type_serde/data_type_serde_decoded_values_test.cpp index 18e4aa8a1cf848..1303fa1e29a032 100644 --- a/be/test/core/data_type_serde/data_type_serde_decoded_values_test.cpp +++ b/be/test/core/data_type_serde/data_type_serde_decoded_values_test.cpp @@ -830,7 +830,7 @@ TEST(DataTypeSerDeDecodedValuesTest, ReadDateTimeV2Millis) { "1970-01-01 00:00:00.001000", "1970-01-01 00:00:01.234000"}); } -TEST(DataTypeSerDeDecodedValuesTest, ReadDateTimeV2Nanos) { +TEST(DataTypeSerDeDecodedValuesTest, ReadTimeStampNss) { auto type = std::make_shared(6); std::vector values = {-1000, 0, 1000, 1234567890}; auto view = make_fixed_view(DecodedValueKind::INT64, values); diff --git a/be/test/exec/sink/tablet_sink_hash_partitioner_test.cpp b/be/test/exec/sink/tablet_sink_hash_partitioner_test.cpp index ba587b3c3ba89b..e1e4e7b7716f71 100644 --- a/be/test/exec/sink/tablet_sink_hash_partitioner_test.cpp +++ b/be/test/exec/sink/tablet_sink_hash_partitioner_test.cpp @@ -36,6 +36,7 @@ #include "common/config.h" #include "core/assert_cast.h" #include "core/column/column_vector.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "exec/operator/exchange_sink_operator.h" @@ -59,6 +60,69 @@ using doris::ExchangeSinkLocalState; using doris::ExchangeSinkOperatorX; using doris::OperatorContext; +TExprNode _make_timestamp_ns_literal(std::string value) { + TExprNode node; + node.__set_node_type(TExprNodeType::DATE_LITERAL); + node.__set_num_children(0); + + TDateLiteral literal; + literal.__set_value(std::move(value)); + node.__set_date_literal(literal); + + TTypeDesc type_desc = create_type_desc(PrimitiveType::TYPE_TIMESTAMP_NS); + type_desc.__set_is_nullable(false); + node.__set_type(type_desc); + node.__set_is_nullable(false); + return node; +} + +void _build_timestamp_ns_schema(OperatorContext& ctx, TOlapTableSchemaParam& tschema, + int64_t& schema_index_id) { + TTypeDesc type_desc = create_type_desc(PrimitiveType::TYPE_TIMESTAMP_NS); + + TDescriptorTableBuilder dtb; + TTupleDescriptorBuilder tuple_builder; + tuple_builder.add_slot(TSlotDescriptorBuilder() + .set_slotType(type_desc) + .nullable(true) + .column_name("dt") + .column_pos(0) + .build()); + tuple_builder.build(&dtb); + + auto thrift_desc_tbl = dtb.desc_tbl(); + DescriptorTbl* desc_tbl = nullptr; + auto st = DescriptorTbl::create(ctx.state.obj_pool(), thrift_desc_tbl, &desc_tbl); + ASSERT_TRUE(st.ok()) << st.to_string(); + ctx.state.set_desc_tbl(desc_tbl); + + tschema.db_id = 1; + tschema.table_id = 2; + tschema.version = 0; + tschema.slot_descs = thrift_desc_tbl.slotDescriptors; + tschema.tuple_desc = thrift_desc_tbl.tupleDescriptors.front(); + + TOlapTableIndexSchema index_schema; + index_schema.id = 10; + index_schema.columns = {"dt"}; + index_schema.schema_hash = 123; + tschema.indexes = {index_schema}; + schema_index_id = index_schema.id; +} + +TOlapTablePartition _make_timestamp_ns_partition(int64_t id, int64_t schema_index_id) { + TOlapTablePartition partition; + partition.id = id; + partition.num_buckets = 1; + partition.__set_is_mutable(true); + + TOlapTableIndexTablets index_tablets; + index_tablets.index_id = schema_index_id; + index_tablets.tablets = {id * 100}; + partition.indexes = {index_tablets}; + return partition; +} + std::shared_ptr _create_parent_operator( OperatorContext& ctx, const std::shared_ptr& row_desc_holder) { TDataStreamSink sink; @@ -275,5 +339,50 @@ TEST(TabletSinkHashPartitionerTest, OlapTabletFinderRoundRobinEveryBatch) { EXPECT_EQ(tablet_index[0], 0); } } + +TEST(TabletSinkHashPartitionerTest, TimeStampNsRangePartitionKey) { + OperatorContext ctx; + + TOlapTableSchemaParam tschema; + int64_t schema_index_id = 0; + _build_timestamp_ns_schema(ctx, tschema, schema_index_id); + + auto schema = std::make_shared(); + auto st = schema->init(tschema); + ASSERT_TRUE(st.ok()) << st.to_string(); + + TOlapTablePartitionParam partition_param; + partition_param.db_id = 1; + partition_param.table_id = 2; + partition_param.version = 0; + partition_param.__set_partition_type(TPartitionType::RANGE_PARTITIONED); + partition_param.__set_partition_columns({"dt"}); + + auto before_epoch = _make_timestamp_ns_partition(1, schema_index_id); + before_epoch.__set_end_keys({_make_timestamp_ns_literal("1970-01-01 00:00:00")}); + + auto epoch = _make_timestamp_ns_partition(2, schema_index_id); + epoch.__set_start_keys({_make_timestamp_ns_literal("1970-01-01 00:00:00")}); + epoch.__set_end_keys({_make_timestamp_ns_literal("1970-01-01 00:00:00.000000002")}); + + auto after_epoch = _make_timestamp_ns_partition(3, schema_index_id); + after_epoch.__set_start_keys({_make_timestamp_ns_literal("1970-01-01 00:00:00.000000002")}); + partition_param.partitions = {epoch, before_epoch, after_epoch}; + + VOlapTablePartitionParam partitions(schema, partition_param); + st = partitions.init(); + ASSERT_TRUE(st.ok()) << st.to_string(); + + auto* slot = schema->tuple_desc()->slots().front(); + auto column = slot->get_empty_mutable_column(); + const TimeStampNsValue min_value(std::numeric_limits::min()); + column->insert_data(reinterpret_cast(&min_value), 0); + Block block({{std::move(column), slot->get_data_type_ptr(), slot->col_name()}}); + + VOlapTablePartition* result = nullptr; + EXPECT_TRUE(partitions.find_partition(&block, 0, result)); + ASSERT_NE(result, nullptr); + EXPECT_EQ(result->id, before_epoch.id); +} } // anonymous namespace } // namespace doris diff --git a/be/test/exprs/aggregate/agg_approx_count_distinct_test.cpp b/be/test/exprs/aggregate/agg_approx_count_distinct_test.cpp new file mode 100644 index 00000000000000..1f8c3f4310fbc9 --- /dev/null +++ b/be/test/exprs/aggregate/agg_approx_count_distinct_test.cpp @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include + +#include "core/column/column_vector.h" +#include "core/data_type/data_type_timestamp_ns.h" +#include "exprs/aggregate/aggregate_function.h" +#include "exprs/aggregate/aggregate_function_simple_factory.h" + +namespace doris { + +void register_aggregate_function_approx_count_distinct(AggregateFunctionSimpleFactory& factory); + +TEST(AggApproxCountDistinctTest, TimeStampNs) { + AggregateFunctionSimpleFactory factory; + register_aggregate_function_approx_count_distinct(factory); + + auto data_type = std::make_shared(); + const DataTypes argument_types = {data_type}; + for (const std::string function_name : {"approx_count_distinct", "ndv"}) { + auto function = factory.get(function_name, argument_types, nullptr, false, -1); + ASSERT_NE(function, nullptr); + + auto column = ColumnTimeStampNs::create(); + for (const int64_t epoch_nanos : {-1, 0, 1, 1, 1'000'000'001}) { + column->insert_value(TimeStampNsValue(epoch_nanos)); + } + + std::unique_ptr memory(new char[function->size_of_data()]); + AggregateDataPtr place = memory.get(); + function->create(place); + Arena arena; + const IColumn* columns[] = {column.get()}; + for (size_t row = 0; row < column->size(); ++row) { + function->add(place, columns, row, arena); + } + + auto result = ColumnInt64::create(); + function->insert_result_into(place, *result); + EXPECT_EQ(result->get_element(0), 4); + function->destroy(place); + } +} + +} // namespace doris diff --git a/be/test/exprs/aggregate/agg_min_max_test.cpp b/be/test/exprs/aggregate/agg_min_max_test.cpp index 3f2080fc8304e6..86b19d7462bad8 100644 --- a/be/test/exprs/aggregate/agg_min_max_test.cpp +++ b/be/test/exprs/aggregate/agg_min_max_test.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -34,6 +35,7 @@ #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "core/field.h" #include "core/string_ref.h" #include "core/types.h" @@ -135,6 +137,41 @@ TEST_P(AggMinMaxTest, min_max_decimal_test) { } } +TEST_P(AggMinMaxTest, min_max_timestamp_ns_test) { + Arena arena; + const std::string min_max_type = GetParam(); + const std::vector epoch_nanos = {0, -1, std::numeric_limits::max(), + std::numeric_limits::min(), 1}; + + auto data_type = std::make_shared(); + auto column = ColumnTimeStampNs::create(); + for (const int64_t value : epoch_nanos) { + column->insert_value(TimeStampNsValue(value)); + } + + AggregateFunctionSimpleFactory factory; + register_aggregate_function_minmax(factory); + DataTypes data_types = {data_type}; + auto agg_function = factory.get(min_max_type, data_types, data_type, false, -1); + ASSERT_NE(agg_function, nullptr); + + std::unique_ptr memory(new char[agg_function->size_of_data()]); + AggregateDataPtr place = memory.get(); + agg_function->create(place); + + const IColumn* columns[1] = {column.get()}; + for (size_t row = 0; row < epoch_nanos.size(); ++row) { + agg_function->add(place, columns, row, arena); + } + + auto result = ColumnTimeStampNs::create(); + agg_function->insert_result_into(place, *result); + const int64_t expected = min_max_type == "min" ? std::numeric_limits::min() + : std::numeric_limits::max(); + EXPECT_EQ(result->get_element(0).epoch_nanos(), expected); + agg_function->destroy(place); +} + TEST_P(AggMinMaxTest, min_max_string_test) { Arena arena; std::string min_max_type = GetParam(); diff --git a/be/test/exprs/vcondition_expr_test.cpp b/be/test/exprs/vcondition_expr_test.cpp index b26f01cc2664c5..83b5ea26aa8fd9 100644 --- a/be/test/exprs/vcondition_expr_test.cpp +++ b/be/test/exprs/vcondition_expr_test.cpp @@ -29,8 +29,10 @@ #include "core/column/column_nullable.h" #include "core/column/column_vector.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" @@ -38,7 +40,7 @@ namespace doris { // Build a minimal TExprNode as the input of VectorizedCoalesceExpr. // Only fields required by the VExpr base ctor (so that create_data_type works) are set. -static TExprNode make_coalesce_node(TPrimitiveType::type ptype, bool is_nullable) { +static TExprNode make_coalesce_node(TPrimitiveType::type ptype, bool is_nullable, int scale = -1) { TExprNode node; node.node_type = TExprNodeType::FUNCTION_CALL; node.num_children = 0; @@ -49,6 +51,9 @@ static TExprNode make_coalesce_node(TPrimitiveType::type ptype, bool is_nullable type_node.type = TTypeNodeType::SCALAR; TScalarType scalar_type; scalar_type.__set_type(ptype); + if (scale >= 0) { + scalar_type.__set_scale(scale); + } type_node.__set_scalar_type(scalar_type); type_desc.types.push_back(type_node); node.__set_type(type_desc); @@ -355,4 +360,53 @@ TEST_F(VConditionExprCoalesceTest, Int32_NormalPathStillWorks) { EXPECT_FALSE(is_null); } +TEST_F(VConditionExprCoalesceTest, TimeStampNs) { + auto coalesce_node = + make_coalesce_node(TPrimitiveType::TIMESTAMP_NS, /*is_nullable=*/true, /*scale=*/9); + auto coalesce_expr = VectorizedCoalesceExpr::create_shared(coalesce_node); + auto nano_type = std::make_shared(); + coalesce_expr->data_type() = std::make_shared(nano_type); + + auto nested0 = ColumnTimeStampNs::create(); + nested0->insert_value(TimeStampNsValue(11)); + nested0->insert_value(TimeStampNsValue(0)); + nested0->insert_value(TimeStampNsValue(-17)); + nested0->insert_value(TimeStampNsValue(0)); + auto null_map0 = ColumnUInt8::create(); + null_map0->insert_value(0); + null_map0->insert_value(1); + null_map0->insert_value(0); + null_map0->insert_value(1); + ColumnPtr col0 = ColumnNullable::create(std::move(nested0), std::move(null_map0)); + + auto col1 = ColumnTimeStampNs::create(); + col1->insert_value(TimeStampNsValue(99)); + col1->insert_value(TimeStampNsValue(22)); + col1->insert_value(TimeStampNsValue(std::numeric_limits::max())); + col1->insert_value(TimeStampNsValue(std::numeric_limits::min())); + + coalesce_expr->add_child( + std::make_shared(col0, std::make_shared(nano_type))); + coalesce_expr->add_child(std::make_shared(std::move(col1), nano_type)); + + VExprContext context(coalesce_expr); + ColumnPtr result; + auto st = coalesce_expr->execute_column_impl(&context, /*block=*/nullptr, + /*selector=*/nullptr, /*count=*/4, result); + ASSERT_TRUE(st.ok()) << st.to_string(); + ASSERT_EQ(result->size(), 4); + + const auto& nullable = assert_cast(*result); + const auto& values = + assert_cast(nullable.get_nested_column()).get_data(); + EXPECT_FALSE(nullable.is_null_at(0)); + EXPECT_EQ(values[0].epoch_nanos(), 11); + EXPECT_FALSE(nullable.is_null_at(1)); + EXPECT_EQ(values[1].epoch_nanos(), 22); + EXPECT_FALSE(nullable.is_null_at(2)); + EXPECT_EQ(values[2].epoch_nanos(), -17); + EXPECT_FALSE(nullable.is_null_at(3)); + EXPECT_EQ(values[3].epoch_nanos(), std::numeric_limits::min()); +} + } // namespace doris diff --git a/be/test/exprs/vexpr_unaligned_int128_test.cpp b/be/test/exprs/vexpr_unaligned_int128_test.cpp index d34a6da7f9430a..844c5fc633e463 100644 --- a/be/test/exprs/vexpr_unaligned_int128_test.cpp +++ b/be/test/exprs/vexpr_unaligned_int128_test.cpp @@ -34,6 +34,7 @@ #include "core/value/decimalv2_value.h" #include "core/value/large_int_value.h" +#include "core/value/timestamp_ns_value.h" #include "exprs/vexpr.h" namespace doris { @@ -104,4 +105,14 @@ TEST(UnalignedInt128Test, DecimalV2LiteralFromUnalignedBuffer) { EXPECT_EQ(node.decimal_literal.value, src.to_string()); } +TEST(UnalignedInt128Test, TimeStampNsLiteralKeepsItsPhysicalType) { + const TimeStampNsValue value(1); + TExprNode node; + Status st = create_texpr_literal_node(&value, &node); + ASSERT_TRUE(st.ok()) << st; + ASSERT_TRUE(node.__isset.date_literal); + ASSERT_EQ(node.type.types.size(), 1); + EXPECT_EQ(node.type.types.front().scalar_type.type, TPrimitiveType::TIMESTAMP_NS); +} + } // namespace doris diff --git a/be/test/format/orc/orc_reader_fill_data_test.cpp b/be/test/format/orc/orc_reader_fill_data_test.cpp index 44177ddb29df46..ae0f035432d1ad 100644 --- a/be/test/format/orc/orc_reader_fill_data_test.cpp +++ b/be/test/format/orc/orc_reader_fill_data_test.cpp @@ -161,6 +161,26 @@ TEST_F(OrcReaderFillDataTest, TimestampDecodeNormalizesCstTimezone) { EXPECT_EQ(data_type->to_string(timestamp_column.get_data()[0]), "2020-01-02 03:04:05.321000"); } +TEST_F(OrcReaderFillDataTest, TimestampDecodeKeepsWideDateTimeV2Range) { + auto batch = create_timestamp_batch(1, {253402300799}, {999999000}); + auto data_type = std::make_shared(6); + auto column = data_type->create_column(); + auto orc_type_ptr = createPrimitiveType(orc::TypeKind::TIMESTAMP); + + TFileScanRangeParams params; + TFileRangeDesc range; + auto reader = OrcReader::create_unique(params, range, 4064, "UTC", nullptr, nullptr, true); + + MutableColumnPtr mutable_column = column->assert_mutable(); + Status status = reader->_fill_doris_data_column( + "test_ts", mutable_column, data_type, const_node, orc_type_ptr.get(), batch.get(), 1); + + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto& timestamp_column = assert_cast(*mutable_column); + ASSERT_EQ(timestamp_column.size(), 1); + EXPECT_EQ(data_type->to_string(timestamp_column.get_data()[0]), "9999-12-31 23:59:59.999999"); +} + TEST_F(OrcReaderFillDataTest, SchemaChangeNullableNullMapUsesAppendedSlice) { std::vector values = {10, 20, 30}; std::vector nulls = {true, false, true}; diff --git a/be/test/storage/key_coder_test.cpp b/be/test/storage/key_coder_test.cpp index 5362645078677c..ee521f3a86df54 100644 --- a/be/test/storage/key_coder_test.cpp +++ b/be/test/storage/key_coder_test.cpp @@ -174,6 +174,7 @@ TEST_F(KeyCoderTest, test_int) { test_integer_encode(); test_integer_encode(); + test_integer_encode(); } TEST_F(KeyCoderTest, test_date) { diff --git a/be/test/storage/segment/bloom_filter_index_reader_writer_test.cpp b/be/test/storage/segment/bloom_filter_index_reader_writer_test.cpp index 17eeac11a7db15..2d798a1498b21b 100644 --- a/be/test/storage/segment/bloom_filter_index_reader_writer_test.cpp +++ b/be/test/storage/segment/bloom_filter_index_reader_writer_test.cpp @@ -476,6 +476,33 @@ TEST_F(BloomFilterIndexReaderWriterTest, test_datetimev2) { delete[] val; } +TEST_F(BloomFilterIndexReaderWriterTest, test_timestamp_ns) { + size_t num = 1024 * 3; + std::vector values(num); + for (size_t i = 0; i < num; ++i) { + values[i] = -1'000'000'000 + static_cast(i); + } + + { + const std::string file_name = "bloom_filter_timestamp_ns"; + int64_t not_exist_value = -1'000'000'001; + auto st = test_bloom_filter_index_reader_writer_template< + FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS>(file_name, values.data(), num, 1, + ¬_exist_value); + EXPECT_TRUE(st.ok()); + } + // Primary key bloom filters consume encoded Slice keys rather than raw column values. + { + const std::string file_name = "bloom_filter_timestamp_ns_pk"; + int64_t not_exist_value = -1'000'000'001; + auto st = test_bloom_filter_index_reader_writer_template< + FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS>(file_name, values.data(), num, 1, + ¬_exist_value, false, true); + EXPECT_FALSE(st.ok()); + EXPECT_EQ(st.code(), TStatusCode::NOT_IMPLEMENTED_ERROR); + } +} + TEST_F(BloomFilterIndexReaderWriterTest, test_timestamptz) { size_t num = 1024 * 3; auto base_dt = make_timestamptz(2025, 11, 14, 14, 37, 30, 999998); diff --git a/be/test/storage/segment/inverted_index_writer_test.cpp b/be/test/storage/segment/inverted_index_writer_test.cpp index 8b7f56221cf8cf..563e9cbdd79c67 100644 --- a/be/test/storage/segment/inverted_index_writer_test.cpp +++ b/be/test/storage/segment/inverted_index_writer_test.cpp @@ -790,6 +790,44 @@ TEST_F(InvertedIndexWriterTest, NumericWrite) { test_numeric_write("test_rowset_3", 0); } +TEST_F(InvertedIndexWriterTest, TimeStampNsWrite) { + TabletColumn field; + field.set_name("dt"); + field.set_unique_id(0); + field.set_type(FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS); + field.set_is_nullable(false); + + TabletIndexPB index_meta_pb; + index_meta_pb.set_index_type(IndexType::INVERTED); + index_meta_pb.set_index_id(1); + index_meta_pb.set_index_name("test_timestamp_ns"); + index_meta_pb.add_col_unique_id(0); + TabletIndex index_meta; + index_meta.init_from_pb(index_meta_pb); + + const std::string rowset_id = "test_timestamp_ns"; + const std::string index_path_prefix {InvertedIndexDescriptor::get_index_file_path_prefix( + local_segment_path(kTestDir, rowset_id, 0))}; + const std::string index_path = + InvertedIndexDescriptor::get_index_file_path_v2(index_path_prefix); + io::FileWriterPtr file_writer; + ASSERT_TRUE(io::global_local_filesystem()->create_file(index_path, &file_writer).ok()); + auto index_file_writer = std::make_unique( + io::global_local_filesystem(), index_path_prefix, rowset_id, 0, + InvertedIndexStorageFormatPB::V2, std::move(file_writer)); + + std::unique_ptr column_writer; + ASSERT_TRUE( + IndexColumnWriter::create(&field, &column_writer, index_file_writer.get(), &index_meta) + .ok()); + const std::vector values = {std::numeric_limits::min(), -1, 0, 1, + std::numeric_limits::max()}; + ASSERT_TRUE(column_writer->add_values("dt", values.data(), values.size()).ok()); + ASSERT_TRUE(column_writer->finish().ok()); + ASSERT_TRUE(index_file_writer->begin_close().ok()); + ASSERT_TRUE(index_file_writer->finish_close().ok()); +} + // Test case for Unicode string values with enable_correct_term_write=true TEST_F(InvertedIndexWriterTest, UnicodeStringWriteEnabled) { test_unicode_string_write("test_rowset_4", 0, true); @@ -1544,4 +1582,4 @@ TEST_F(InvertedIndexWriterTest, NormsFileCreationWithTokenization) { << "inverted_index_writer.cpp where .nrm file creation depends on _should_analyzer."; } -} // namespace doris::segment_v2 \ No newline at end of file +} // namespace doris::segment_v2 diff --git a/be/test/storage/segment/zone_map_index_test.cpp b/be/test/storage/segment/zone_map_index_test.cpp index db4b692b1f1477..079a2d237dc553 100644 --- a/be/test/storage/segment/zone_map_index_test.cpp +++ b/be/test/storage/segment/zone_map_index_test.cpp @@ -1119,6 +1119,44 @@ TEST_F(ColumnZoneMapTest, TimestamptzPage) { EXPECT_EQ(false, zone_maps[4].has_not_null()); } +TEST_F(ColumnZoneMapTest, TimeStampNsPage) { + auto column = std::make_shared(); + column->_unique_id = 0; + column->_col_name = "timestamp_ns"; + column->_type = FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS; + column->_is_key = true; + column->_is_nullable = true; + column->_length = 8; + column->_index_length = 8; + + auto data_type = DataTypeFactory::instance().create_data_type(TYPE_TIMESTAMP_NS, false, 0, 9); + std::unique_ptr writer; + ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, column.get(), writer).ok()); + + TimeStampNsValue values[] = { + TimeStampNsValue(std::numeric_limits::min()), + TimeStampNsValue(-1), + TimeStampNsValue(0), + TimeStampNsValue(std::numeric_limits::max()), + }; + writer->add_values(values, std::size(values)); + ASSERT_TRUE(writer->flush().ok()); + + const std::string file_path = kTestDir + "/timestamp_ns_zonemap"; + io::FileWriterPtr file_writer; + ASSERT_TRUE(_fs->create_file(file_path, &file_writer).ok()); + ColumnIndexMetaPB index_meta; + ASSERT_TRUE(writer->finish(file_writer.get(), &index_meta).ok()); + ASSERT_TRUE(file_writer->close().ok()); + + ZoneMap segment_zone_map; + ASSERT_TRUE(ZoneMap::from_proto(index_meta.zone_map_index().segment_zone_map(), data_type, + segment_zone_map) + .ok()); + EXPECT_EQ(segment_zone_map.min_value.get(), values[0]); + EXPECT_EQ(segment_zone_map.max_value.get(), values[3]); +} + // Regression test for "all-null page after a value page" — int variant. // // Page 1 has integers, page 2 is all nulls. The fix in flush() guards the diff --git a/be/test/testutil/datetime_ut_util.h b/be/test/testutil/datetime_ut_util.h index 9f0967ce1d76dc..9ced7af876c0a0 100644 --- a/be/test/testutil/datetime_ut_util.h +++ b/be/test/testutil/datetime_ut_util.h @@ -27,6 +27,15 @@ inline auto make_datetime(int year, int month, int day, int hour, int minute, in return dt; } +inline auto make_datetime_nano(int year, int month, int day, int hour, int minute, int second, + int nanosecond) { + DateV2Value datetime; + datetime.unchecked_set_time(year, month, day, hour, minute, second, nanosecond / 1000); + TimeStampNsValue result; + DORIS_CHECK(result.from_datetime(datetime, nanosecond % 1000)); + return result; +} + inline auto make_timestamptz(int year, int month, int day, int hour, int minute, int second, int microsecond) { DateV2Value dt; @@ -34,4 +43,4 @@ inline auto make_timestamptz(int year, int month, int day, int hour, int minute, return TimestampTzValue(dt.to_date_int_val()); } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/cloud/src/meta-service/meta_service_schema.h b/cloud/src/meta-service/meta_service_schema.h index ffade3a150fbcf..4293de93d02508 100644 --- a/cloud/src/meta-service/meta_service_schema.h +++ b/cloud/src/meta-service/meta_service_schema.h @@ -24,6 +24,9 @@ namespace doris::cloud { class Transaction; struct ValueBuf; +bool check_tablet_schema(const doris::TabletSchemaCloudPB& schema, + doris::TabletSchemaCloudPB& saved_schema); + void put_schema_kv(MetaServiceCode& code, std::string& msg, Transaction* txn, std::string_view schema_key, const doris::TabletSchemaCloudPB& schema); diff --git a/cloud/test/schema_kv_test.cpp b/cloud/test/schema_kv_test.cpp index 32a9302a0150e4..146dbd7e2e1e64 100644 --- a/cloud/test/schema_kv_test.cpp +++ b/cloud/test/schema_kv_test.cpp @@ -335,6 +335,20 @@ TEST(DetachSchemaKVTest, PutSchemaKvTest) { ASSERT_EQ(document_get(txn.get(), versioned_key, &saved_schema), TxnErrorCode::TXN_OK); EXPECT_EQ(saved_schema.schema_version(), schema_version); } + + { + doris::TabletSchemaCloudPB timestamp_ns_schema; + auto* timestamp_ns_column = timestamp_ns_schema.add_column(); + timestamp_ns_column->set_unique_id(1); + timestamp_ns_column->set_type("TIMESTAMP_NS"); + + doris::TabletSchemaCloudPB saved_schema(timestamp_ns_schema); + EXPECT_TRUE(check_tablet_schema(timestamp_ns_schema, saved_schema)); + + doris::TabletSchemaCloudPB datetimev2_schema(timestamp_ns_schema); + datetimev2_schema.mutable_column(0)->set_type("DATETIMEV2"); + EXPECT_FALSE(check_tablet_schema(datetimev2_schema, saved_schema)); + } } static void begin_txn(MetaServiceProxy* meta_service, int64_t db_id, const std::string& label, diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/DateLiteral.java index edb4c7b3c533c1..88ec6851995cba 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -422,6 +422,9 @@ public int compareLiteral(LiteralExpr expr) { long diff = getMicroPartWithinScale() - other.getMicroPartWithinScale(); return diff < 0 ? -1 : (diff == 0 ? 0 : 1); } + if (expr instanceof TimeStampNsLiteral) { + return -expr.compareLiteral(this); + } // date time will not overflow when doing addition and subtraction return Integer.signum(getStringValue().compareTo(expr.getStringValue())); } diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/ExprVisitor.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/ExprVisitor.java index fbed2844989f71..2fb6d46f2bdae9 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/ExprVisitor.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/ExprVisitor.java @@ -18,7 +18,7 @@ package org.apache.doris.analysis; /** - * Abstract visitor base class for {@link Expr} and all 39 concrete subclasses. + * Abstract visitor base class for {@link Expr} and all concrete subclasses. * Follows the same {@code } pattern as * {@code org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor}. * @@ -69,6 +69,10 @@ public R visitDateLiteral(DateLiteral expr, C context) { return visit(expr, context); } + public R visitTimeStampNsLiteral(TimeStampNsLiteral expr, C context) { + return visit(expr, context); + } + public R visitTimeV2Literal(TimeV2Literal expr, C context) { return visit(expr, context); } diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/TimeStampNsLiteral.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/TimeStampNsLiteral.java new file mode 100644 index 00000000000000..42fc483c69259f --- /dev/null +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/TimeStampNsLiteral.java @@ -0,0 +1,376 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; + +import com.google.common.base.Preconditions; +import com.google.gson.annotations.SerializedName; + +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.time.DateTimeException; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.Year; +import java.time.ZoneOffset; +import java.util.TimeZone; + +/** + * Legacy literal for the fixed nanosecond-precision TIMESTAMP_NS type. + * + *

TIMESTAMP_NS owns its value and range logic instead of inheriting the unrelated calendar + * encodings and microsecond rules in {@link DateLiteral}.

+ */ +public final class TimeStampNsLiteral extends LiteralExpr { + private static final long NANOSECONDS_PER_SECOND = 1_000_000_000L; + private static final long MAX_NANOSECOND = NANOSECONDS_PER_SECOND - 1; + private static final LocalDateTime MIN_VALUE + = LocalDateTime.of(1677, 9, 21, 0, 12, 43, 145224192); + private static final LocalDateTime MAX_VALUE + = LocalDateTime.of(2262, 4, 11, 23, 47, 16, 854775807); + + @SerializedName("y") + private long year; + @SerializedName("m") + private long month; + @SerializedName("d") + private long day; + @SerializedName("h") + private long hour; + @SerializedName("M") + private long minute; + @SerializedName("s") + private long second; + @SerializedName("ns") + private long nanosecond; + + public TimeStampNsLiteral() { + type = Type.TIMESTAMP_NS; + nullable = false; + } + + public TimeStampNsLiteral(boolean isMax) { + this(isMax ? MAX_VALUE : MIN_VALUE); + } + + public TimeStampNsLiteral(long year, long month, long day, long hour, long minute, long second, + long nanosecond) { + this(); + this.year = year; + this.month = month; + this.day = day; + this.hour = hour; + this.minute = minute; + this.second = second; + this.nanosecond = nanosecond; + } + + public TimeStampNsLiteral(LocalDateTime value) { + this(value.getYear(), value.getMonthValue(), value.getDayOfMonth(), value.getHour(), + value.getMinute(), value.getSecond(), value.getNano()); + } + + private TimeStampNsLiteral(TimeStampNsLiteral other) { + super(other); + year = other.year; + month = other.month; + day = other.day; + hour = other.hour; + minute = other.minute; + second = other.second; + nanosecond = other.nanosecond; + type = Type.TIMESTAMP_NS; + } + + public static TimeStampNsLiteral createMinValue() { + return new TimeStampNsLiteral(false); + } + + @Override + public Expr clone() { + return new TimeStampNsLiteral(this); + } + + @Override + public boolean isMinValue() { + return compareBoundary(MIN_VALUE) == 0; + } + + @Override + public Object getRealValue() { + LocalDateTime value = toLocalDateTime(); + return BigInteger.valueOf(value.toEpochSecond(ZoneOffset.UTC)) + .multiply(BigInteger.valueOf(NANOSECONDS_PER_SECOND)) + .add(BigInteger.valueOf(nanosecond)) + .longValueExact(); + } + + @Override + public ByteBuffer getHashValue(PrimitiveType primitiveType) { + Preconditions.checkArgument(primitiveType == PrimitiveType.TIMESTAMP_NS, + "Expected TIMESTAMP_NS hash type, but got %s", primitiveType); + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES).order(ByteOrder.LITTLE_ENDIAN); + buffer.putLong((long) getRealValue()); + return buffer; + } + + @Override + public int compareLiteral(LiteralExpr expr) { + if (expr instanceof PlaceHolderExpr) { + return compareLiteral(((PlaceHolderExpr) expr).getLiteral()); + } + if (expr instanceof NullLiteral) { + return 1; + } + if (expr == MaxLiteral.MAX_VALUE) { + return -1; + } + if (expr instanceof TimeStampNsLiteral) { + return compareDateTime((TimeStampNsLiteral) expr); + } + if (expr instanceof DateLiteral) { + DateLiteral other = (DateLiteral) expr; + int result = compareDateTime(other.getYear(), other.getMonth(), other.getDay(), + other.getHour(), other.getMinute(), other.getSecond(), other.getMicrosecond() * 1000); + if (result == 0 && other.isDateType()) { + return 1; + } + return result; + } + return Integer.signum(getStringValue().compareTo(expr.getStringValue())); + } + + private int compareDateTime(TimeStampNsLiteral other) { + return compareDateTime(other.year, other.month, other.day, other.hour, other.minute, + other.second, other.nanosecond); + } + + private int compareDateTime(long otherYear, long otherMonth, long otherDay, long otherHour, + long otherMinute, long otherSecond, long otherNanosecond) { + int result = Long.compare(year, otherYear); + if (result == 0) { + result = Long.compare(month, otherMonth); + } + if (result == 0) { + result = Long.compare(day, otherDay); + } + if (result == 0) { + result = Long.compare(hour, otherHour); + } + if (result == 0) { + result = Long.compare(minute, otherMinute); + } + if (result == 0) { + result = Long.compare(second, otherSecond); + } + if (result == 0) { + result = Long.compare(nanosecond, otherNanosecond); + } + return result; + } + + @Override + public String getStringValue() { + return String.format("%04d-%02d-%02d %02d:%02d:%02d.%09d", + year, month, day, hour, minute, second, nanosecond); + } + + public void roundFloor(int newScale) { + Preconditions.checkArgument(newScale == ScalarType.TIMESTAMP_NS_SCALE, + "TIMESTAMP_NS has fixed scale %s, but got %s", + ScalarType.TIMESTAMP_NS_SCALE, newScale); + } + + @Override + public void checkValueValid() throws AnalysisException { + if (year < 0 || year > 9999) { + throw new AnalysisException("TimeStampNsLiteral has invalid year value: " + year); + } + if (month < 1 || month > 12) { + throw new AnalysisException("TimeStampNsLiteral has invalid month value: " + month); + } + if (day < 1 || day > daysInMonth(year, month)) { + throw new AnalysisException("TimeStampNsLiteral has invalid day value: " + day); + } + if (hour < 0 || hour > 23) { + throw new AnalysisException("TimeStampNsLiteral has invalid hour value: " + hour); + } + if (minute < 0 || minute > 59) { + throw new AnalysisException("TimeStampNsLiteral has invalid minute value: " + minute); + } + if (second < 0 || second > 59) { + throw new AnalysisException("TimeStampNsLiteral has invalid second value: " + second); + } + if (nanosecond < 0 || nanosecond > MAX_NANOSECOND) { + throw new AnalysisException("TimeStampNsLiteral has invalid nanosecond value: " + nanosecond); + } + if (checkRange()) { + throw new AnalysisException("TimeStampNsLiteral is outside Int64 epoch nanosecond range: " + + getStringValue()); + } + } + + boolean checkRange() { + if (year < 0 || year > 9999 || month < 1 || month > 12 + || hour < 0 || hour > 23 || minute < 0 || minute > 59 + || second < 0 || second > 59) { + return true; + } + if (day < 1 || day > daysInMonth(year, month)) { + return true; + } + if (nanosecond < 0 || nanosecond > MAX_NANOSECOND) { + return true; + } + return compareBoundary(MIN_VALUE) < 0 || compareBoundary(MAX_VALUE) > 0; + } + + private int compareBoundary(LocalDateTime boundary) { + return compareDateTime(boundary.getYear(), boundary.getMonthValue(), boundary.getDayOfMonth(), + boundary.getHour(), boundary.getMinute(), boundary.getSecond(), boundary.getNano()); + } + + private static int daysInMonth(long year, long month) { + switch ((int) month) { + case 2: + return Year.isLeap(year) ? 29 : 28; + case 4: + case 6: + case 9: + case 11: + return 30; + default: + return 31; + } + } + + public LocalDateTime toLocalDateTime() { + try { + return LocalDateTime.of((int) year, (int) month, (int) day, (int) hour, + (int) minute, (int) second, (int) nanosecond); + } catch (DateTimeException e) { + throw new IllegalStateException("Invalid TIMESTAMP_NS literal: " + getStringValue(), e); + } + } + + public TimeStampNsLiteral plusYears(long years) { + return new TimeStampNsLiteral(toLocalDateTime().plusYears(years)); + } + + public TimeStampNsLiteral plusMonths(long months) { + return new TimeStampNsLiteral(toLocalDateTime().plusMonths(months)); + } + + public TimeStampNsLiteral plusDays(long days) { + return new TimeStampNsLiteral(toLocalDateTime().plusDays(days)); + } + + public TimeStampNsLiteral plusHours(long hours) { + return new TimeStampNsLiteral(toLocalDateTime().plusHours(hours)); + } + + public TimeStampNsLiteral plusMinutes(long minutes) { + return new TimeStampNsLiteral(toLocalDateTime().plusMinutes(minutes)); + } + + public TimeStampNsLiteral plusSeconds(long seconds) { + return new TimeStampNsLiteral(toLocalDateTime().plusSeconds(seconds)); + } + + public void setMinValue() { + TimeStampNsLiteral minimum = createMinValue(); + year = minimum.year; + month = minimum.month; + day = minimum.day; + hour = minimum.hour; + minute = minimum.minute; + second = minimum.second; + nanosecond = minimum.nanosecond; + } + + public long getUnixTimestampWithNanoseconds(TimeZone timeZone) { + Instant instant = toLocalDateTime().atZone(timeZone.toZoneId()).toInstant(); + return Math.addExact(Math.multiplyExact(instant.getEpochSecond(), NANOSECONDS_PER_SECOND), + instant.getNano()); + } + + public long getYear() { + return year; + } + + public long getMonth() { + return month; + } + + public long getDay() { + return day; + } + + public long getHour() { + return hour; + } + + public long getMinute() { + return minute; + } + + public long getSecond() { + return second; + } + + public long getMicrosecond() { + return nanosecond / 1000; + } + + public long getNanosecond() { + return nanosecond; + } + + @Override + public long getLongValue() { + return (year * 10000 + month * 100 + day) * 1000000L + hour * 10000 + minute * 100 + second; + } + + @Override + public double getDoubleValue() { + return getLongValue(); + } + + public double getDoubleValueAsDateTime() { + return getDoubleValue(); + } + + @Override + public int hashCode() { + int legacyHash = Long.hashCode(getLongValue()); + if (nanosecond % 1000 == 0) { + return legacyHash; + } + return 31 * legacyHash + Long.hashCode(nanosecond); + } + + @Override + public R accept(ExprVisitor visitor, C context) { + return visitor.visitTimeStampNsLiteral(this, context); + } +} diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/AggregateType.java b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/AggregateType.java index 2bcdc2165f68db..a49644ce505b89 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/AggregateType.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/AggregateType.java @@ -86,6 +86,7 @@ public enum AggregateType { primitiveTypeList.add(PrimitiveType.DATETIME); primitiveTypeList.add(PrimitiveType.DATEV2); primitiveTypeList.add(PrimitiveType.DATETIMEV2); + primitiveTypeList.add(PrimitiveType.TIMESTAMP_NS); primitiveTypeList.add(PrimitiveType.TIMESTAMPTZ); primitiveTypeList.add(PrimitiveType.CHAR); primitiveTypeList.add(PrimitiveType.VARCHAR); @@ -108,6 +109,7 @@ public enum AggregateType { primitiveTypeList.add(PrimitiveType.DATETIME); primitiveTypeList.add(PrimitiveType.DATEV2); primitiveTypeList.add(PrimitiveType.DATETIMEV2); + primitiveTypeList.add(PrimitiveType.TIMESTAMP_NS); primitiveTypeList.add(PrimitiveType.TIMESTAMPTZ); primitiveTypeList.add(PrimitiveType.CHAR); primitiveTypeList.add(PrimitiveType.VARCHAR); diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/ColumnType.java b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/ColumnType.java index 483f22e2425f16..d3e6ef3fbbcf64 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/ColumnType.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/ColumnType.java @@ -149,17 +149,26 @@ public abstract class ColumnType { schemaChangeMatrix[PrimitiveType.DATETIME.ordinal()][PrimitiveType.DATEV2.ordinal()] = true; schemaChangeMatrix[PrimitiveType.DATE.ordinal()][PrimitiveType.DATETIMEV2.ordinal()] = true; schemaChangeMatrix[PrimitiveType.DATETIME.ordinal()][PrimitiveType.DATETIMEV2.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.DATE.ordinal()][PrimitiveType.TIMESTAMP_NS.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.DATETIME.ordinal()][PrimitiveType.TIMESTAMP_NS.ordinal()] = true; schemaChangeMatrix[PrimitiveType.DATE.ordinal()][PrimitiveType.DATEV2.ordinal()] = true; schemaChangeMatrix[PrimitiveType.DATETIMEV2.ordinal()][PrimitiveType.DATE.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.TIMESTAMP_NS.ordinal()][PrimitiveType.DATE.ordinal()] = true; schemaChangeMatrix[PrimitiveType.DATEV2.ordinal()][PrimitiveType.DATETIME.ordinal()] = true; schemaChangeMatrix[PrimitiveType.DATETIMEV2.ordinal()][PrimitiveType.DATEV2.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.TIMESTAMP_NS.ordinal()][PrimitiveType.DATEV2.ordinal()] = true; schemaChangeMatrix[PrimitiveType.DATEV2.ordinal()][PrimitiveType.DATETIMEV2.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.DATEV2.ordinal()][PrimitiveType.TIMESTAMP_NS.ordinal()] = true; schemaChangeMatrix[PrimitiveType.DATETIMEV2.ordinal()][PrimitiveType.DATETIME.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.TIMESTAMP_NS.ordinal()][PrimitiveType.DATETIME.ordinal()] = true; schemaChangeMatrix[PrimitiveType.DATEV2.ordinal()][PrimitiveType.DATE.ordinal()] = true; // we should support schema change between different precision schemaChangeMatrix[PrimitiveType.DATETIMEV2.ordinal()][PrimitiveType.DATETIMEV2.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.DATETIMEV2.ordinal()][PrimitiveType.TIMESTAMP_NS.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.TIMESTAMP_NS.ordinal()][PrimitiveType.DATETIMEV2.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.TIMESTAMP_NS.ordinal()][PrimitiveType.TIMESTAMP_NS.ordinal()] = true; // Currently, we do not support schema change between complex types with subtypes. schemaChangeMatrix[PrimitiveType.ARRAY.ordinal()][PrimitiveType.ARRAY.ordinal()] = true; diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/persist/gson/GsonUtilsCatalog.java b/fe/fe-catalog/src/main/java/org/apache/doris/persist/gson/GsonUtilsCatalog.java index ec65e8db39df99..13bee42ee1e91d 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/persist/gson/GsonUtilsCatalog.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/persist/gson/GsonUtilsCatalog.java @@ -54,6 +54,7 @@ import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.StringLiteral; import org.apache.doris.analysis.StructLiteral; +import org.apache.doris.analysis.TimeStampNsLiteral; import org.apache.doris.analysis.TimeV2Literal; import org.apache.doris.analysis.TimestampArithmeticExpr; import org.apache.doris.analysis.TryCastExpr; @@ -131,6 +132,7 @@ public class GsonUtilsCatalog { .registerSubtype(ArrayLiteral.class, ArrayLiteral.class.getSimpleName()) .registerSubtype(BoolLiteral.class, BoolLiteral.class.getSimpleName()) .registerSubtype(DateLiteral.class, DateLiteral.class.getSimpleName()) + .registerSubtype(TimeStampNsLiteral.class, TimeStampNsLiteral.class.getSimpleName()) .registerSubtype(IPv4Literal.class, IPv4Literal.class.getSimpleName()) .registerSubtype(IPv6Literal.class, IPv6Literal.class.getSimpleName()) .registerSubtype(JsonLiteral.class, JsonLiteral.class.getSimpleName()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java index 597a006707ed2c..f9a475fe78fa63 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java @@ -323,6 +323,7 @@ public static void validateDefaultValue(Type type, String defaultValue, DefaultV switch (primitiveType) { case DATETIME: case DATETIMEV2: + case TIMESTAMP_NS: case TIMESTAMPTZ: break; default: @@ -400,16 +401,19 @@ public static void validateDefaultValue(Type type, String defaultValue, DefaultV break; case DATETIME: case DATETIMEV2: + case TIMESTAMP_NS: case TIMESTAMPTZ: if (defaultValueExprDef == null) { - DateLiteralUtils.createDateLiteral(defaultValue, scalarType); + DateLiteralUtils.createLiteral(defaultValue, scalarType); } else { if (defaultValueExprDef.getExprName().equals(DefaultValue.NOW)) { if (defaultValueExprDef.getPrecision() != null) { Long defaultValuePrecision = defaultValueExprDef.getPrecision(); String typeStr = scalarType.toString(); - int typePrecision = - Integer.parseInt(typeStr.substring(typeStr.indexOf("(") + 1, typeStr.indexOf(")"))); + int typePrecision = scalarType.isTimeStampNs() + ? ScalarType.TIMESTAMP_NS_SCALE + : Integer.parseInt(typeStr.substring( + typeStr.indexOf("(") + 1, typeStr.indexOf(")"))); if (defaultValuePrecision > typePrecision) { typeStr = typeStr.replace("V2", ""); throw new AnalysisException("default value precision: " + defaultValue diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteralUtils.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteralUtils.java index 84d8f2bb0ea22b..709ac3f2912820 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteralUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteralUtils.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.nereids.util.DateUtils; import com.google.common.base.Preconditions; @@ -37,11 +38,7 @@ import java.util.regex.Pattern; import javax.annotation.Nullable; -/** - * Utility class providing a static factory method for creating {@link DateLiteral} instances - * from string representations. This replicates the parsing logic of the - * {@code DateLiteral(String, Type)} constructor and its {@code init()} method. - */ +/** Utility methods for parsing legacy date-like literals from strings. */ public class DateLiteralUtils { private static final Pattern HAS_OFFSET_PART = Pattern.compile("[\\+\\-]\\d{2}:\\d{2}"); @@ -49,16 +46,22 @@ public class DateLiteralUtils { private DateLiteralUtils() { } - /** - * Creates a {@link DateLiteral} by parsing the given string, replicating the exact behavior - * of the {@code DateLiteral(String, Type)} constructor and its internal {@code init()} method. - * - * @param s the date/datetime string to parse - * @param type the target date type, or {@code null} for auto-detection - * @return a new {@link DateLiteral} instance - * @throws AnalysisException if the string cannot be parsed or the resulting value is out of range - */ + /** Parse a DATE, DATETIME, DATETIMEV2, TIMESTAMPTZ, or TIMESTAMP_NS literal. */ + public static LiteralExpr createLiteral(String s, @Nullable Type type) throws AnalysisException { + return createDateTimeLiteral(s, type); + } + + /** Parse a literal that is represented by {@link DateLiteral}. */ public static DateLiteral createDateLiteral(String s, @Nullable Type type) throws AnalysisException { + LiteralExpr literal = createDateTimeLiteral(s, type); + if (!(literal instanceof DateLiteral)) { + throw new AnalysisException("date literal [" + s + + "] resolves to TIMESTAMP_NS; use DateLiteralUtils.createLiteral instead"); + } + return (DateLiteral) literal; + } + + private static LiteralExpr createDateTimeLiteral(String s, @Nullable Type type) throws AnalysisException { try { if (type != null) { Preconditions.checkArgument(type.isDateType()); @@ -84,6 +87,12 @@ public static DateLiteral createDateLiteral(String s, @Nullable Type type) throw sourceZone = ZoneId.of(tzString); } + int nanosecondGuardDigit = -1; + if (type != null && (type.isDatetimeV2() || type.isTimeStampNs())) { + nanosecondGuardDigit = DateUtils.getNanosecondGuardDigit(s); + s = DateUtils.truncateFractionalSecondForJavaParser(s); + } + if (!s.contains("-")) { // handle format like 20210106, but should not handle 2021-1-6 for (DateTimeFormatter formatter : DateLiteral.formatterList) { @@ -140,7 +149,8 @@ public static DateLiteral createDateLiteral(String s, @Nullable Type type) throw throw new AnalysisException("Invalid date value: " + s); } if (timePart.length == 0 && type != null - && (type.equals(Type.DATETIME) || type.equals(Type.DATETIMEV2))) { + && (type.equals(Type.DATETIME) || type.equals(Type.DATETIMEV2) + || type.equals(Type.TIMESTAMP_NS))) { throw new AnalysisException("Invalid datetime value: " + s); } for (int i = 0; i < timePart.length; i++) { @@ -159,7 +169,7 @@ public static DateLiteral createDateLiteral(String s, @Nullable Type type) throw ? timePart[i].split("\\.")[0].length() : timePart[i].length(), "s"))); if (timePart[i].contains(".")) { - builder.appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, true); + builder.appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true); } break; default: @@ -183,29 +193,32 @@ public static DateLiteral createDateLiteral(String s, @Nullable Type type) throw long hour = getOrDefault(dateTime, ChronoField.HOUR_OF_DAY, 0); long minute = getOrDefault(dateTime, ChronoField.MINUTE_OF_HOUR, 0); long second = getOrDefault(dateTime, ChronoField.SECOND_OF_MINUTE, 0); - long microsecond = getOrDefault(dateTime, ChronoField.MICRO_OF_SECOND, 0); + long nanosecond = getOrDefault(dateTime, ChronoField.NANO_OF_SECOND, 0); + long microsecond = nanosecond / 1000; if (type != null) { - if (microsecond != 0 && type.isDatetime()) { + if (nanosecond != 0 && type.isDatetime()) { int dotIndex = s.lastIndexOf("."); int scale = s.length() - dotIndex - 1; - type = ScalarType.createDatetimeV2Type(scale); + type = scale > ScalarType.MAX_DATETIMEV2_SCALE + ? ScalarType.createTimeStampNsType() + : ScalarType.createDatetimeV2Type(scale); } } else { - if (hour == 0 && minute == 0 && second == 0 && microsecond == 0) { + if (hour == 0 && minute == 0 && second == 0 && nanosecond == 0) { type = ScalarType.getDefaultDateType(Type.DATE); } else { type = ScalarType.getDefaultDateType(Type.DATETIME); - if (type.isDatetimeV2() && microsecond != 0) { - int scale = 6; - for (int i = 0; i < 6; i++) { - if (microsecond % Math.pow(10.0, i + 1) > 0) { - break; - } else { - scale -= 1; - } + if (type.isDatetimeV2() && nanosecond != 0) { + int scale = ScalarType.TIMESTAMP_NS_SCALE; + long fractionalSecond = nanosecond; + while (fractionalSecond % 10 == 0) { + fractionalSecond /= 10; + scale--; } - type = ScalarType.createDatetimeV2Type(scale); + type = scale > ScalarType.MAX_DATETIMEV2_SCALE + ? ScalarType.createTimeStampNsType() + : ScalarType.createDatetimeV2Type(scale); } } } @@ -230,7 +243,8 @@ public static DateLiteral createDateLiteral(String s, @Nullable Type type) throw } LocalDateTime parsedLdt = LocalDateTime.of( (int) year, (int) month, (int) day, - (int) hour, (int) minute, (int) second); + (int) hour, (int) minute, (int) second, + (int) nanosecond); Instant targetInstant = parsedLdt.atZone(sourceZone).toInstant(); LocalDateTime destLdt = LocalDateTime.ofInstant(targetInstant, dorisZone); year = destLdt.getYear(); @@ -243,11 +257,31 @@ public static DateLiteral createDateLiteral(String s, @Nullable Type type) throw // the original init() behavior (not affected by zone conversion) } - // Construct DateLiteral using the appropriate constructor based on the determined type + if (type.isDatetimeV2() || type.isTimeStampNs()) { + LocalDateTime roundedDateTime = roundFractionalSecond( + year, month, day, hour, minute, second, nanosecond, + type.isTimeStampNs() ? 9 : ((ScalarType) type).getScalarScale(), nanosecondGuardDigit); + year = roundedDateTime.getYear(); + month = roundedDateTime.getMonthValue(); + day = roundedDateTime.getDayOfMonth(); + hour = roundedDateTime.getHour(); + minute = roundedDateTime.getMinute(); + second = roundedDateTime.getSecond(); + nanosecond = roundedDateTime.getNano(); + microsecond = nanosecond / 1000; + } + + if (type.isTimeStampNs()) { + TimeStampNsLiteral result = new TimeStampNsLiteral( + year, month, day, hour, minute, second, nanosecond); + result.checkValueValid(); + return result; + } + DateLiteral result; if (type.isDate() || type.isDateV2()) { result = new DateLiteral(year, month, day, type); - } else if (microsecond != 0 && (type.isDatetimeV2() || type.isTimeStampTz())) { + } else if (nanosecond != 0 && (type.isDatetimeV2() || type.isTimeStampTz())) { result = new DateLiteral(year, month, day, hour, minute, second, microsecond, type); } else { result = new DateLiteral(year, month, day, hour, minute, second, type); @@ -256,6 +290,7 @@ public static DateLiteral createDateLiteral(String s, @Nullable Type type) throw if (result.checkRange() || result.checkDate()) { throw new AnalysisException("Datetime value is out of range"); } + result.checkValueValid(); return result; } catch (Exception ex) { throw new AnalysisException("date literal [" + s + "] is invalid: " + ex.getMessage()); @@ -266,6 +301,20 @@ private static int getOrDefault(TemporalAccessor accessor, ChronoField field, in return accessor.isSupported(field) ? accessor.get(field) : defaultValue; } + private static LocalDateTime roundFractionalSecond(long year, long month, long day, long hour, + long minute, long second, long nanosecond, int scale, int nanosecondGuardDigit) { + long factor = (long) Math.pow(10, ScalarType.TIMESTAMP_NS_SCALE - scale); + long roundingOffset = scale == ScalarType.TIMESTAMP_NS_SCALE && nanosecondGuardDigit >= 5 + ? 1 : factor / 2; + long roundedNanosecond = (nanosecond + roundingOffset) / factor * factor; + LocalDateTime result = LocalDateTime.of( + (int) year, (int) month, (int) day, (int) hour, (int) minute, (int) second); + if (roundedNanosecond == 1000000000L) { + return result.plusSeconds(1); + } + return result.withNano((int) roundedNanosecond); + } + private static boolean haveTimeZoneOffset(String arg) { Preconditions.checkArgument(arg.length() > 6); return HAS_OFFSET_PART.matcher(arg.substring(arg.length() - 6)).matches(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToSqlVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToSqlVisitor.java index 3bbbbc902cc9c0..003fca86bfd959 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToSqlVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToSqlVisitor.java @@ -86,6 +86,11 @@ public String visitDateLiteral(DateLiteral expr, ToSqlParams context) { return "'" + expr.getStringValue() + "'"; } + @Override + public String visitTimeStampNsLiteral(TimeStampNsLiteral expr, ToSqlParams context) { + return "'" + expr.getStringValue() + "'"; + } + @Override public String visitTimeV2Literal(TimeV2Literal expr, ToSqlParams context) { return "\"" + expr.getStringValue() + "\""; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToStringValueVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToStringValueVisitor.java index 0c59cf99f5b068..46fe332ccbcff9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToStringValueVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToStringValueVisitor.java @@ -105,6 +105,14 @@ public String visitDateLiteral(DateLiteral expr, StringValueContext ctx) { return value; } + @Override + public String visitTimeStampNsLiteral(TimeStampNsLiteral expr, StringValueContext ctx) { + if (ctx.isInComplexType()) { + return wrapWithQuotes(expr.getStringValue(), ctx); + } + return expr.getStringValue(); + } + @Override public String visitFloatLiteral(FloatLiteral expr, StringValueContext ctx) { String value; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToThriftVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToThriftVisitor.java index 3df252e8fed56b..3f1521ff376459 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToThriftVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToThriftVisitor.java @@ -202,6 +202,18 @@ public Void visitDateLiteral(DateLiteral expr, TExprNode msg) { return null; } + @Override + public Void visitTimeStampNsLiteral(TimeStampNsLiteral expr, TExprNode msg) { + msg.node_type = TExprNodeType.DATE_LITERAL; + msg.date_literal = new TDateLiteral(expr.getStringValue()); + try { + expr.checkValueValid(); + } catch (AnalysisException e) { + LOG.warn("meet invalid value when plan to translate " + expr + " to thrift node"); + } + return null; + } + @Override public Void visitTimeV2Literal(TimeV2Literal expr, TExprNode msg) { msg.node_type = TExprNodeType.TIMEV2_LITERAL; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExprUtils.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExprUtils.java index b9a5d392a349ac..2e0d833f652179 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExprUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExprUtils.java @@ -68,8 +68,9 @@ public static LiteralExpr createLiteral(String value, Type type) throws Analysis case DATETIME: case DATEV2: case DATETIMEV2: + case TIMESTAMP_NS: case TIMESTAMPTZ: - literalExpr = DateLiteralUtils.createDateLiteral(value, type); + literalExpr = DateLiteralUtils.createLiteral(value, type); break; case IPV4: literalExpr = new IPv4Literal(value); @@ -104,6 +105,8 @@ public static LiteralExpr createInfinity(Type type, boolean isMax) throws Analys case DATETIMEV2: case TIMESTAMPTZ: return DateLiteral.createMinValue(type); + case TIMESTAMP_NS: + return TimeStampNsLiteral.createMinValue(); default: throw new AnalysisException("Invalid data type for creating infinity: " + type); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java index 43e1b53badc91c..96421206125a1a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java @@ -36,6 +36,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -100,6 +101,11 @@ public static FunctionIntervalInfo getFunctionIntervalInfo(ArrayList parti public static DateLiteral getRangeEnd(DateLiteral beginTime, FunctionIntervalInfo intervalInfo) throws AnalysisException { + LocalDateTime endTime = getRangeEnd(beginTime.getTimeFormatter(), intervalInfo); + return endTime == null ? null : new DateLiteral(endTime, beginTime.getType()); + } + + private static LocalDateTime getRangeEnd(LocalDateTime beginTime, FunctionIntervalInfo intervalInfo) { String timeUnit = intervalInfo.timeUnit; long interval = intervalInfo.interval; switch (timeUnit) { @@ -161,11 +167,18 @@ public static Map getAddPartitionClauseFromPartitionValu if (partitionType == PartitionType.RANGE) { String beginTime = curPartitionValues.get(0); // have check range type size must be 1 Type partitionColumnType = partitionColumn.get(0).getType(); - DateLiteral beginDateTime = DateLiteralUtils.createDateLiteral(beginTime, partitionColumnType); + LiteralExpr beginDateTime = DateLiteralUtils.createLiteral(beginTime, partitionColumnType); + LocalDateTime beginLocalDateTime = beginDateTime instanceof DateLiteral + ? ((DateLiteral) beginDateTime).getTimeFormatter() + : ((TimeStampNsLiteral) beginDateTime).toLocalDateTime(); partitionName += String.format(DATETIME_NAME_FORMATTER, - beginDateTime.getYear(), beginDateTime.getMonth(), beginDateTime.getDay(), - beginDateTime.getHour(), beginDateTime.getMinute(), beginDateTime.getSecond()); - DateLiteral endDateTime = getRangeEnd(beginDateTime, intervalInfo); + beginLocalDateTime.getYear(), beginLocalDateTime.getMonthValue(), + beginLocalDateTime.getDayOfMonth(), beginLocalDateTime.getHour(), + beginLocalDateTime.getMinute(), beginLocalDateTime.getSecond()); + LocalDateTime endLocalDateTime = getRangeEnd(beginLocalDateTime, intervalInfo); + LiteralExpr endDateTime = beginDateTime instanceof DateLiteral + ? new DateLiteral(endLocalDateTime, beginDateTime.getType()) + : new TimeStampNsLiteral(endLocalDateTime); partitionKeyDesc = createPartitionKeyDescWithRange(beginDateTime, endDateTime, partitionColumnType); } else if (partitionType == PartitionType.LIST) { List> listValues = new ArrayList<>(); @@ -232,8 +245,8 @@ public static Map getAddPartitionClauseFromPartitionValu return result; } - private static PartitionKeyDesc createPartitionKeyDescWithRange(DateLiteral beginDateTime, - DateLiteral endDateTime, Type partitionColumnType) throws AnalysisException { + private static PartitionKeyDesc createPartitionKeyDescWithRange(LiteralExpr beginDateTime, + LiteralExpr endDateTime, Type partitionColumnType) throws AnalysisException { PartitionValue lowerValue = getPartitionFromDate(partitionColumnType, beginDateTime); PartitionValue upperValue = getPartitionFromDate(partitionColumnType, endDateTime); return PartitionKeyDesc.createFixed( @@ -241,7 +254,7 @@ private static PartitionKeyDesc createPartitionKeyDescWithRange(DateLiteral begi Collections.singletonList(upperValue)); } - private static PartitionValue getPartitionFromDate(Type partitionColumnType, DateLiteral dateLiteral) + private static PartitionValue getPartitionFromDate(Type partitionColumnType, LiteralExpr dateLiteral) throws AnalysisException { // check out of range. try { @@ -252,15 +265,19 @@ private static PartitionValue getPartitionFromDate(Type partitionColumnType, Dat return PartitionValue.MAX_VALUE; } + LocalDateTime dateTime = dateLiteral instanceof DateLiteral + ? ((DateLiteral) dateLiteral).getTimeFormatter() + : ((TimeStampNsLiteral) dateLiteral).toLocalDateTime(); String timeString; if (partitionColumnType.isDate() || partitionColumnType.isDateV2()) { - timeString = String.format(DATE_FORMATTER, dateLiteral.getYear(), dateLiteral.getMonth(), - dateLiteral.getDay()); + timeString = String.format(DATE_FORMATTER, dateTime.getYear(), dateTime.getMonthValue(), + dateTime.getDayOfMonth()); } else if (partitionColumnType.isDatetime() || partitionColumnType.isDatetimeV2() + || partitionColumnType.isTimeStampNs() || partitionColumnType.isTimeStampTz()) { timeString = String.format(DATETIME_FORMATTER, - dateLiteral.getYear(), dateLiteral.getMonth(), dateLiteral.getDay(), - dateLiteral.getHour(), dateLiteral.getMinute(), dateLiteral.getSecond()); + dateTime.getYear(), dateTime.getMonthValue(), dateTime.getDayOfMonth(), + dateTime.getHour(), dateTime.getMinute(), dateTime.getSecond()); } else { throw new AnalysisException( "not support range partition with column type : " + partitionColumnType.toString()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnToProtobuf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnToProtobuf.java index d9c17012a64d28..aa0c89a85bd25c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnToProtobuf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnToProtobuf.java @@ -153,6 +153,7 @@ static int getFieldLengthByType(PrimitiveType type, int stringLength) throws Ddl case DATETIME: return 8; case DATETIMEV2: + case TIMESTAMP_NS: case TIMESTAMPTZ: return 8; case FLOAT: diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java index f23954280f3a65..ab6f610003a74f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java @@ -24,6 +24,7 @@ import org.apache.doris.analysis.PartitionDesc; import org.apache.doris.analysis.PartitionValue; import org.apache.doris.analysis.SinglePartitionDesc; +import org.apache.doris.analysis.TimeStampNsLiteral; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; @@ -398,7 +399,7 @@ public static List toPartitionValue(PartitionKey partitionKey) { return partitionKey.getKeys().stream().map(expr -> { if (expr == MaxLiteral.MAX_VALUE) { return PartitionValue.MAX_VALUE; - } else if (expr instanceof DateLiteral) { + } else if (expr instanceof DateLiteral || expr instanceof TimeStampNsLiteral) { return new PartitionValue(expr.getStringValue()); } else if (expr instanceof NullLiteral) { return new PartitionValue("NULL", true); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java index 3832876cd84401..5aaa0d602922eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java @@ -34,8 +34,10 @@ import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral; import org.apache.doris.nereids.trees.expressions.literal.TimestampTzLiteral; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.persist.gson.GsonUtils; @@ -118,7 +120,8 @@ public static PartitionKey createPartitionKey(List keys, List keys) { value = expr.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITH_TABLE); } else { value = expr.getRealValue(); - if (expr instanceof DateLiteral) { - DateLiteral dateLiteral = (DateLiteral) expr; - value = dateLiteral.getStringValue(); + if (expr instanceof DateLiteral || expr instanceof org.apache.doris.analysis.TimeStampNsLiteral) { + value = expr.getStringValue(); } } if (keys.size() - 1 == i) { @@ -571,7 +581,11 @@ public PartitionKey deserialize(JsonElement json, java.lang.reflect.Type typeOfT LOG.warn("Value {} for partition key [type = {}] is invalid! This is a bug exists " + "in Doris 1.2.0 and fixed since Doris 1.2.1. You should create this table " + "again using Doris 1.2.1+ .", key.getStringValue(), type); - ((DateLiteral) key).setMinValue(); + if (key instanceof org.apache.doris.analysis.TimeStampNsLiteral) { + ((org.apache.doris.analysis.TimeStampNsLiteral) key).setMinValue(); + } else { + ((DateLiteral) key).setMinValue(); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/BuildIndexProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/BuildIndexProcDir.java index a0552af614d94e..2f2ca45c2f562d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/BuildIndexProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/BuildIndexProcDir.java @@ -23,6 +23,7 @@ import org.apache.doris.analysis.DateLiteralUtils; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.LimitElement; +import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.analysis.StringLiteral; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Type; @@ -88,7 +89,8 @@ boolean filterResult(String columnName, Comparable element, HashMap tablets) { typeMap.put("DATE", Type.DATE); typeMap.put("DATEV2", Type.DATEV2); typeMap.put("DATETIMEV2", Type.DATETIMEV2); + typeMap.put("TIMESTAMP_NS", Type.TIMESTAMP_NS); typeMap.put("DATETIME", Type.DATETIME); typeMap.put("DECIMAL32", Type.DECIMAL32); typeMap.put("DECIMAL64", Type.DECIMAL64); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 0b56c1a244dda5..308c0b0bb9fdbb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -410,8 +410,10 @@ public static DataProperty analyzeDataProperty(Map properties, f } } else if (key.equalsIgnoreCase(PROPERTIES_STORAGE_COOLDOWN_TIME)) { try { + // Cooldown timestamps are stored in milliseconds, so DATETIMEV2(6) preserves their + // fractional part without imposing the narrower TIMESTAMP_NS epoch range. DateLiteral dateLiteral = DateLiteralUtils.createDateLiteral(value, - ScalarType.getDefaultDateType(Type.DATETIME)); + ScalarType.createDatetimeV2Type(6)); cooldownTimestamp = dateLiteral.unixTimestamp(TimeUtils.getTimeZone()); } catch (AnalysisException e) { LOG.warn("dateLiteral failed, use max cool down time", e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java index 0a4e8f967e310e..2f706a4c660492 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java @@ -78,6 +78,7 @@ public class Util { TYPE_STRING_MAP.put(PrimitiveType.DATETIME, "datetime"); TYPE_STRING_MAP.put(PrimitiveType.DATEV2, "datev2"); TYPE_STRING_MAP.put(PrimitiveType.DATETIMEV2, "datetimev2"); + TYPE_STRING_MAP.put(PrimitiveType.TIMESTAMP_NS, "timestamp_ns"); TYPE_STRING_MAP.put(PrimitiveType.TIMESTAMPTZ, "timestamptz"); TYPE_STRING_MAP.put(PrimitiveType.CHAR, "char(%d)"); TYPE_STRING_MAP.put(PrimitiveType.VARCHAR, "varchar(%d)"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ConnectorColumnConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ConnectorColumnConverter.java index 62598eec708bef..e90209ecb06031 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ConnectorColumnConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ConnectorColumnConverter.java @@ -359,6 +359,8 @@ private static Type convertScalarType(String typeName, int precision, int scale) return ScalarType.createDatetimeV2Type(precision); } return ScalarType.DATETIMEV2; + case "TIMESTAMP_NS": + return ScalarType.createTimeStampNsType(); case "TIMESTAMPTZ": if (precision >= 0) { return ScalarType.createTimeStampTzType(precision); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ExprToConnectorExpressionConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ExprToConnectorExpressionConverter.java index 36cd21e3194084..b79b6f72c1b8eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ExprToConnectorExpressionConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ExprToConnectorExpressionConverter.java @@ -37,6 +37,7 @@ import org.apache.doris.analysis.NullLiteral; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.analysis.TimeStampNsLiteral; import org.apache.doris.analysis.ToSqlParams; import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.MapType; @@ -299,6 +300,8 @@ private static ConnectorExpression convertLiteral(LiteralExpr literal) { return new ConnectorLiteral(ct, ((DecimalLiteral) literal).getValue()); } else if (literal instanceof StringLiteral) { return new ConnectorLiteral(ct, ((StringLiteral) literal).getValue()); + } else if (literal instanceof TimeStampNsLiteral) { + return new ConnectorLiteral(ct, ((TimeStampNsLiteral) literal).toLocalDateTime()); } else if (literal instanceof DateLiteral) { return convertDateLiteral((DateLiteral) literal, ct); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java index bcdc9ccc03bd69..e6b8b56f91bf9d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java @@ -450,11 +450,13 @@ private Type dorisTypeToDoris(JdbcFieldSchema fieldSchema) { case "DATETIMEV2": { int scale = (openParen == -1) ? 0 : Integer.parseInt(upperType.substring(openParen + 1, upperType.length() - 1)); - if (scale > 6) { - scale = 6; + if (scale > ScalarType.MAX_DATETIMEV2_SCALE) { + scale = ScalarType.MAX_DATETIMEV2_SCALE; } return ScalarType.createDatetimeV2Type(scale); } + case "TIMESTAMP_NS": + return ScalarType.createTimeStampNsType(); case "CHAR": case "CHARACTER": return ScalarType.createCharType(fieldSchema.requiredColumnSize()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/StatementSubmitter.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/StatementSubmitter.java index 60b44ec9384bfb..03c85ba771e2fd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/StatementSubmitter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/StatementSubmitter.java @@ -225,7 +225,8 @@ private ExecutionResultSet generateResultSet(ResultSet rs, long startTime, boole for (int i = 1; i <= colNum && (!isCopyStmt || i <= copyResult.length); ++i) { String type = rs.getMetaData().getColumnTypeName(i); if ("DATE".equalsIgnoreCase(type) || "DATETIME".equalsIgnoreCase(type) - || "DATEV2".equalsIgnoreCase(type) || "DATETIMEV2".equalsIgnoreCase(type)) { + || "DATEV2".equalsIgnoreCase(type) || "DATETIMEV2".equalsIgnoreCase(type) + || "TIMESTAMP_NS".equalsIgnoreCase(type)) { row.add(rs.getString(i)); } else { row.add(rs.getObject(i)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/streamresponse/JsonStreamResponse.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/streamresponse/JsonStreamResponse.java index ba77589b710d27..61afcffdcffde6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/streamresponse/JsonStreamResponse.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/streamresponse/JsonStreamResponse.java @@ -154,7 +154,8 @@ private void writeResultSetData(ResultSet rs, JsonWriter jsonWriter, long startT for (int i = 1; i <= colNum; ++i) { String type = rs.getMetaData().getColumnTypeName(i); if ("DATE".equalsIgnoreCase(type) || "DATETIME".equalsIgnoreCase(type) - || "DATEV2".equalsIgnoreCase(type) || "DATETIMEV2".equalsIgnoreCase(type)) { + || "DATEV2".equalsIgnoreCase(type) || "DATETIMEV2".equalsIgnoreCase(type) + || "TIMESTAMP_NS".equalsIgnoreCase(type)) { row.add(rs.getString(i)); } else { row.add(rs.getObject(i)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java index 6a23cabdb93e0a..f563c34f25659b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java @@ -247,7 +247,8 @@ private String dateTimeToStr(DateTimeV2Literal literal, if (partitionColumnType.isDate() || partitionColumnType.isDateV2()) { return String.format(PartitionExprUtil.DATE_FORMATTER, literal.getYear(), literal.getMonth(), literal.getDay()); - } else if (partitionColumnType.isDatetime() || partitionColumnType.isDatetimeV2()) { + } else if (partitionColumnType.isDatetime() || partitionColumnType.isDatetimeV2() + || partitionColumnType.isTimeStampNs()) { return String.format(PartitionExprUtil.DATETIME_FORMATTER, literal.getYear(), literal.getMonth(), literal.getDay(), literal.getHour(), literal.getMinute(), literal.getSecond()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java index 4469830f320eb3..e67f843acefcdc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java @@ -18,6 +18,7 @@ package org.apache.doris.mysql; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.MysqlColType; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; @@ -187,7 +188,7 @@ public void writeField(FieldInfo fieldInfo, Type type) { // Column length: four byte integer writeInt4(getMysqlTypeLength(type)); // Column type: one byte integer - writeInt1(type.getPrimitiveType().toMysqlType().getCode()); + writeInt1(getMysqlResultType(type).getCode()); // Flags: two byte integer writeInt2(getMysqlFlags(type)); // Decimals: one byte integer @@ -216,7 +217,7 @@ public void writeField(String db, String table, Column column, boolean sendDefau // TODO(zhaochun): fix Column length: four byte integer writeInt4(getMysqlTypeLength(column.getType())); // Column type: one byte integer - writeInt1(column.getDataType().toMysqlType().getCode()); + writeInt1(getMysqlResultType(column.getType()).getCode()); // Flags: two byte integer writeInt2(getMysqlFlags(column.getType())); // Decimals: one byte integer @@ -251,7 +252,7 @@ public void writeField(String colName, Type type) { // Column length: four byte integer writeInt4(getMysqlTypeLength(type)); // Column type: one byte integer - writeInt1(type.getPrimitiveType().toMysqlType().getCode()); + writeInt1(getMysqlResultType(type).getCode()); // Flags: two byte integer writeInt2(getMysqlFlags(type)); // Decimals: one byte integer @@ -268,7 +269,7 @@ public void writeField(String colName, Type type) { * @param type * @return */ - private int getMysqlTypeLength(Type type) { + int getMysqlTypeLength(Type type) { switch (type.getPrimitiveType()) { // MySQL use Tinyint(1) to represent boolean case BOOLEAN: @@ -291,6 +292,9 @@ private int getMysqlTypeLength(Type type) { case TIMESTAMPTZ: // yyyy-MM-dd HH:mm:ss[.ffffff]+HH:mm return 32; + case TIMESTAMP_NS: + // yyyy-MM-dd HH:mm:ss.nnnnnnnnn + return 29; case DATETIME: case DATETIMEV2: { if (type.getPrimitiveType().isTimeType()) { @@ -332,6 +336,9 @@ private int getMysqlTypeLength(Type type) { // this is used for decimal scale public int getMysqlDecimals(Type type) { + if (isTimestampNs(type)) { + return 0; + } switch (type.getPrimitiveType()) { case DECIMALV2: case DECIMAL32: @@ -349,6 +356,17 @@ public int getMysqlDecimals(Type type) { } } + private MysqlColType getMysqlResultType(Type type) { + // MySQL's binary temporal value contains at most microseconds. TIMESTAMP_NS is + // serialized by BE as a length-encoded string so the metadata must advertise a string too. + return isTimestampNs(type) ? MysqlColType.MYSQL_TYPE_STRING + : type.getPrimitiveType().toMysqlType(); + } + + private boolean isTimestampNs(Type type) { + return type.isTimeStampNs(); + } + // see https://github.com/mysql/mysql-server/blob/trunk/include/mysql_com.h#L161 private int getMysqlFlags(Type type) { int flags = 0; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java index b6b3127994682d..3f92c072afbb87 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java @@ -50,6 +50,7 @@ import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.types.TimeV2Type; import org.apache.doris.nereids.types.TinyIntType; @@ -150,6 +151,11 @@ public class CheckCast implements ExpressionPatternRuleFactory { allowedTypes.add(TimeStampTzType.class); strictCastWhiteList.put(DateTimeV2Type.class, allowedTypes); + // TimestampNs + allowedTypes = Sets.newHashSet(); + allowToStringLikeType(allowedTypes); + strictCastWhiteList.put(TimeStampNsType.class, allowedTypes); + // timestamp tz allowedTypes = Sets.newHashSet(); allowedTypes.add(DateTimeV2Type.class); @@ -180,6 +186,7 @@ public class CheckCast implements ExpressionPatternRuleFactory { allowedTypes.add(IPv6Type.class); allowedTypes.add(VarBinaryType.class); allowedTypes.add(TimeStampTzType.class); + allowedTypes.add(TimeStampNsType.class); allowToComplexType(allowedTypes); allowedTypes.remove(HllType.class); allowedTypes.remove(BitmapType.class); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java index 3f6812405e864b..5db22ad2c7136b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java @@ -68,6 +68,7 @@ import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.expressions.literal.StructLiteral; +import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.DataType; @@ -76,6 +77,7 @@ import org.apache.doris.nereids.types.MapType; import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.proto.InternalService; import org.apache.doris.proto.InternalService.PConstantExprResult; import org.apache.doris.proto.Types.PScalarType; @@ -107,6 +109,7 @@ import java.time.DateTimeException; import java.time.LocalDate; import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -125,6 +128,7 @@ public class FoldConstantRuleOnBE implements ExpressionPatternRuleFactory { public static final FoldConstantRuleOnBE INSTANCE = new FoldConstantRuleOnBE(); private static final Logger LOG = LogManager.getLogger(FoldConstantRuleOnBE.class); + private static final long NANOS_PER_SECOND = 1_000_000_000L; @Override public List> buildRules() { @@ -472,17 +476,26 @@ public static List getResultExpression(DataType type, PValues resultCon res.add(literal); } } else if (type.isDateTimeV2Type()) { - int num = resultContent.getUint64ValueCount(); - for (int i = 0; i < num; ++i) { - long uint64Value = resultContent.getUint64Value(i); - LocalDateTime dateTimeV2 = convertToJavaDateTimeV2(uint64Value); - if (dateTimeV2 == null && resultContent.hasHasNull()) { - res.add(new NullLiteral(type)); - } else { - Literal literal = new DateTimeV2Literal((DateTimeV2Type) type, dateTimeV2.getYear(), - dateTimeV2.getMonthValue(), dateTimeV2.getDayOfMonth(), dateTimeV2.getHour(), - dateTimeV2.getMinute(), dateTimeV2.getSecond(), dateTimeV2.getNano() / 1000); - res.add(literal); + DateTimeV2Type dateTimeV2Type = (DateTimeV2Type) type; + if (dateTimeV2Type instanceof TimeStampNsType) { + // TIMESTAMP_NS is sent as signed epoch nanoseconds, unlike the packed uint64 civil + // layout of DATETIMEV2. + int num = resultContent.getInt64ValueCount(); + for (int i = 0; i < num; ++i) { + LocalDateTime dateTimeV2 = convertEpochNanosToJavaDateTimeV2( + resultContent.getInt64Value(i)); + res.add(createDateTimeLikeLiteral(dateTimeV2Type, dateTimeV2)); + } + } else { + int num = resultContent.getUint64ValueCount(); + for (int i = 0; i < num; ++i) { + long uint64Value = resultContent.getUint64Value(i); + LocalDateTime dateTimeV2 = convertToJavaDateTimeV2(uint64Value); + if (dateTimeV2 == null && resultContent.hasHasNull()) { + res.add(new NullLiteral(type)); + } else { + res.add(createDateTimeLikeLiteral(dateTimeV2Type, dateTimeV2)); + } } } } else if (type.isDateV2Type()) { @@ -688,6 +701,22 @@ private static LocalDateTime convertToJavaDateTimeV2(long time) { } } + private static LocalDateTime convertEpochNanosToJavaDateTimeV2(long epochNanos) { + long epochSecond = Math.floorDiv(epochNanos, NANOS_PER_SECOND); + int nanoOfSecond = (int) Math.floorMod(epochNanos, NANOS_PER_SECOND); + return LocalDateTime.ofEpochSecond(epochSecond, nanoOfSecond, ZoneOffset.UTC); + } + + private static Literal createDateTimeLikeLiteral( + DateTimeV2Type type, LocalDateTime dateTime) { + if (type instanceof TimeStampNsType) { + return TimeStampNsLiteral.fromJavaDateType(dateTime); + } + return new DateTimeV2Literal(type, dateTime.getYear(), dateTime.getMonthValue(), + dateTime.getDayOfMonth(), dateTime.getHour(), dateTime.getMinute(), + dateTime.getSecond(), dateTime.getNano() / 1000); + } + private static LocalDate convertToJavaDateV2(int date) { int year = date >> 9; int month = (date >> 5) & 0XF; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java index 5338bf7df9b2ef..97911650afb0a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java @@ -82,8 +82,6 @@ import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.expressions.literal.ComparableLiteral; import org.apache.doris.nereids.trees.expressions.literal.DateLiteral; -import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; -import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; @@ -702,10 +700,10 @@ public Expression visitDate(Date date, ExpressionRewriteContext context) { } DataType dataType = child.getDataType(); if (dataType.isDateTimeType()) { - DateTimeLiteral dateTimeLiteral = (DateTimeLiteral) child; + DateLiteral dateTimeLiteral = (DateLiteral) child; return new DateLiteral(dateTimeLiteral.getYear(), dateTimeLiteral.getMonth(), dateTimeLiteral.getDay()); } else if (dataType.isDateTimeV2Type()) { - DateTimeV2Literal dateTimeLiteral = (DateTimeV2Literal) child; + DateLiteral dateTimeLiteral = (DateLiteral) child; return new DateV2Literal(dateTimeLiteral.getYear(), dateTimeLiteral.getMonth(), dateTimeLiteral.getDay()); } return date; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java index 335fe78307180f..03c61517e86f4c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java @@ -48,6 +48,7 @@ import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.NumericLiteral; import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral; +import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.DataType; @@ -58,6 +59,7 @@ import org.apache.doris.nereids.types.DecimalV3Type; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.SmallIntType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.types.coercion.DateLikeType; import org.apache.doris.nereids.types.coercion.IntegralType; @@ -183,6 +185,10 @@ private static Expression processDateLikeTypeCoercion(ComparisonPredicate cp, Ex if (cast.child().getDataType() instanceof DateTimeType || cast.child().getDataType() instanceof DateTimeV2Type) { // right is datetime + if (right instanceof TimeStampNsLiteral) { + return processDateTimeLikeComparisonPredicateTimeStampNsLiteral( + cp, cast.child(), (TimeStampNsLiteral) right); + } if (right instanceof DateTimeV2Literal) { return processDateTimeLikeComparisonPredicateDateTimeV2Literal( cp, cast.child(), (DateTimeV2Literal) right); @@ -203,6 +209,37 @@ private static Expression processDateLikeTypeCoercion(ComparisonPredicate cp, Ex return cp; } + private static Expression processDateTimeLikeComparisonPredicateTimeStampNsLiteral( + ComparisonPredicate comparisonPredicate, Expression left, TimeStampNsLiteral right) { + DataType leftType = left.getDataType(); + if (!(leftType instanceof DateTimeType) && !(leftType instanceof DateTimeV2Type)) { + return comparisonPredicate; + } + int toScale = leftType instanceof DateTimeV2Type ? ((DateTimeV2Type) leftType).getScale() : 0; + if (toScale >= TimeStampNsType.SCALE) { + return comparisonPredicate; + } + + DateTimeV2Literal rounded; + if (comparisonPredicate instanceof EqualTo || comparisonPredicate instanceof NullSafeEqual) { + rounded = right.roundFloorToDateTimeV2(toScale); + if (rounded.getNanoSecond() != right.getNanoSecond()) { + return comparisonPredicate instanceof NullSafeEqual + ? BooleanLiteral.FALSE : ExpressionUtils.falseOrNull(left); + } + } else if (comparisonPredicate instanceof GreaterThan + || comparisonPredicate instanceof LessThanEqual) { + rounded = right.roundFloorToDateTimeV2(toScale); + } else if (comparisonPredicate instanceof LessThan + || comparisonPredicate instanceof GreaterThanEqual) { + rounded = right.roundCeilingToDateTimeV2(toScale); + } else { + return comparisonPredicate; + } + Expression newRight = leftType instanceof DateTimeType ? migrateToDateTime(rounded) : rounded; + return comparisonPredicate.withChildren(left, newRight); + } + // process cast(datetime as datetime) cmp datetime private static Expression processDateTimeLikeComparisonPredicateDateTimeV2Literal( ComparisonPredicate comparisonPredicate, Expression left, DateTimeV2Literal right) { @@ -214,9 +251,9 @@ private static Expression processDateTimeLikeComparisonPredicateDateTimeV2Litera DateTimeV2Type rightType = right.getDataType(); if (toScale < rightType.getScale()) { if (comparisonPredicate instanceof EqualTo) { - long originValue = right.getMicroSecond(); + long originValue = right.getNanoSecond(); right = right.roundFloor(toScale); - if (right.getMicroSecond() != originValue) { + if (right.getNanoSecond() != originValue) { // TODO: the ideal way is to return an If expr like: // return new If(new IsNull(left), new NullLiteral(BooleanType.INSTANCE), // BooleanLiteral.of(false)); @@ -226,9 +263,9 @@ private static Expression processDateTimeLikeComparisonPredicateDateTimeV2Litera return ExpressionUtils.falseOrNull(left); } } else if (comparisonPredicate instanceof NullSafeEqual) { - long originValue = right.getMicroSecond(); + long originValue = right.getNanoSecond(); right = right.roundFloor(toScale); - if (right.getMicroSecond() != originValue) { + if (right.getNanoSecond() != originValue) { return BooleanLiteral.of(false); } } else if (comparisonPredicate instanceof GreaterThan @@ -284,21 +321,50 @@ private static Expression processDateTimeLikeComparisonPredicateDateLiteral( return comparisonPredicate; } - DateTimeLiteral lowBound = null; - DateTimeLiteral upBound = null; + DateLiteral lowBound; + DateLiteral upBound; if (leftType instanceof DateTimeType) { lowBound = new DateTimeLiteral(right.getYear(), right.getMonth(), right.getDay(), 0, 0, 0); upBound = new DateTimeLiteral(right.getYear(), right.getMonth(), right.getDay(), 23, 59, 59); } else { - long upMicroSecond = 0; + long upperFraction = 0; for (int i = 0; i < ((DateTimeV2Type) leftType).getScale(); i++) { - upMicroSecond = 10 * upMicroSecond + 9; + upperFraction = 10 * upperFraction + 9; + } + DateTimeV2Type dateTimeV2Type = (DateTimeV2Type) leftType; + int scale = dateTimeV2Type.getScale(); + // DateTimeV2Literal accepts a 6-digit microsecond value for DATETIMEV2, but a + // 9-digit nanosecond value for TIMESTAMP_NS. + boolean isTimestampNs = dateTimeV2Type instanceof TimeStampNsType; + int fractionalWidth = isTimestampNs ? TimeStampNsType.SCALE : DateTimeV2Type.MAX_SCALE; + upperFraction *= (long) Math.pow(10, fractionalWidth - scale); + if (isTimestampNs) { + // The signed Int64 epoch-nanosecond range starts and ends in the middle of its + // boundary dates. Clamp the rewritten whole-day interval to the exact min/max on + // those dates, and simplify dates completely outside the range without constructing + // an invalid midnight or end-of-day literal. + TimeStampNsLiteral minValue = TimeStampNsLiteral.getMinValue(); + TimeStampNsLiteral maxValue = TimeStampNsLiteral.getMaxValue(); + DateV2Literal minDate = new DateV2Literal( + minValue.getYear(), minValue.getMonth(), minValue.getDay()); + DateV2Literal maxDate = new DateV2Literal( + maxValue.getYear(), maxValue.getMonth(), maxValue.getDay()); + int compareToMin = right.compareTo(minDate); + int compareToMax = right.compareTo(maxDate); + if (compareToMin < 0 || compareToMax > 0) { + return simplifyDateComparisonOutsideRange( + comparisonPredicate, left, compareToMin < 0); + } + lowBound = compareToMin == 0 ? minValue : new TimeStampNsLiteral( + right.getYear(), right.getMonth(), right.getDay(), 0, 0, 0, 0); + upBound = compareToMax == 0 ? maxValue : new TimeStampNsLiteral( + right.getYear(), right.getMonth(), right.getDay(), 23, 59, 59, upperFraction); + } else { + lowBound = new DateTimeV2Literal(dateTimeV2Type, + right.getYear(), right.getMonth(), right.getDay(), 0, 0, 0, 0); + upBound = new DateTimeV2Literal(dateTimeV2Type, + right.getYear(), right.getMonth(), right.getDay(), 23, 59, 59, upperFraction); } - upMicroSecond *= (int) Math.pow(10, 6 - ((DateTimeV2Type) leftType).getScale()); - lowBound = new DateTimeV2Literal((DateTimeV2Type) leftType, - right.getYear(), right.getMonth(), right.getDay(), 0, 0, 0, 0); - upBound = new DateTimeV2Literal((DateTimeV2Type) leftType, - right.getYear(), right.getMonth(), right.getDay(), 23, 59, 59, upMicroSecond); } if (comparisonPredicate instanceof GreaterThanEqual || comparisonPredicate instanceof LessThan) { @@ -321,17 +387,43 @@ private static Expression processDateTimeLikeComparisonPredicateDateLiteral( return comparisonPredicate; } + /** + * Simplify a date/datetime comparison whose right literal is outside the range representable by + * the left expression. For every non-null left value, the result is constant and depends only on + * the comparison direction and whether the literal is below the minimum or above the maximum. + * Null-safe equality is always false; other predicates return true-or-null or false-or-null to + * preserve SQL three-valued logic when the left expression is nullable. + * + * @param comparisonPredicate comparison to simplify + * @param left left expression whose type defines the representable range + * @param rightBeforeMin true if the right literal is below that range; false if it is above + * @return the constant-equivalent expression with the original NULL semantics + */ + private static Expression simplifyDateComparisonOutsideRange( + ComparisonPredicate comparisonPredicate, Expression left, boolean rightBeforeMin) { + if (comparisonPredicate instanceof NullSafeEqual) { + return BooleanLiteral.FALSE; + } + if (comparisonPredicate instanceof EqualTo) { + return ExpressionUtils.falseOrNull(left); + } + boolean alwaysTrue = rightBeforeMin + ? comparisonPredicate instanceof GreaterThan || comparisonPredicate instanceof GreaterThanEqual + : comparisonPredicate instanceof LessThan || comparisonPredicate instanceof LessThanEqual; + return alwaysTrue ? ExpressionUtils.trueOrNull(left) : ExpressionUtils.falseOrNull(left); + } + // process cast(date as datetime/date) cmp datetime/date private static Expression processDateLikeComparisonPredicateDateLiteral( ComparisonPredicate comparisonPredicate, Expression left, DateLiteral right) { if (!(left.getDataType() instanceof DateType) && !(left.getDataType() instanceof DateV2Type)) { return comparisonPredicate; } - if (right instanceof DateTimeLiteral) { - DateTimeLiteral dateTimeLiteral = (DateTimeLiteral) right; - right = migrateToDateV2(dateTimeLiteral); - if (dateTimeLiteral.getHour() != 0 || dateTimeLiteral.getMinute() != 0 - || dateTimeLiteral.getSecond() != 0 || dateTimeLiteral.getMicroSecond() != 0) { + if (right.getDataType().isDateTimeType() || right.getDataType().isDateTimeV2Type() + || right.getDataType().isTimeStampTzType()) { + boolean isMidnight = right.isMidnight(); + right = new DateV2Literal(right.getYear(), right.getMonth(), right.getDay()); + if (!isMidnight) { if (comparisonPredicate instanceof EqualTo) { return ExpressionUtils.falseOrNull(left); } else if (comparisonPredicate instanceof NullSafeEqual) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyInPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyInPredicate.java index b66db8e66f0d03..1176881719b455 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyInPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyInPredicate.java @@ -26,6 +26,7 @@ import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal; import org.apache.doris.nereids.types.DateTimeV2Type; +import org.apache.doris.nereids.types.TimeStampNsType; import com.google.common.collect.ImmutableList; @@ -61,6 +62,7 @@ && canLosslessConvertToDateV2Literal((DateTimeV2Literal) literal))) { return expr.withChildren(children.build()); } } else if (cast.child().getDataType().isDateTimeV2Type() + && !(cast.child().getDataType() instanceof TimeStampNsType) && expr.child(1) instanceof DateTimeV2Literal) { List literals = expr.children().subList(1, expr.children().size()); DateTimeV2Type castType = (DateTimeV2Type) cast.getDataType(); @@ -93,7 +95,7 @@ && canLosslessConvertToLowScaleLiteral( */ private static boolean canLosslessConvertToDateV2Literal(DateTimeV2Literal literal) { return (literal.getHour() | literal.getMinute() | literal.getSecond() - | literal.getMicroSecond()) == 0L; + | literal.getNanoSecond()) == 0L; } private static DateV2Literal convertToDateV2Literal(DateTimeV2Literal literal) { @@ -101,7 +103,7 @@ private static DateV2Literal convertToDateV2Literal(DateTimeV2Literal literal) { } private static boolean canLosslessConvertToLowScaleLiteral(DateTimeV2Literal literal, int targetScale) { - long scaleFactor = (long) Math.pow(10, DateTimeV2Type.MAX_SCALE - targetScale); - return literal.getMicroSecond() % scaleFactor == 0; + long scaleFactor = (long) Math.pow(10, TimeStampNsType.SCALE - targetScale); + return literal.getNanoSecond() % scaleFactor == 0; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java index 482f7872be6015..c14628ca3b0b14 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java @@ -21,6 +21,7 @@ import org.apache.doris.analysis.DateLiteralUtils; import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.analysis.TimeStampNsLiteral; import org.apache.doris.nereids.stats.FilterEstimation.EstimationContext; import org.apache.doris.nereids.trees.expressions.And; import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; @@ -475,7 +476,7 @@ private Statistics estimateColumnToConstant(ComparisonPredicate cp, EstimationCo return estimateColumnEqualToConstant(cp, statsForLeft, statsForRight, context); } else { // literal Map used to covert dateLiteral back to stringLiteral - Map literalMap = new HashMap<>(); + Map literalMap = new HashMap<>(); DataType compareType = cp.left().getDataType(); Optional statsForLeftMayConvertedOpt = tryConvertStringColStatsToDateColStats(statsForLeft, literalMap); @@ -523,7 +524,7 @@ private Statistics estimateColumnToConstant(ComparisonPredicate cp, EstimationCo } private ColumnStatistic convertDateColStatsToStringColStats(ColumnStatistic colStats, - Map literalMap) { + Map literalMap) { ColumnStatisticBuilder builder = new ColumnStatisticBuilder(colStats); if (colStats.minExpr != null) { builder.setMinExpr(literalMap.get(colStats.minExpr)) @@ -553,18 +554,18 @@ private ColumnStatistic convertDateColStatsToStringColStats(ColumnStatistic colS } private Optional tryConvertStringColStatsToDateColStats(ColumnStatistic colStats, - Map literalMap) { + Map literalMap) { if (colStats.minExpr == null || colStats.maxExpr == null) { return Optional.empty(); } if (!(colStats.minExpr instanceof StringLiteral) || !(colStats.maxExpr instanceof StringLiteral)) { return Optional.empty(); } - Optional newMinExpr = tryConvertStrLiteralToDateLiteral(colStats.minExpr); + Optional newMinExpr = tryConvertStrLiteralToDateLiteral(colStats.minExpr); if (!newMinExpr.isPresent()) { return Optional.empty(); } - Optional newMaxExpr = tryConvertStrLiteralToDateLiteral(colStats.maxExpr); + Optional newMaxExpr = tryConvertStrLiteralToDateLiteral(colStats.maxExpr); if (!newMaxExpr.isPresent()) { return Optional.empty(); } @@ -578,7 +579,10 @@ private Optional tryConvertStringColStatsToDateColStats(ColumnS if (colStats.hotValues != null) { for (Literal oneHot : colStats.hotValues.keySet()) { try { - DateTimeLiteral oneHotDate = new DateTimeLiteral(oneHot.getStringValue()); + Literal oneHotDate = newMinExpr.get().getType().isTimeStampNs() + ? new org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral( + oneHot.getStringValue()) + : new DateTimeLiteral(oneHot.getStringValue()); newHotValues.put(oneHotDate, colStats.hotValues.get(oneHot)); } catch (Exception e) { @@ -588,15 +592,15 @@ private Optional tryConvertStringColStatsToDateColStats(ColumnS } ColumnStatisticBuilder builder = new ColumnStatisticBuilder(colStats); - return Optional.of(builder.setMinValue(newMinExpr.get().getDoubleValueAsDateTime()) + return Optional.of(builder.setMinValue(getDoubleValueAsDateTime(newMinExpr.get())) .setMinExpr(newMinExpr.get()) - .setMaxValue(newMaxExpr.get().getDoubleValueAsDateTime()) + .setMaxValue(getDoubleValueAsDateTime(newMaxExpr.get())) .setMaxExpr(newMaxExpr.get()) .setHotValues(newHotValues.isEmpty() ? null : newHotValues) .build()); } - private Optional tryConvertStrLiteralToDateLiteral(LiteralExpr literal) { + private Optional tryConvertStrLiteralToDateLiteral(LiteralExpr literal) { if (literal == null) { return Optional.empty(); } @@ -604,9 +608,9 @@ private Optional tryConvertStrLiteralToDateLiteral(LiteralExpr lite return Optional.empty(); } - DateLiteral dt = null; + LiteralExpr dt = null; try { - dt = DateLiteralUtils.createDateLiteral(literal.getStringValue(), null); + dt = DateLiteralUtils.createLiteral(literal.getStringValue(), null); dt.checkValueValid(); } catch (Exception e) { // ignore @@ -614,6 +618,15 @@ private Optional tryConvertStrLiteralToDateLiteral(LiteralExpr lite return dt == null ? Optional.empty() : Optional.of(dt); } + private double getDoubleValueAsDateTime(LiteralExpr literal) { + if (literal instanceof DateLiteral) { + return ((DateLiteral) literal).getDoubleValueAsDateTime(); + } + Preconditions.checkState(literal instanceof TimeStampNsLiteral, + "Expected a date-like literal, but got %s", literal.getClass()); + return ((TimeStampNsLiteral) literal).getDoubleValueAsDateTime(); + } + private Statistics estimateColumnEqualToConstant(ComparisonPredicate cp, ColumnStatistic statsForLeft, ColumnStatistic statsForRight, EstimationContext context) { double selectivity = DEFAULT_ISNULL_SELECTIVITY; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java index 3d1f5017be3b14..6082b71447cf76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java @@ -254,14 +254,14 @@ static Result normalize(String s) { sb.append(":00"); } - // parse MicroSecond - // Keep up to 7 digits at most, 7th digit is use for overflow. + // Keep one more digit than Java's NANO_OF_SECOND supports. The tenth digit is the guard + // digit used when a DATETIMEV2(9) literal is rounded. int j = i; if (partNumber == 6 && i < s.length() && s.charAt(i) == '.') { sb.append(s.charAt(i)); i += 1; while (i < s.length() && Character.isDigit(s.charAt(i))) { - if (i - j <= 7) { + if (i - j <= DateUtils.NANOSECOND_SCALE + 1) { sb.append(s.charAt(i)); } i += 1; @@ -329,6 +329,7 @@ public static Result parseDateTime(String s } if (!containsPunctuation) { s = normalizeBasic(s); + s = DateUtils.truncateFractionalSecondForJavaParser(s); // mysql reject "20200219 010101" "200219 010101", can't use ' ' spilt basic date time. if (!s.contains("T")) { @@ -344,6 +345,7 @@ public static Result parseDateTime(String s return normalizeResult.cast(); } s = normalizeResult.get(); + s = DateUtils.truncateFractionalSecondForJavaParser(s); if (!s.contains(" ")) { dateTime = DateTimeFormatterUtils.ZONE_DATE_FORMATTER.parse(s); @@ -397,7 +399,7 @@ private static boolean checkDatetime(TemporalAccessor dateTime) { return DateUtils.getOrDefault(dateTime, ChronoField.HOUR_OF_DAY) != 0 || DateUtils.getOrDefault(dateTime, ChronoField.MINUTE_OF_HOUR) != 0 || DateUtils.getOrDefault(dateTime, ChronoField.SECOND_OF_MINUTE) != 0 - || DateUtils.getOrDefault(dateTime, ChronoField.MICRO_OF_SECOND) != 0; + || DateUtils.getOrDefault(dateTime, ChronoField.NANO_OF_SECOND) != 0; } @Override @@ -472,10 +474,8 @@ public int compareTo(ComparableLiteral other) { return cmp; } - long thisMicrosecond = this instanceof DateTimeV2Literal ? ((DateTimeV2Literal) this).getMicroSecond() : 0L; - long otherMicrosecond = other instanceof DateTimeV2Literal - ? ((DateTimeV2Literal) other).getMicroSecond() : 0L; - return Long.compare(thisMicrosecond, otherMicrosecond); + return Long.compare(getFractionalSecondInNanoseconds(), + ((DateLiteral) other).getFractionalSecondInNanoseconds()); } if (other instanceof NullLiteral) { return 1; @@ -500,6 +500,21 @@ public long getDay() { return day; } + /** Return the time of day in nanoseconds; DATE literals have no time part. */ + public long getTimePartInNanoseconds() { + return 0; + } + + /** Return the fractional second in nanoseconds; DATE literals have no fractional part. */ + public long getFractionalSecondInNanoseconds() { + return 0; + } + + /** DATE values represent the beginning of a day. Datetime subclasses override this as needed. */ + public boolean isMidnight() { + return true; + } + public int getDayOfYear() { if (year == 0 && month == 3 && (day == 1 || day == 2)) { return toJavaDateType().getDayOfYear() - 1; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java index c50f98b45a1119..caf185c4f9907b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java @@ -103,6 +103,7 @@ public DateTimeLiteral(DateLikeType dataType, long year, long month, long day, this.day = day; } + @Override public boolean isMidnight() { return hour == 0 && minute == 0 && second == 0 && microSecond == 0; } @@ -319,6 +320,16 @@ public long timePartToMicroSecond() { return ((hour * 60L + minute) * 60L + second) * 1000L * 1000L + microSecond; } + @Override + public long getTimePartInNanoseconds() { + return timePartToMicroSecond() * 1000L; + } + + @Override + public long getFractionalSecondInNanoseconds() { + return microSecond * 1000L; + } + @Override public double getDouble() { return (double) getValue(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java index bfdf1e3be6eba5..82c994b65c9b7b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java @@ -30,6 +30,8 @@ import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.qe.ConnectContext; +import com.google.common.base.Preconditions; + import java.time.LocalDateTime; import java.util.Objects; @@ -46,7 +48,7 @@ public DateTimeV2Literal(String s) { } public DateTimeV2Literal(DateTimeV2Type dateType, String s) { - super(dateType, s); + super(requireDateTimeV2Type(dateType), s); roundMicroSecond(dateType.getScale()); } @@ -60,10 +62,16 @@ public DateTimeV2Literal(long year, long month, long day, long hour, long minute public DateTimeV2Literal(DateTimeV2Type dateType, long year, long month, long day, long hour, long minute, long second, long microSecond) { - super(dateType, year, month, day, hour, minute, second, microSecond); + super(requireDateTimeV2Type(dateType), year, month, day, hour, minute, second, microSecond); roundMicroSecond(dateType.getScale()); } + private static DateTimeV2Type requireDateTimeV2Type(DateTimeV2Type dateType) { + Preconditions.checkArgument(dateType.getClass() == DateTimeV2Type.class, + "Use TimeStampNsLiteral for TIMESTAMP_NS values"); + return dateType; + } + /** Date difference rounded toward zero by time part. */ public static long dateDiffInDaysRoundToZeroByTime(DateLiteral lhs, DateLiteral rhs) { long days = DateV2Literal.dateDiffInDays(lhs, rhs); @@ -421,6 +429,11 @@ public int getScale() { return ((DateTimeV2Type) dataType).getScale(); } + /** Return DATETIMEV2's microsecond fraction in the common nanosecond comparison unit. */ + public long getNanoSecond() { + return microSecond * 1000L; + } + public int commonScale(DateTimeV2Literal other) { return (int) Math.max(getScale(), other.getScale()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java index 1c368179005be3..b252f30a86a60c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java @@ -335,11 +335,22 @@ public static Literal fromLegacyLiteral(LiteralExpr literalExpr, Type type) { } case DATETIMEV2: { org.apache.doris.analysis.DateLiteral dateLiteral = (org.apache.doris.analysis.DateLiteral) literalExpr; + DateTimeV2Type dateTimeV2Type = (DateTimeV2Type) DateType.fromCatalogType(type); + long fractionalSecond = dateLiteral.getMicrosecond(); return new DateTimeV2Literal( - (DateTimeV2Type) DateType.fromCatalogType(type), + dateTimeV2Type, dateLiteral.getYear(), dateLiteral.getMonth(), dateLiteral.getDay(), dateLiteral.getHour(), dateLiteral.getMinute(), dateLiteral.getSecond(), - dateLiteral.getMicrosecond() + fractionalSecond + ); + } + case TIMESTAMP_NS: { + org.apache.doris.analysis.TimeStampNsLiteral timestampNsLiteral + = (org.apache.doris.analysis.TimeStampNsLiteral) literalExpr; + return new TimeStampNsLiteral( + timestampNsLiteral.getYear(), timestampNsLiteral.getMonth(), timestampNsLiteral.getDay(), + timestampNsLiteral.getHour(), timestampNsLiteral.getMinute(), timestampNsLiteral.getSecond(), + timestampNsLiteral.getNanosecond() ); } case TIMESTAMPTZ: { @@ -628,7 +639,8 @@ private static Literal handleDateTimeLiteral(ByteBuffer data) { microsecond = data.getInt(); } if (Config.enable_date_conversion) { - return new DateTimeV2Literal(DateTimeV2Type.MAX, year, month, day, hour, minute, second, microsecond); + return new DateTimeV2Literal(DateTimeV2Type.MAX, + year, month, day, hour, minute, second, microsecond); } return new DateTimeLiteral(DateTimeType.INSTANCE, year, month, day, hour, minute, second, microsecond); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/NumericLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/NumericLiteral.java index c871a7dc03cc7a..802721b15b36db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/NumericLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/NumericLiteral.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.DateType; import org.apache.doris.nereids.types.DateV2Type; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import java.math.BigDecimal; @@ -154,21 +155,24 @@ protected String getDateTimeString(long value) { } protected Expression getDateLikeLiteral(String s, DataType targetType) { - DateTimeV2Literal l; + DateTimeV2Literal dateTime; try { - l = new DateTimeV2Literal(DateTimeV2Type.MAX, s); + dateTime = new DateTimeV2Literal(DateTimeV2Type.MAX, s); } catch (AnalysisException e) { throw new CastException(e.getMessage(), e); } if (targetType instanceof DateType) { - return new DateLiteral(l.getYear(), l.getMonth(), l.getDay()); + return new DateLiteral(dateTime.getYear(), dateTime.getMonth(), dateTime.getDay()); } if (targetType instanceof DateV2Type) { - return new DateV2Literal(l.getYear(), l.getMonth(), l.getDay()); + return new DateV2Literal(dateTime.getYear(), dateTime.getMonth(), dateTime.getDay()); } if (targetType instanceof DateTimeType) { return new DateTimeLiteral(s); } + if (targetType instanceof TimeStampNsType) { + return new TimeStampNsLiteral(s); + } if (targetType instanceof DateTimeV2Type) { return new DateTimeV2Literal((DateTimeV2Type) targetType, s); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java index afe473ccf75ca7..e9cbf6197ca931 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.expressions.literal.format.DateTimeChecker; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.DateTimeV2Type; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.types.TimeV2Type; import org.apache.doris.qe.SessionVariable; @@ -144,7 +145,8 @@ protected Expression uncheckedCastTo(DataType targetType) throws AnalysisExcepti if (DateTimeChecker.hasTimeZone(value)) { return new TimestampTzLiteral(timeStampTzType, value); } - DateTimeV2Literal datetime = (DateTimeV2Literal) castToDateTime(DateTimeV2Type.MAX, strictCast); + DateTimeV2Literal datetime = (DateTimeV2Literal) castToDateTime( + DateTimeV2Type.MAX, strictCast); return TimestampTzLiteral.fromSessionTimeZone(timeStampTzType, datetime); } else if (targetType.isDateTimeV2Type()) { return castToDateTime(targetType, strictCast); @@ -379,6 +381,12 @@ protected DateLiteral getDateTimeLiteral(String year, String month, String date, } catch (AnalysisException e) { throw new CastException(e.getMessage(), e); } + } else if (targetType instanceof TimeStampNsType) { + try { + return new TimeStampNsLiteral(format); + } catch (AnalysisException e) { + throw new CastException(e.getMessage(), e); + } } else { try { return new DateTimeV2Literal((DateTimeV2Type) targetType, format); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimeStampNsLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimeStampNsLiteral.java new file mode 100644 index 00000000000000..194add743c9a41 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimeStampNsLiteral.java @@ -0,0 +1,234 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.literal; + +import org.apache.doris.analysis.LiteralExpr; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.TimeStampNsType; +import org.apache.doris.nereids.util.DateUtils; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.temporal.ChronoField; +import java.time.temporal.TemporalAccessor; +import java.time.temporal.TemporalQueries; +import java.util.Objects; + +/** Literal for the fixed nanosecond-precision TIMESTAMP_NS type. */ +public final class TimeStampNsLiteral extends DateLiteral { + private static final long NANOS_PER_SECOND = 1_000_000_000L; + private static final long MAX_NANOSECOND = NANOS_PER_SECOND - 1; + private static final LocalDateTime MIN_VALUE + = LocalDateTime.of(1677, 9, 21, 0, 12, 43, 145224192); + private static final LocalDateTime MAX_VALUE + = LocalDateTime.of(2262, 4, 11, 23, 47, 16, 854775807); + + private final long hour; + private final long minute; + private final long second; + private final long nanosecond; + + public TimeStampNsLiteral(String value) { + this(parse(value)); + } + + /** Construct a TIMESTAMP_NS literal from civil datetime fields. */ + public TimeStampNsLiteral(long year, long month, long day, long hour, long minute, long second, + long nanosecond) { + super(TimeStampNsType.INSTANCE, year, month, day); + this.hour = hour; + this.minute = minute; + this.second = second; + this.nanosecond = nanosecond; + if (checkRange()) { + throw new AnalysisException("timestamp_ns literal [" + toString() + + "] is outside Int64 epoch nanosecond range"); + } + } + + private TimeStampNsLiteral(LocalDateTime value) { + this(value.getYear(), value.getMonthValue(), value.getDayOfMonth(), + value.getHour(), value.getMinute(), value.getSecond(), value.getNano()); + } + + private static LocalDateTime parse(String value) { + TemporalAccessor temporal = parseDateTime(value).get(); + long year = DateUtils.getOrDefault(temporal, ChronoField.YEAR); + long month = DateUtils.getOrDefault(temporal, ChronoField.MONTH_OF_YEAR); + long day = DateUtils.getOrDefault(temporal, ChronoField.DAY_OF_MONTH); + long hour = DateUtils.getOrDefault(temporal, ChronoField.HOUR_OF_DAY); + long minute = DateUtils.getOrDefault(temporal, ChronoField.MINUTE_OF_HOUR); + long second = DateUtils.getOrDefault(temporal, ChronoField.SECOND_OF_MINUTE); + + ZoneId zoneId = temporal.query(TemporalQueries.zone()); + if (zoneId != null) { + LocalDateTime converted = DateUtils.convertTimeZone( + year, month, day, hour, minute, second, zoneId, DateUtils.getTimeZone()); + year = converted.getYear(); + month = converted.getMonthValue(); + day = converted.getDayOfMonth(); + hour = converted.getHour(); + minute = converted.getMinute(); + second = converted.getSecond(); + } + + LocalDateTime result = LocalDateTime.of((int) year, (int) month, (int) day, + (int) hour, (int) minute, (int) second, + DateUtils.getOrDefault(temporal, ChronoField.NANO_OF_SECOND)); + if (DateUtils.getNanosecondGuardDigit(value) >= 5) { + result = result.plusNanos(1); + } + return result; + } + + public static TimeStampNsLiteral getMinValue() { + return new TimeStampNsLiteral(MIN_VALUE); + } + + public static TimeStampNsLiteral getMaxValue() { + return new TimeStampNsLiteral(MAX_VALUE); + } + + public static TimeStampNsLiteral createEndOfDay(long year, long month, long day) { + return new TimeStampNsLiteral(year, month, day, 23, 59, 59, MAX_NANOSECOND); + } + + public static TimeStampNsLiteral fromJavaDateType(LocalDateTime dateTime) { + return new TimeStampNsLiteral(dateTime); + } + + /** Return whether the civil fields are invalid or outside the signed epoch-nanosecond range. */ + public boolean checkRange() { + if (checkRange(year, month, day) || month < 1 || day < 1 || checkDate(year, month, day) + || hour < 0 || hour > 23 || minute < 0 || minute > 59 + || second < 0 || second > 59 || nanosecond < 0 || nanosecond > MAX_NANOSECOND) { + return true; + } + LocalDateTime value = toJavaDateType(); + return value.isBefore(MIN_VALUE) || value.isAfter(MAX_VALUE); + } + + @Override + public boolean isMidnight() { + return hour == 0 && minute == 0 && second == 0 && nanosecond == 0; + } + + @Override + public TimeStampNsType getDataType() { + return TimeStampNsType.INSTANCE; + } + + @Override + public Long getValue() { + return (year * 10000 + month * 100 + day) * 1000000L + hour * 10000 + minute * 100 + second; + } + + @Override + public long getTimePartInNanoseconds() { + return ((hour * 60L + minute) * 60L + second) * NANOS_PER_SECOND + nanosecond; + } + + @Override + public long getFractionalSecondInNanoseconds() { + return nanosecond; + } + + @Override + public double getDouble() { + return getValue() + nanosecond / (double) NANOS_PER_SECOND; + } + + @Override + public String getStringValue() { + return DateUtils.formatDateTime(year, month, day, hour, minute, second, + nanosecond, TimeStampNsType.SCALE); + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitTimeStampNsLiteral(this, context); + } + + @Override + public LiteralExpr toLegacyLiteral() { + return new org.apache.doris.analysis.TimeStampNsLiteral( + year, month, day, hour, minute, second, nanosecond); + } + + public DateTimeV2Literal roundFloorToDateTimeV2(int scale) { + long factor = (long) Math.pow(10, DateUtils.NANOSECOND_SCALE - scale); + LocalDateTime rounded = toJavaDateType().withNano((int) (nanosecond / factor * factor)); + return DateTimeV2Literal.fromJavaDateType(rounded, scale); + } + + public DateTimeV2Literal roundCeilingToDateTimeV2(int scale) { + long factor = (long) Math.pow(10, DateUtils.NANOSECOND_SCALE - scale); + long remainder = nanosecond % factor; + LocalDateTime rounded = remainder == 0 ? toJavaDateType() + : toJavaDateType().plusNanos(factor - remainder); + return DateTimeV2Literal.fromJavaDateType(rounded, scale); + } + + @Override + public LocalDateTime toJavaDateType() { + return LocalDateTime.of((int) year, (int) month, (int) day, + (int) hour, (int) minute, (int) second, (int) nanosecond); + } + + public long getHour() { + return hour; + } + + public long getMinute() { + return minute; + } + + public long getSecond() { + return second; + } + + public long getMicroSecond() { + return nanosecond / 1000; + } + + public long getNanoSecond() { + return nanosecond; + } + + public int getScale() { + return TimeStampNsType.SCALE; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof TimeStampNsLiteral)) { + return false; + } + TimeStampNsLiteral literal = (TimeStampNsLiteral) o; + return Objects.equals(getValue(), literal.getValue()) && nanosecond == literal.nanosecond; + } + + @Override + protected int computeHashCode() { + return Objects.hash(getValue(), nanosecond); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java index a25b110dfb438a..acf4d0a35711d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java @@ -120,6 +120,7 @@ import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.expressions.literal.StructLiteral; +import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral; import org.apache.doris.nereids.trees.expressions.literal.TimeV2Literal; import org.apache.doris.nereids.trees.expressions.literal.TimestampTzLiteral; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; @@ -337,6 +338,10 @@ public R visitDateTimeV2Literal(DateTimeV2Literal dateTimeV2Literal, C context) return visitLiteral(dateTimeV2Literal, context); } + public R visitTimeStampNsLiteral(TimeStampNsLiteral timeStampNsLiteral, C context) { + return visitLiteral(timeStampNsLiteral, context); + } + public R visitTimestampTzLiteral(TimestampTzLiteral timestampTzLiteral, C context) { return visitLiteral(timestampTzLiteral, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java index 66a4c1f428c82f..555b22e715bcf5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java @@ -425,6 +425,10 @@ private void checkUdfSupportedTypes() throws AnalysisException { } private void checkUdfSupportedType(Type type, String typePosition) throws AnalysisException { + if (binaryType == Function.BinaryType.JAVA_UDF && type.isTimeStampNs()) { + throw new AnalysisException(String.format( + "%s does not support %s type %s", binaryType, typePosition, type.toSql())); + } // Reject bitmap/hll/quantile_state type if (type.isObjectStored()) { throw new AnalysisException(String.format( @@ -1063,6 +1067,9 @@ private Types.PGenericType convertToPParameterType(Type arg) throws AnalysisExce case TIMEV2: typeBuilder.setId(Types.PGenericType.TypeId.DATETIMEV2); break; + case TIMESTAMP_NS: + typeBuilder.setId(Types.PGenericType.TypeId.TIMESTAMP_NS); + break; case TIMESTAMPTZ: typeBuilder.setId(Types.PGenericType.TypeId.TIMESTAMPTZ); break; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java index ad5f41ec2d1816..b6a0bb014cb480 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java @@ -18,9 +18,9 @@ package org.apache.doris.nereids.trees.plans.commands.info; import org.apache.doris.analysis.DefaultValueExprDef; -import org.apache.doris.catalog.ScalarType; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.types.DateTimeV2Type; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; @@ -81,9 +81,9 @@ public DefaultValue(String value, String exprName, Long precision) { * default value current_timestamp(precision) */ public static DefaultValue currentTimeStampDefaultValueWithPrecision(Long precision) { - if (precision > ScalarType.MAX_DATETIMEV2_SCALE || precision < 0) { + if (precision > DateTimeV2Type.MAX_SCALE || precision < 0) { throw new AnalysisException("column's default value current_timestamp" - + " precision must be between 0 and 6"); + + " precision must be between 0 and " + DateTimeV2Type.MAX_SCALE); } if (precision == 0) { return new DefaultValue(CURRENT_TIMESTAMP, NOW); @@ -118,22 +118,10 @@ public String getRawValue() { return LocalDateTime.now(TimeUtils.getTimeZone().toZoneId()).toString().replace('T', ' '); } else if (isCurrentTimeStampWithPrecision()) { long precision = getCurrentTimeStampPrecision(); - String format = "yyyy-MM-dd HH:mm:ss"; if (precision == 0) { return LocalDateTime.now(TimeUtils.getTimeZone().toZoneId()).toString().replace('T', ' '); - } else if (precision == 1) { - format = "yyyy-MM-dd HH:mm:ss.S"; - } else if (precision == 2) { - format = "yyyy-MM-dd HH:mm:ss.SS"; - } else if (precision == 3) { - format = "yyyy-MM-dd HH:mm:ss.SSS"; - } else if (precision == 4) { - format = "yyyy-MM-dd HH:mm:ss.SSSS"; - } else if (precision == 5) { - format = "yyyy-MM-dd HH:mm:ss.SSSSS"; - } else if (precision == 6) { - format = "yyyy-MM-dd HH:mm:ss.SSSSSS"; } + String format = "yyyy-MM-dd HH:mm:ss." + "S".repeat((int) precision); return LocalDateTime.now(TimeUtils.getTimeZone().toZoneId()) .format(DateTimeFormatter.ofPattern(format)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java index c7ef8f21de9cfc..d8655e5fe2d01c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java @@ -77,6 +77,7 @@ private static Map initMap() { .put(Type.DATEV2.getPrimitiveType(), DateType.INSTANCE) .put(Type.DATETIME.getPrimitiveType(), DateTimeType.INSTANCE) .put(Type.DATETIMEV2.getPrimitiveType(), DateTimeV2Type.SYSTEM_DEFAULT) + .put(Type.TIMESTAMP_NS.getPrimitiveType(), TimeStampNsType.INSTANCE) .put(Type.TIMESTAMPTZ.getPrimitiveType(), TimeStampTzType.SYSTEM_DEFAULT) .put(Type.DECIMALV2.getPrimitiveType(), DecimalV2Type.SYSTEM_DEFAULT) .put(Type.DECIMAL32.getPrimitiveType(), DecimalV3Type.SYSTEM_DEFAULT) @@ -118,7 +119,8 @@ private static Map initMap() { .add(DateType.class, () -> ImmutableList.of( DateTimeType.INSTANCE, DateV2Type.INSTANCE, StringType.INSTANCE)) .add(DateV2Type.class, () -> ImmutableList.of(DateTimeV2Type.SYSTEM_DEFAULT, StringType.INSTANCE)) - .add(TimeV2Type.class, () -> ImmutableList.of(DateTimeV2Type.MAX, StringType.INSTANCE)) + .add(TimeV2Type.class, () -> ImmutableList.of( + DateTimeV2Type.MAX, StringType.INSTANCE)) .build(); public static Map legacyTypeToNereidsType() { @@ -351,6 +353,12 @@ public static DataType convertPrimitiveFromStrings(List types) { throw new AnalysisException("Nereids do not support type: " + type); } break; + case "timestamp_ns": + if (types.size() != 1) { + throw new AnalysisException("timestamp_ns does not support precision"); + } + dataType = TimeStampNsType.INSTANCE; + break; case "timestamptz": switch (types.size()) { case 1: @@ -424,6 +432,7 @@ public static DataType fromCatalogType(Type type) { case DOUBLE: return DoubleType.INSTANCE; case NULL_TYPE: return NullType.INSTANCE; case DATETIMEV2: return DateTimeV2Type.of(((ScalarType) type).getScalarScale()); + case TIMESTAMP_NS: return TimeStampNsType.INSTANCE; case DATETIME: return DateTimeType.INSTANCE; case DATEV2: return DateV2Type.INSTANCE; case DATE: return DateType.INSTANCE; @@ -1098,20 +1107,35 @@ private static void validateScalarType(ScalarType scalarType) { + precision + " in not supported."); } } + case TIMESTAMP_NS: + // TIMESTAMP_NS has fixed nanosecond precision and carries no variable precision/scale metadata. + break; + case DATETIMEV2: { + int precision = scalarType.decimalPrecision(); + int scale = scalarType.decimalScale(); + if (precision != ScalarType.DATETIME_PRECISION) { + throw new AnalysisException( + "Precision of Datetime must be " + ScalarType.DATETIME_PRECISION + + "." + " Precision was set to: " + precision + "."); + } + if (scale < 0 || scale > ScalarType.MAX_DATETIMEV2_SCALE) { + throw new AnalysisException("Scale of Datetime must between 0 and " + + ScalarType.MAX_DATETIMEV2_SCALE + "." + + " Scale was set to: " + scale + "."); + } + break; + } case TIMEV2: - case DATETIMEV2: case TIMESTAMPTZ: { int precision = scalarType.decimalPrecision(); int scale = scalarType.decimalScale(); - // precision: 18 if (precision != ScalarType.DATETIME_PRECISION) { throw new AnalysisException( - "Precision of Datetime/Time must be " + ScalarType.DATETIME_PRECISION + "Precision of Time must be " + ScalarType.DATETIME_PRECISION + "." + " Precision was set to: " + precision + "."); } - // scale: [0, 6] if (scale < 0 || scale > 6) { - throw new AnalysisException("Scale of Datetime/Time must between 0 and 6." + throw new AnalysisException("Scale of Time must between 0 and 6." + " Scale was set to: " + scale + "."); } break; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java index 13097339554e3e..5f414c9af1dfbe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java @@ -28,6 +28,8 @@ import org.apache.doris.nereids.types.coercion.IntegralType; import org.apache.doris.nereids.types.coercion.ScaleTimeType; +import com.google.common.base.Preconditions; + import java.time.DateTimeException; import java.time.LocalDateTime; import java.time.temporal.ChronoUnit; @@ -37,7 +39,7 @@ * Datetime type in Nereids. */ public class DateTimeV2Type extends DateLikeType implements ScaleTimeType { - public static final int MAX_SCALE = 6; + public static final int MAX_SCALE = ScalarType.MAX_DATETIMEV2_SCALE; public static final DateTimeV2Type SYSTEM_DEFAULT = new DateTimeV2Type(0); public static final DateTimeV2Type MAX = new DateTimeV2Type(MAX_SCALE); public static final DateTimeV2Type WILDCARD = new DateTimeV2Type(-1); @@ -46,7 +48,7 @@ public class DateTimeV2Type extends DateLikeType implements ScaleTimeType { private final int scale; - private DateTimeV2Type(int scale) { + protected DateTimeV2Type(int scale) { this.scale = scale; } @@ -63,7 +65,11 @@ public static DateTimeV2Type of(int scale) { } } + /** Return the wider DATETIMEV2 type. */ public static DateTimeV2Type getWiderDatetimeV2Type(DateTimeV2Type t1, DateTimeV2Type t2) { + Preconditions.checkArgument(t1.getClass() == DateTimeV2Type.class + && t2.getClass() == DateTimeV2Type.class, + "TIMESTAMP_NS does not belong to the DATETIMEV2 precision family"); if (t1.scale > t2.scale) { return t1; } @@ -85,10 +91,10 @@ public static DateTimeV2Type forType(DataType dataType) { return SYSTEM_DEFAULT; } if (dataType instanceof DecimalV3Type) { - return DateTimeV2Type.of(Math.min(((DecimalV3Type) dataType).getScale(), 6)); + return DateTimeV2Type.of(Math.min(((DecimalV3Type) dataType).getScale(), MAX_SCALE)); } if (dataType instanceof DecimalV2Type) { - return DateTimeV2Type.of(Math.min(((DecimalV2Type) dataType).getScale(), 6)); + return DateTimeV2Type.of(Math.min(((DecimalV2Type) dataType).getScale(), MAX_SCALE)); } if (dataType instanceof TimeV2Type) { return DateTimeV2Type.of(((TimeV2Type) dataType).getScale()); @@ -96,6 +102,19 @@ public static DateTimeV2Type forType(DataType dataType) { return MAX; } + /** + * Return the DATETIMEV2 type inferred from {@code dataType}, widened only when the operation + * itself requires more fractional digits. + */ + public static DateTimeV2Type forTypeWithMinimumScale(DataType dataType, int minimumScale) { + if (dataType instanceof CharacterType) { + return DateTimeV2Type.of(minimumScale); + } + DateTimeV2Type dateTimeV2Type = forType(dataType); + return dateTimeV2Type instanceof TimeStampNsType + ? dateTimeV2Type : DateTimeV2Type.of(Math.max(dateTimeV2Type.getScale(), minimumScale)); + } + public ScaleTimeType scaleTypeForType(DataType dataType) { return forType(dataType); } @@ -118,10 +137,7 @@ public static DateTimeV2Type forTypeFromString(String s) { // let be to process it } } - if (scale > MAX_SCALE) { - scale = MAX_SCALE; - } - return DateTimeV2Type.of(scale); + return DateTimeV2Type.of(Math.min(scale, MAX_SCALE)); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeStampNsType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeStampNsType.java new file mode 100644 index 00000000000000..e16b647b1d023c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeStampNsType.java @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.types; + +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.Type; +import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; +import org.apache.doris.nereids.types.coercion.ScaleTimeType; + +/** Timestamp represented as signed nanoseconds since the Unix epoch. */ +public final class TimeStampNsType extends DateTimeV2Type { + public static final int SCALE = ScalarType.TIMESTAMP_NS_SCALE; + public static final TimeStampNsType INSTANCE = new TimeStampNsType(); + + private TimeStampNsType() { + super(SCALE); + } + + @Override + public Type toCatalogDataType() { + return ScalarType.createTimeStampNsType(); + } + + @Override + public ScaleTimeType scaleTypeForType(DataType dataType) { + return INSTANCE; + } + + @Override + public ScaleTimeType forTypeFromString(StringLikeLiteral str) { + return INSTANCE; + } + + @Override + public String toSql() { + return "timestamp_ns"; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeStampTzType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeStampTzType.java index 4f9c09b5f7f805..b5cd1ce2a3e738 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeStampTzType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeStampTzType.java @@ -85,13 +85,13 @@ public ScaleTimeType forTypeFromString(StringLikeLiteral s) { public static TimeStampTzType forTypeFromString(String s) { DateTimeV2Type dateTimeV2Type = DateTimeV2Type.forTypeFromString(s); - return TimeStampTzType.of(dateTimeV2Type.getScale()); + return TimeStampTzType.of(Math.min(dateTimeV2Type.getScale(), MAX_SCALE)); } @Override public ScaleTimeType scaleTypeForType(DataType dataType) { DateTimeV2Type dateTimeV2Type = DateTimeV2Type.forType(dataType); - return TimeStampTzType.of(dateTimeV2Type.getScale()); + return TimeStampTzType.of(Math.min(dateTimeV2Type.getScale(), MAX_SCALE)); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeV2Type.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeV2Type.java index af758c02cb522b..e3d7748c59ce36 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeV2Type.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeV2Type.java @@ -117,7 +117,7 @@ public static TimeV2Type forType(DataType dataType) { return TimeV2Type.of(Math.min(((DecimalV2Type) dataType).getScale(), 6)); } if (dataType instanceof DateTimeV2Type) { - return TimeV2Type.of(((DateTimeV2Type) dataType).getScale()); + return TimeV2Type.of(Math.min(((DateTimeV2Type) dataType).getScale(), MAX_SCALE)); } return MAX; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/DateLikeType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/DateLikeType.java index 1f8130215b04e2..f931ff2ce27c10 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/DateLikeType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/DateLikeType.java @@ -22,10 +22,12 @@ import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal; +import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral; import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.DateType; import org.apache.doris.nereids.types.DateV2Type; +import org.apache.doris.nereids.types.TimeStampNsType; import java.time.LocalDate; import java.time.LocalDateTime; @@ -70,6 +72,8 @@ public DateLiteral fromString(String s) { return new DateV2Literal(l.getYear(), l.getMonth(), l.getDay()); } else if (this instanceof DateTimeType) { return new DateTimeLiteral(s); + } else if (this instanceof TimeStampNsType) { + return new TimeStampNsLiteral(s); } else if (this instanceof DateTimeV2Type) { return new DateTimeV2Literal((DateTimeV2Type) this, s); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java index e8ff854945a0a2..e86e7ebdbbea3e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; import org.apache.doris.nereids.trees.expressions.literal.TimeV2Literal; +import org.apache.doris.nereids.types.TimeStampNsType; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; @@ -56,14 +57,13 @@ public class DateTimeFormatterUtils { .appendLiteral('-').appendValue(ChronoField.MONTH_OF_YEAR, 2) .appendLiteral('-').appendValue(ChronoField.DAY_OF_MONTH, 2) .toFormatter().withResolverStyle(ResolverStyle.STRICT); - // HH[:mm][:ss][.microsecond] + // HH[:mm][:ss][.fraction] public static final DateTimeFormatter TIME_FORMATTER = new DateTimeFormatterBuilder() .appendValue(ChronoField.HOUR_OF_DAY, 2) .appendLiteral(':').appendValue(ChronoField.MINUTE_OF_HOUR, 2) .appendLiteral(':').appendValue(ChronoField.SECOND_OF_MINUTE, 2) - // microsecond maxWidth is 7, we may need 7th digit to judge overflow .appendOptional(new DateTimeFormatterBuilder() - .appendFraction(ChronoField.NANO_OF_SECOND, 1, 7, true).toFormatter()) + .appendFraction(ChronoField.NANO_OF_SECOND, 1, TimeStampNsType.SCALE, true).toFormatter()) .toFormatter().withResolverStyle(ResolverStyle.STRICT); // Time without delimiter: HHmmss[.microsecond] private static final DateTimeFormatter BASIC_TIME_FORMATTER = new DateTimeFormatterBuilder() @@ -71,7 +71,7 @@ public class DateTimeFormatterUtils { .appendValue(ChronoField.MINUTE_OF_HOUR, 2) .appendValue(ChronoField.SECOND_OF_MINUTE, 2) .appendOptional(new DateTimeFormatterBuilder() - .appendFraction(ChronoField.NANO_OF_SECOND, 1, 7, true).toFormatter()) + .appendFraction(ChronoField.NANO_OF_SECOND, 1, TimeStampNsType.SCALE, true).toFormatter()) .toFormatter().withResolverStyle(ResolverStyle.STRICT); // yyyymmdd private static final DateTimeFormatter BASIC_DATE_FORMATTER = new DateTimeFormatterBuilder() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateUtils.java index e339471e447fb9..685819c481d579 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateUtils.java @@ -27,6 +27,7 @@ import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; +import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; import java.time.format.ResolverStyle; @@ -37,6 +38,9 @@ import java.time.temporal.TemporalAccessor; import java.time.temporal.WeekFields; import java.time.zone.ZoneOffsetTransition; +import java.time.zone.ZoneRules; +import java.util.Arrays; +import java.util.List; import java.util.Locale; import java.util.Set; @@ -44,6 +48,7 @@ * date util tools. */ public class DateUtils { + public static final int NANOSECOND_SCALE = 9; public static final Set monoFormat = ImmutableSet.of("yyyyMMdd", "yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss", "%Y", "%Y-%m", "%Y-%m-%d", "%Y-%m-%d %H", "%Y-%m-%d %H:%i", "%Y-%m-%d %H:%i:%s", "%Y-%m-%d %H:%i:%S", "%Y-%m-%d %T", "%Y%m%d", "%Y%m"); @@ -389,6 +394,98 @@ public static int getOrDefault(final TemporalAccessor accessor, final ChronoFiel return accessor.isSupported(field) ? accessor.get(field) : /* default value */ 0; } + /** + * Remove the TIMESTAMP_NS rounding guard digit before parsing because Java accepts at most + * nine fractional-second digits. The guard remains available in the original string and can + * be retrieved by {@link #getNanosecondGuardDigit(String)} during target-scale rounding. + */ + public static String truncateFractionalSecondForJavaParser(String s) { + int dot = s.lastIndexOf('.'); + if (dot < 0) { + return s; + } + int fractionEnd = dot + 1; + while (fractionEnd < s.length() && Character.isDigit(s.charAt(fractionEnd))) { + fractionEnd++; + } + int retainedEnd = Math.min(dot + NANOSECOND_SCALE + 1, fractionEnd); + return retainedEnd == fractionEnd ? s : s.substring(0, retainedEnd) + s.substring(fractionEnd); + } + + /** Return the TIMESTAMP_NS rounding guard digit, or -1 when the input has none. */ + public static int getNanosecondGuardDigit(String s) { + int dot = s.lastIndexOf('.'); + int guardIndex = dot + NANOSECOND_SCALE + 1; + if (dot < 0 || guardIndex >= s.length() || !Character.isDigit(s.charAt(guardIndex))) { + return -1; + } + return s.charAt(guardIndex) - '0'; + } + + /** Format a datetime with the requested number of fractional-second digits. */ + public static String formatDateTime(long year, long month, long day, long hour, long minute, + long second, long nanosecond, int scale) { + if (0 <= year && year <= 9999 && 0 <= month && month <= 99 && 0 <= day && day <= 99 + && 0 <= hour && hour <= 99 && 0 <= minute && minute <= 99 + && 0 <= second && second <= 99 && 0 <= nanosecond && nanosecond < 1_000_000_000L) { + int length = scale == 0 ? 19 : 20 + scale; + char[] value = new char[length]; + Arrays.fill(value, '0'); + value[4] = '-'; + value[7] = '-'; + value[10] = ' '; + value[13] = ':'; + value[16] = ':'; + fillPaddedValue(value, 0, year, 4); + fillPaddedValue(value, 5, month, 2); + fillPaddedValue(value, 8, day, 2); + fillPaddedValue(value, 11, hour, 2); + fillPaddedValue(value, 14, minute, 2); + fillPaddedValue(value, 17, second, 2); + if (scale > 0) { + value[19] = '.'; + long divisor = (long) Math.pow(10, NANOSECOND_SCALE - scale); + fillPaddedValue(value, 20, nanosecond / divisor, scale); + } + return String.valueOf(value); + } + String fractionFormat = scale > 0 ? ".%0" + scale + "d" : ""; + long divisor = (long) Math.pow(10, NANOSECOND_SCALE - scale); + return String.format("%04d-%02d-%02d %02d:%02d:%02d" + fractionFormat, + year, month, day, hour, minute, second, nanosecond / divisor); + } + + private static void fillPaddedValue(char[] buffer, int start, long value, int length) { + for (int i = start + length - 1; i >= start; i--) { + buffer[i] = (char) ('0' + value % 10); + value /= 10; + } + } + + /** Convert a civil datetime between time zones using the same DST policy as BE cctz. */ + public static LocalDateTime convertTimeZone(long year, long month, long day, long hour, long minute, + long second, ZoneId fromZone, ZoneId toZone) { + LocalDateTime localDateTime = LocalDateTime.of((int) year, (int) month, (int) day, + (int) hour, (int) minute, (int) second); + return LocalDateTime.ofInstant(convertLocalToInstant(localDateTime, fromZone), toZone); + } + + /** + * Convert a local civil datetime to an instant with the same overlap/gap policy as BE cctz. + * Repeated local times use the pre-transition offset; skipped local times map to the transition + * instant. + */ + public static Instant convertLocalToInstant(LocalDateTime localDateTime, ZoneId fromZone) { + ZoneRules rules = fromZone.getRules(); + List validOffsets = rules.getValidOffsets(localDateTime); + int size = validOffsets.size(); + if (size == 1 || size == 2) { + return localDateTime.atOffset(validOffsets.get(0)).toInstant(); + } + ZoneOffsetTransition transition = rules.getTransition(localDateTime); + return transition.getInstant(); + } + /** * get hour from accessor, if not support hour field, return 0 */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index 4a5fa860a90703..6596a4befb515d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -64,6 +64,7 @@ import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral; import org.apache.doris.nereids.trees.expressions.literal.TimeV2Literal; import org.apache.doris.nereids.trees.expressions.literal.TimestampTzLiteral; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; @@ -96,6 +97,7 @@ import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.types.TimeV2Type; import org.apache.doris.nereids.types.TinyIntType; @@ -131,6 +133,7 @@ import java.util.Map; import java.util.Optional; import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Collectors; /** @@ -234,6 +237,21 @@ public static Optional implicitCast(DataType input, DataType expected) * Return Optional.empty() if we cannot do implicit cast. */ public static Optional implicitCastPrimitive(DataType input, DataType expected) { + // TIMESTAMP_NS has a different physical representation from DATETIMEV2. Until explicit + // cross-type casts are implemented, only its string conversions can be implicit. + if (input instanceof TimeStampNsType || expected instanceof TimeStampNsType) { + if (input.equals(expected) || expected instanceof AnyDataType) { + return Optional.of(input); + } else if (input instanceof NullType) { + return Optional.of(expected.defaultConcreteType()); + } else if (input instanceof CharacterType && expected instanceof TimeStampNsType) { + return Optional.of(expected); + } else if (input instanceof TimeStampNsType && expected instanceof CharacterType) { + return Optional.of(expected.defaultConcreteType()); + } + return Optional.empty(); + } + Optional castType = implicitCastPrimitiveInternal(input, expected); // TODO: complete the cast logic like FunctionCallExpr.analyzeImpl boolean legacyCastCompatible = false; @@ -333,7 +351,11 @@ public static boolean hasDecimalV3Type(DataType dataType) { } public static boolean hasDateTimeV2Type(DataType dataType) { - return hasSpecifiedType(dataType, DateTimeV2Type.class); + return hasSpecifiedType(dataType, type -> type.getClass() == DateTimeV2Type.class); + } + + public static boolean hasTimeStampNsType(DataType dataType) { + return hasSpecifiedType(dataType, TimeStampNsType.class); } public static boolean hasTimeV2Type(DataType dataType) { @@ -345,16 +367,20 @@ public static boolean hasTimestampTzType(DataType dataType) { } private static boolean hasSpecifiedType(DataType dataType, Class specifiedType) { + return hasSpecifiedType(dataType, type -> specifiedType.isAssignableFrom(type.getClass())); + } + + private static boolean hasSpecifiedType(DataType dataType, Predicate predicate) { if (dataType instanceof ArrayType) { - return hasSpecifiedType(((ArrayType) dataType).getItemType(), specifiedType); + return hasSpecifiedType(((ArrayType) dataType).getItemType(), predicate); } else if (dataType instanceof MapType) { - return hasSpecifiedType(((MapType) dataType).getKeyType(), specifiedType) - || hasSpecifiedType(((MapType) dataType).getValueType(), specifiedType); + return hasSpecifiedType(((MapType) dataType).getKeyType(), predicate) + || hasSpecifiedType(((MapType) dataType).getValueType(), predicate); } else if (dataType instanceof StructType) { return ((StructType) dataType).getFields().stream() - .anyMatch(f -> hasSpecifiedType(f.getDataType(), specifiedType)); + .anyMatch(f -> hasSpecifiedType(f.getDataType(), predicate)); } - return specifiedType.isAssignableFrom(dataType.getClass()); + return predicate.test(dataType); } /** @@ -377,7 +403,14 @@ public static DataType replaceDecimalV3WithWildcard(DataType dataType) { } public static DataType replaceDateTimeV2WithMax(DataType dataType) { - return replaceSpecifiedType(dataType, DateTimeV2Type.class, DateTimeV2Type.MAX); + return replaceSpecifiedType(dataType, + type -> type.getClass() == DateTimeV2Type.class, DateTimeV2Type.MAX); + } + + private static DataType replaceDateLikeWithMaxPrecision(DataType dataType) { + return replaceSpecifiedType(dataType, + type -> type instanceof DateLikeType && !(type instanceof TimeStampNsType), + DateTimeV2Type.MAX); } /** @@ -386,10 +419,14 @@ public static DataType replaceDateTimeV2WithMax(DataType dataType) { public static DataType replaceTimesWithTargetPrecision(DataType dataType, int targetScale) { return replaceSpecifiedType( replaceSpecifiedType( - replaceSpecifiedType(dataType, DateTimeV2Type.class, - DateTimeV2Type.of(targetScale)), - TimeV2Type.class, TimeV2Type.of(targetScale)), - TimeStampTzType.class, TimeStampTzType.of(targetScale)); + // Preserve TimeStampNsType: its DateTimeV2Type inheritance is for code reuse, + // not type-family precision promotion. + replaceSpecifiedType(dataType, + type -> type.getClass() == DateTimeV2Type.class, + DateTimeV2Type.of(Math.min(targetScale, DateTimeV2Type.MAX_SCALE))), + TimeV2Type.class, TimeV2Type.of(Math.min(targetScale, TimeV2Type.MAX_SCALE))), + TimeStampTzType.class, + TimeStampTzType.of(Math.min(targetScale, TimeStampTzType.MAX_SCALE))); } /** @@ -397,17 +434,27 @@ public static DataType replaceTimesWithTargetPrecision(DataType dataType, int ta */ public static DataType replaceSpecifiedType(DataType dataType, Class specifiedType, DataType newType) { + return replaceSpecifiedType(dataType, + type -> specifiedType.isAssignableFrom(type.getClass()), newType); + } + + private static DataType replaceSpecifiedType(DataType dataType, + Predicate shouldReplace, DataType newType) { if (dataType instanceof ArrayType) { - return ArrayType.of(replaceSpecifiedType(((ArrayType) dataType).getItemType(), specifiedType, newType)); + return ArrayType.of(replaceSpecifiedType( + ((ArrayType) dataType).getItemType(), shouldReplace, newType)); } else if (dataType instanceof MapType) { - return MapType.of(replaceSpecifiedType(((MapType) dataType).getKeyType(), specifiedType, newType), - replaceSpecifiedType(((MapType) dataType).getValueType(), specifiedType, newType)); + return MapType.of(replaceSpecifiedType( + ((MapType) dataType).getKeyType(), shouldReplace, newType), + replaceSpecifiedType( + ((MapType) dataType).getValueType(), shouldReplace, newType)); } else if (dataType instanceof StructType) { List newFields = ((StructType) dataType).getFields().stream() - .map(f -> f.withDataType(replaceSpecifiedType(f.getDataType(), specifiedType, newType))) + .map(f -> f.withDataType(replaceSpecifiedType( + f.getDataType(), shouldReplace, newType))) .collect(ImmutableList.toImmutableList()); return new StructType(newFields); - } else if (specifiedType.isAssignableFrom(dataType.getClass())) { + } else if (shouldReplace.test(dataType)) { return newType; } else { return dataType; @@ -665,6 +712,8 @@ public static Optional characterLiteralTypeCoercion(String value, Da ret = new VarcharLiteral(value, ((VarcharType) dataType).getLen()); } else if (dataType instanceof StringType) { ret = new StringLiteral(value); + } else if (dataType instanceof TimeStampNsType && DateTimeChecker.isValidDateTime(value)) { + ret = new TimeStampNsLiteral(value); } else if ((dataType.isDateTimeV2Type() || dataType.isDateTimeType()) && DateTimeChecker.isValidDateTime(value)) { ret = DateTimeLiteral.parseDateTimeLiteral(value, true).orElse(null); @@ -681,10 +730,14 @@ public static Optional characterLiteralTypeCoercion(String value, Da if (parseResult.isOk()) { ret = parseResult.get(); } else { - Result parseResult2 - = DateTimeV2Literal.parseDateTimeLiteral(value, true); - if (parseResult2.isOk()) { - ret = parseResult2.get(); + if (DateTimeLiteral.determineScale(value) > DateTimeV2Type.MAX_SCALE) { + ret = new TimeStampNsLiteral(value); + } else { + Result parseResult2 + = DateTimeV2Literal.parseDateTimeLiteral(value, true); + if (parseResult2.isOk()) { + ret = parseResult2.get(); + } } } } else if (dataType instanceof TimeV2Type && TimeChecker.isValidTime(value)) { @@ -953,10 +1006,12 @@ private static Optional getCommonDataTypeWithDateType(DataType leftTyp return Optional.of(DateTimeV2Type.SYSTEM_DEFAULT); } else if (rightType instanceof DecimalV2Type) { DecimalV2Type decimalV2Type = (DecimalV2Type) rightType; - return Optional.of(DateTimeV2Type.of(Math.min(DateTimeV2Type.MAX_SCALE, decimalV2Type.getScale()))); + return Optional.of(DateTimeV2Type.of(Math.min(DateTimeV2Type.MAX_SCALE, + decimalV2Type.getScale()))); } else if (rightType instanceof DecimalV3Type) { DecimalV3Type decimalV3Type = (DecimalV3Type) rightType; - return Optional.of(DateTimeV2Type.of(Math.min(DateTimeV2Type.MAX_SCALE, decimalV3Type.getScale()))); + return Optional.of(DateTimeV2Type.of(Math.min(DateTimeV2Type.MAX_SCALE, + decimalV3Type.getScale()))); } else { return Optional.of(DateTimeV2Type.MAX); } @@ -972,7 +1027,7 @@ private static Optional getCommonDataTypeWithDateType(DataType leftTyp } } else if (rightType instanceof TimeV2Type) { TimeV2Type timeV2Type = (TimeV2Type) rightType; - return Optional.of(DateTimeV2Type.of(Math.min(DateTimeV2Type.MAX_SCALE, timeV2Type.getScale()))); + return Optional.of(DateTimeV2Type.of(timeV2Type.getScale())); } else if (rightType.isStringLikeType()) { return Optional.of(DateTimeV2Type.MAX); } @@ -984,35 +1039,52 @@ private static Optional getCommonDataTypeWithDateType(DataType leftTyp */ private static Optional getCommonDataTypeWithDateTimeV2Type( DateTimeV2Type leftType, DataType rightType) { + Preconditions.checkArgument(leftType.getClass() == DateTimeV2Type.class, + "TIMESTAMP_NS does not belong to the DATETIMEV2 precision family"); if (rightType.isNumericType()) { if (rightType instanceof IntegralType) { return Optional.of(leftType); } else if (rightType instanceof DecimalV2Type) { DecimalV2Type decimalV2Type = (DecimalV2Type) rightType; - return Optional.of(DateTimeV2Type.of(Math.min(DateTimeV2Type.MAX_SCALE, - Math.max(leftType.getScale(), decimalV2Type.getScale())))); + return Optional.of(DateTimeV2Type.forTypeWithMinimumScale(leftType, + Math.min(DateTimeV2Type.MAX_SCALE, decimalV2Type.getScale()))); } else if (rightType instanceof DecimalV3Type) { DecimalV3Type decimalV3Type = (DecimalV3Type) rightType; - return Optional.of(DateTimeV2Type.of(Math.min(DateTimeV2Type.MAX_SCALE, - Math.max(leftType.getScale(), decimalV3Type.getScale())))); + return Optional.of(DateTimeV2Type.forTypeWithMinimumScale(leftType, + Math.min(DateTimeV2Type.MAX_SCALE, decimalV3Type.getScale()))); } else { - return Optional.of(DateTimeV2Type.MAX); + return Optional.of(DateTimeV2Type.forTypeWithMinimumScale( + leftType, DateTimeV2Type.MAX_SCALE)); } } else if (rightType.isDateLikeType()) { if (rightType instanceof DateTimeV2Type) { DateTimeV2Type dateTimeV2Type = (DateTimeV2Type) rightType; - return Optional.of(DateTimeV2Type.of(Math.max(leftType.getScale(), dateTimeV2Type.getScale()))); + return Optional.of(DateTimeV2Type.getWiderDatetimeV2Type(leftType, dateTimeV2Type)); } else if (rightType instanceof TimeStampTzType) { TimeStampTzType timeStampTzType = (TimeStampTzType) rightType; - return Optional.of(DateTimeV2Type.of(Math.max(leftType.getScale(), timeStampTzType.getScale()))); + return Optional.of(DateTimeV2Type.of( + Math.max(leftType.getScale(), timeStampTzType.getScale()))); } else { return Optional.of(leftType); } } else if (rightType instanceof TimeV2Type) { TimeV2Type timeV2Type = (TimeV2Type) rightType; - return Optional.of(DateTimeV2Type.of(Math.max(leftType.getScale(), timeV2Type.getScale()))); + return Optional.of(DateTimeV2Type.of( + Math.max(leftType.getScale(), timeV2Type.getScale()))); } else if (rightType.isStringLikeType()) { - return Optional.of(DateTimeV2Type.MAX); + return Optional.of(DateTimeV2Type.forTypeWithMinimumScale( + leftType, DateTimeV2Type.MAX_SCALE)); + } + return Optional.empty(); + } + + /** + * TIMESTAMP_NS is not a high-scale DATETIMEV2. It can only share a coercion target with + * strings until casts between TIMESTAMP_NS and other temporal or numeric types are available. + */ + private static Optional getCommonDataTypeWithTimeStampNsType(DataType otherType) { + if (otherType instanceof TimeStampNsType || otherType.isStringLikeType()) { + return Optional.of(TimeStampNsType.INSTANCE); } return Optional.empty(); } @@ -1124,16 +1196,16 @@ public static Optional findWiderTypeForTwo(DataType left, DataType rig return findCommonVariantType((VariantType) left, (VariantType) right); } else if (left instanceof VariantType) { return Optional.of(replaceSpecifiedType(replaceDecimalV3WithTarget(replaceSpecifiedType( - replaceSpecifiedType(replaceSpecifiedType(replaceCharacterToString(right), - IntegralType.class, DecimalV3Type.SYSTEM_DEFAULT), - DateLikeType.class, DateTimeV2Type.MAX), + replaceDateLikeWithMaxPrecision(replaceSpecifiedType( + replaceCharacterToString(right), + IntegralType.class, DecimalV3Type.SYSTEM_DEFAULT)), DecimalV2Type.class, DecimalV3Type.SYSTEM_DEFAULT), DecimalV3Type.SYSTEM_DEFAULT), FloatType.class, DoubleType.INSTANCE)); } else if (right instanceof VariantType) { return Optional.of(replaceSpecifiedType(replaceDecimalV3WithTarget(replaceSpecifiedType( - replaceSpecifiedType(replaceSpecifiedType(replaceCharacterToString(left), - IntegralType.class, DecimalV3Type.SYSTEM_DEFAULT), - DateLikeType.class, DateTimeV2Type.MAX), + replaceDateLikeWithMaxPrecision(replaceSpecifiedType( + replaceCharacterToString(left), + IntegralType.class, DecimalV3Type.SYSTEM_DEFAULT)), DecimalV2Type.class, DecimalV3Type.SYSTEM_DEFAULT), DecimalV3Type.SYSTEM_DEFAULT), FloatType.class, DoubleType.INSTANCE)); } else if (left instanceof ComplexDataType || right instanceof ComplexDataType) { @@ -1211,6 +1283,12 @@ private static Optional findWiderPrimitiveTypeForTwo( } } + if (leftType instanceof TimeStampNsType) { + return getCommonDataTypeWithTimeStampNsType(rightType); + } else if (rightType instanceof TimeStampNsType) { + return getCommonDataTypeWithTimeStampNsType(leftType); + } + // we process date like type first if (!leftType.isDateLikeType() && rightType.isDateLikeType()) { DataType temp = leftType; @@ -1739,6 +1817,12 @@ private static Optional findCommonPrimitiveTypeForComparison( return Optional.of(leftType); } + if (leftType instanceof TimeStampNsType) { + return getCommonDataTypeWithTimeStampNsType(rightType); + } else if (rightType instanceof TimeStampNsType) { + return getCommonDataTypeWithTimeStampNsType(leftType); + } + // decimal v3 if (leftType.isDecimalV3Type() && rightType.isDecimalV3Type()) { return Optional.of(DecimalV3Type.widerDecimalV3Type( @@ -1760,14 +1844,16 @@ private static Optional findCommonPrimitiveTypeForComparison( || rightType.isDateV2Type() || rightType.isDateTimeType()) { return Optional.of(leftType); } else { - return Optional.of(DateTimeV2Type.MAX); + return Optional.of(DateTimeV2Type.forTypeWithMinimumScale( + leftType, DateTimeV2Type.MAX_SCALE)); } } else if (rightType.isDateTimeV2Type()) { if (leftType instanceof IntegralType || leftType.isDateType() || leftType.isDateV2Type() || leftType.isDateTimeType()) { return Optional.of(rightType); } else { - return Optional.of(DateTimeV2Type.MAX); + return Optional.of(DateTimeV2Type.forTypeWithMinimumScale( + rightType, DateTimeV2Type.MAX_SCALE)); } } else if (leftType.isDateV2Type()) { if (rightType instanceof IntegralType || rightType.isDateType() || rightType.isDateV2Type()) { @@ -2010,6 +2096,10 @@ public static Optional findCommonPrimitiveTypeForCaseWhen(DataType t1, return Optional.of(StringType.INSTANCE); } + if (t1 instanceof TimeStampNsType || t2 instanceof TimeStampNsType) { + return Optional.empty(); + } + // decimal with date should return double if ((t1.isDecimalV2Type() && t2.isDateType()) || (t2.isDecimalV2Type() && t1.isDateType())) { return Optional.of(DoubleType.INSTANCE); diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java index a02f5bbf204b1a..3e61102801bfe7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java @@ -59,6 +59,7 @@ import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.StringLiteral; import org.apache.doris.analysis.StructLiteral; +import org.apache.doris.analysis.TimeStampNsLiteral; import org.apache.doris.analysis.TimeV2Literal; import org.apache.doris.analysis.TimestampArithmeticExpr; import org.apache.doris.analysis.TryCastExpr; @@ -266,6 +267,7 @@ public class GsonUtils { .registerSubtype(ArrayLiteral.class, ArrayLiteral.class.getSimpleName()) .registerSubtype(BoolLiteral.class, BoolLiteral.class.getSimpleName()) .registerSubtype(DateLiteral.class, DateLiteral.class.getSimpleName()) + .registerSubtype(TimeStampNsLiteral.class, TimeStampNsLiteral.class.getSimpleName()) .registerSubtype(IPv4Literal.class, IPv4Literal.class.getSimpleName()) .registerSubtype(IPv6Literal.class, IPv6Literal.class.getSimpleName()) .registerSubtype(JsonLiteral.class, JsonLiteral.class.getSimpleName()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index c83397f58dc0e7..9dd256b70c3bea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -888,7 +888,7 @@ private static LiteralExpr getLiteralExprFromThrift(TExprNode node) throws Analy case DECIMAL_LITERAL: return DecimalLiteralUtils.create(node.decimal_literal.value); case STRING_LITERAL: return new StringLiteral(node.string_literal.value); case JSON_LITERAL: return new JsonLiteral(node.json_literal.value); - case DATE_LITERAL: return DateLiteralUtils.createDateLiteral(node.date_literal.value, null); + case DATE_LITERAL: return DateLiteralUtils.createLiteral(node.date_literal.value, null); case IPV4_LITERAL: return new IPv4Literal(node.ipv4_literal.value); case IPV6_LITERAL: return new IPv6Literal(node.ipv6_literal.value); default: throw new AnalysisException("Wrong type from thrift;"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index b117b334f611f7..21ff1318ef92b0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -29,13 +29,17 @@ import org.apache.doris.analysis.StorageBackend.StorageType; import org.apache.doris.analysis.StringValueContext; import org.apache.doris.analysis.ToSqlParams; +import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.EnvFactory; import org.apache.doris.catalog.FsBroker; +import org.apache.doris.catalog.MapType; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.StructField; +import org.apache.doris.catalog.StructType; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.Type; import org.apache.doris.cloud.catalog.CloudEnv; @@ -1394,6 +1398,10 @@ private void handleQueryStmt() throws Exception { return; } + if (context.getConnectType() == ConnectType.ARROW_FLIGHT_SQL) { + checkArrowFlightSqlOutput(queryStmt.getResultExprs()); + } + if (parsedStmt instanceof LogicalPlanAdapter) { LogicalPlanAdapter logicalPlanAdapter = (LogicalPlanAdapter) parsedStmt; LogicalPlan logicalPlan = logicalPlanAdapter.getLogicalPlan(); @@ -1450,6 +1458,32 @@ private void handleQueryStmt() throws Exception { executeAndSendResult(isOutfileQuery, false, queryStmt, channel, null, null); } + static void checkArrowFlightSqlOutput(List resultExprs) throws AnalysisException { + for (Expr resultExpr : resultExprs) { + if (containsTimeStampNs(resultExpr.getType())) { + throw new AnalysisException("TIMESTAMP_NS is not supported by Arrow Flight SQL"); + } + } + } + + private static boolean containsTimeStampNs(Type type) { + if (type.isTimeStampNs()) { + return true; + } else if (type.isArrayType()) { + return containsTimeStampNs(((ArrayType) type).getItemType()); + } else if (type.isMapType()) { + MapType mapType = (MapType) type; + return containsTimeStampNs(mapType.getKeyType()) || containsTimeStampNs(mapType.getValueType()); + } else if (type.isStructType()) { + for (StructField field : ((StructType) type).getFields()) { + if (containsTimeStampNs(field.getType())) { + return true; + } + } + } + return false; + } + public void executeAndSendResult(boolean isOutfileQuery, boolean isSendFields, Queriable queryStmt, MysqlChannel channel, CacheAnalyzer cacheAnalyzer, InternalService.PFetchCacheResult cacheResult) throws Exception { @@ -2025,6 +2059,11 @@ protected void sendBinaryResultRow(ResultSet resultSet) throws IOException { serializer.writeInt4((int) microSecond); } break; + case TIMESTAMP_NS: + // MySQL temporal binary values cannot carry nanoseconds. The metadata advertises + // MYSQL_TYPE_STRING, so encode the result as length-encoded text. + serializer.writeLenEncodedString(item); + break; default: serializer.writeLenEncodedString(item); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionRange.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionRange.java index acdb244e482939..bbf0984a765f32 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionRange.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionRange.java @@ -129,6 +129,7 @@ public boolean init(Type type, LiteralExpr expr) { case TIMEV2: case DATETIME: case DATETIMEV2: + case TIMESTAMP_NS: case TIMESTAMPTZ: case FLOAT: case DOUBLE: diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelper.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelper.java index b89abf16e83395..03859b923720d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelper.java @@ -134,6 +134,8 @@ private static ArrowType getArrowType(PrimitiveType primitiveType, Integer preci } else { return new ArrowType.Timestamp(TimeUnit.SECOND, timeZone); } + case TIMESTAMP_NS: + throw new IllegalArgumentException("TIMESTAMP_NS is not supported by Arrow Flight SQL"); case TIMESTAMPTZ: if (scale > 3) { return new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java index e3b67a4dd51baf..e22db8c7d7a18c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java @@ -261,8 +261,9 @@ public static LiteralExpr readableValue(Type type, String columnValue) throws An case DATETIME: case DATEV2: case DATETIMEV2: + case TIMESTAMP_NS: case TIMESTAMPTZ: - return DateLiteralUtils.createDateLiteral(columnValue, type); + return DateLiteralUtils.createLiteral(columnValue, type); case CHAR: case VARCHAR: case STRING: @@ -313,6 +314,7 @@ public static double convertToDouble(Type type, String columnValue) throws Analy return literal.getDouble(); case DATETIMEV2: + case TIMESTAMP_NS: case DATETIME: DateTimeLiteral dateTimeLiteral = new DateTimeLiteral(columnValue); return dateTimeLiteral.getDouble(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index 19ac5631c93482..6ac88ab8a56df3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -17,6 +17,7 @@ package org.apache.doris.tablefunction; +import org.apache.doris.analysis.DateLiteralUtils; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.authentication.AuthenticationIntegrationMeta; import org.apache.doris.authentication.RoleMappingMeta; @@ -2157,6 +2158,11 @@ private static List partitionValuesRows(List partitionCols, List org.apache.doris.nereids.trees.plans.commands.info.DefaultValue + .currentTimeStampDefaultValueWithPrecision(invalidPrecision)); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralUtilsTest.java index 08d472d54865ff..a2d515835c2528 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralUtilsTest.java @@ -23,15 +23,23 @@ import org.apache.doris.common.util.TimeUtils; import org.apache.doris.foundation.format.FormatOptions; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TExpr; +import org.apache.doris.thrift.TExprNodeType; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.time.Instant; +import java.time.LocalDateTime; import java.util.TimeZone; public class DateLiteralUtilsTest { + private static TimeStampNsLiteral createTimeStampNsLiteral(String value) throws AnalysisException { + return (TimeStampNsLiteral) DateLiteralUtils.createLiteral( + value, ScalarType.createTimeStampNsType()); + } + @Test public void testTimestampTzInit() throws AnalysisException { String value; @@ -205,6 +213,226 @@ public void testDatetimeV2WithMicroseconds() throws AnalysisException { Assertions.assertTrue(dl0.getType().isDatetimeV2()); } + @Test + public void testTimestampNsWithNanoseconds() throws AnalysisException { + TimeStampNsLiteral dl7 = createTimeStampNsLiteral("2023-06-15 14:30:45.1234567"); + Assertions.assertTrue(dl7.getType().isTimeStampNs()); + Assertions.assertEquals(123456700, dl7.getNanosecond()); + Assertions.assertEquals(123456, dl7.getMicrosecond()); + Assertions.assertEquals("2023-06-15 14:30:45.123456700", dl7.getStringValue()); + + TimeStampNsLiteral dl8 = createTimeStampNsLiteral("2023-06-15 14:30:45.12345678"); + Assertions.assertTrue(dl8.getType().isTimeStampNs()); + Assertions.assertEquals(123456780, dl8.getNanosecond()); + Assertions.assertEquals("2023-06-15 14:30:45.123456780", dl8.getStringValue()); + + TimeStampNsLiteral dl9 = createTimeStampNsLiteral("2023-06-15 14:30:45.123456789"); + Assertions.assertTrue(dl9.getType().isTimeStampNs()); + Assertions.assertEquals(123456789, dl9.getNanosecond()); + Assertions.assertEquals("2023-06-15 14:30:45.123456789", dl9.getStringValue()); + } + + @Test + public void testDateTimeV2MaximumSentinelUsesMicroseconds() throws AnalysisException { + DateLiteral maximum = new DateLiteral(Type.DATETIMEV2_WITH_MAX_SCALAR, true); + Assertions.assertEquals("9999-12-31 23:59:59.999999", maximum.getStringValue()); + Assertions.assertEquals(999999, maximum.getMicrosecond()); + } + + @Test + public void testTimestampNsUsesFixedNanosecondPrecision() throws AnalysisException { + TimeStampNsLiteral rounded7 = createTimeStampNsLiteral("2023-06-15 14:30:45.12345675"); + TimeStampNsLiteral canonical7 = createTimeStampNsLiteral("2023-06-15 14:30:45.123456750"); + Assertions.assertEquals("2023-06-15 14:30:45.123456750", rounded7.getStringValue()); + Assertions.assertEquals(123456750, rounded7.getNanosecond()); + Assertions.assertEquals(canonical7, rounded7); + Assertions.assertEquals(canonical7.hashCode(), rounded7.hashCode()); + + TimeStampNsLiteral rounded8 = createTimeStampNsLiteral("2023-06-15 14:30:45.123456785"); + Assertions.assertEquals("2023-06-15 14:30:45.123456785", rounded8.getStringValue()); + Assertions.assertEquals(123456785, rounded8.getNanosecond()); + + TimeStampNsLiteral rounded9 = createTimeStampNsLiteral("2023-06-15 14:30:45.1234567895"); + Assertions.assertEquals("2023-06-15 14:30:45.123456790", rounded9.getStringValue()); + Assertions.assertEquals(123456790, rounded9.getNanosecond()); + + TimeStampNsLiteral carry7 = createTimeStampNsLiteral("2023-06-15 14:30:45.99999995"); + TimeStampNsLiteral carry8 = createTimeStampNsLiteral("2023-06-15 14:30:45.999999995"); + TimeStampNsLiteral carry9 = createTimeStampNsLiteral("2023-06-15 14:30:45.9999999995"); + Assertions.assertEquals("2023-06-15 14:30:45.999999950", carry7.getStringValue()); + Assertions.assertEquals("2023-06-15 14:30:45.999999995", carry8.getStringValue()); + Assertions.assertEquals("2023-06-15 14:30:46.000000000", carry9.getStringValue()); + } + + @Test + public void testTimestampNsRoundFloorRequiresFixedScale() throws AnalysisException { + TimeStampNsLiteral timestampNs = createTimeStampNsLiteral("2023-06-15 14:30:45.123456789"); + + timestampNs.roundFloor(ScalarType.TIMESTAMP_NS_SCALE); + Assertions.assertEquals(123456789, timestampNs.getNanosecond()); + + int[] invalidScales = {-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10}; + for (int scale : invalidScales) { + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, () -> timestampNs.roundFloor(scale)); + Assertions.assertEquals("TIMESTAMP_NS has fixed scale " + ScalarType.TIMESTAMP_NS_SCALE + + ", but got " + scale, exception.getMessage()); + Assertions.assertEquals(123456789, timestampNs.getNanosecond()); + } + } + + @Test + public void testTimestampNsTypeInference() throws AnalysisException { + TimeStampNsLiteral upgraded = (TimeStampNsLiteral) DateLiteralUtils.createLiteral( + "2023-06-15 14:30:45.123456789", Type.DATETIME); + Assertions.assertTrue(upgraded.getType().isTimeStampNs()); + Assertions.assertEquals(123456789, upgraded.getNanosecond()); + + TimeStampNsLiteral inferredScale7 = (TimeStampNsLiteral) DateLiteralUtils.createLiteral( + "2023-06-15 14:30:45.123456700", null); + Assertions.assertTrue(inferredScale7.getType().isTimeStampNs()); + Assertions.assertEquals(123456700, inferredScale7.getNanosecond()); + Assertions.assertEquals("2023-06-15 14:30:45.123456700", inferredScale7.getStringValue()); + + TimeStampNsLiteral inferredScale9 = (TimeStampNsLiteral) DateLiteralUtils.createLiteral( + "1970-01-01 00:00:00.000000001", null); + Assertions.assertTrue(inferredScale9.getType().isTimeStampNs()); + Assertions.assertEquals(1, inferredScale9.getNanosecond()); + Assertions.assertEquals(1L, inferredScale9.getRealValue()); + } + + @Test + public void testTimestampNsEpochAndBoundaries() throws AnalysisException { + TimeStampNsLiteral beforeEpoch = createTimeStampNsLiteral("1969-12-31 23:59:59.999999999"); + Assertions.assertEquals(-1L, beforeEpoch.getRealValue()); + + TimeStampNsLiteral epoch = createTimeStampNsLiteral("1970-01-01 00:00:00.000000000"); + Assertions.assertEquals(0L, epoch.getRealValue()); + + TimeStampNsLiteral lowerBoundary = createTimeStampNsLiteral("1677-09-21 00:12:43.145224192"); + Assertions.assertEquals(Long.MIN_VALUE, lowerBoundary.getRealValue()); + + TimeStampNsLiteral upperBoundary = createTimeStampNsLiteral("2262-04-11 23:47:16.854775807"); + Assertions.assertEquals(Long.MAX_VALUE, upperBoundary.getRealValue()); + + TimeStampNsLiteral minimum = new TimeStampNsLiteral(false); + Assertions.assertEquals(Long.MIN_VALUE, minimum.getRealValue()); + Assertions.assertTrue(minimum.isMinValue()); + + TimeStampNsLiteral maximum = new TimeStampNsLiteral(true); + Assertions.assertEquals(Long.MAX_VALUE, maximum.getRealValue()); + + epoch.setMinValue(); + Assertions.assertEquals(Long.MIN_VALUE, epoch.getRealValue()); + Assertions.assertTrue(epoch.isMinValue()); + } + + @Test + public void testTimestampNsDateArithmeticPreservesLiteralType() throws AnalysisException { + TimeStampNsLiteral literal = new TimeStampNsLiteral( + 1970, 1, 1, 0, 0, 0, 123456789); + + TimeStampNsLiteral result = literal.plusSeconds(1).plusDays(1); + + Assertions.assertInstanceOf(TimeStampNsLiteral.class, result); + Assertions.assertEquals("1970-01-02 00:00:01.123456789", result.getStringValue()); + Assertions.assertEquals(86401123456789L, result.getRealValue()); + } + + @Test + public void testTimestampNsIsIndependentFromDateLiteral() throws AnalysisException { + TimeStampNsLiteral timestampNs = createTimeStampNsLiteral( + "2024-01-02 03:04:05.123456000"); + DateLiteral datetimeV2 = DateLiteralUtils.createDateLiteral( + "2024-01-02 03:04:05.123456", ScalarType.createDatetimeV2Type(6)); + + Assertions.assertInstanceOf(LiteralExpr.class, timestampNs); + Assertions.assertFalse(DateLiteral.class.isInstance(timestampNs)); + Assertions.assertEquals(0, timestampNs.compareLiteral(datetimeV2)); + Assertions.assertEquals(0, datetimeV2.compareLiteral(timestampNs)); + Assertions.assertEquals(timestampNs, datetimeV2); + Assertions.assertEquals(datetimeV2, timestampNs); + Assertions.assertEquals(datetimeV2.hashCode(), timestampNs.hashCode()); + Assertions.assertInstanceOf(TimeStampNsLiteral.class, timestampNs.clone()); + Assertions.assertEquals("'2024-01-02 03:04:05.123456000'", + timestampNs.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITH_TABLE)); + TExpr thrift = ExprToThriftVisitor.treeToThrift(timestampNs); + Assertions.assertEquals(TExprNodeType.DATE_LITERAL, thrift.nodes.get(0).node_type); + Assertions.assertEquals(timestampNs.getStringValue(), thrift.nodes.get(0).date_literal.value); + + Assertions.assertThrows(AnalysisException.class, () -> DateLiteralUtils.createDateLiteral( + timestampNs.getStringValue(), ScalarType.createTimeStampNsType())); + } + + @Test + public void testTimestampNsCheckRangeUsesInt64EpochNanosecondBounds() { + Assertions.assertFalse(new TimeStampNsLiteral( + 1677, 9, 21, 0, 12, 43, 145224192).checkRange()); + Assertions.assertTrue(new TimeStampNsLiteral( + 1677, 9, 21, 0, 12, 43, 145224191).checkRange()); + Assertions.assertFalse(new TimeStampNsLiteral( + 2262, 4, 11, 23, 47, 16, 854775807).checkRange()); + Assertions.assertTrue(new TimeStampNsLiteral( + 2262, 4, 11, 23, 47, 16, 854775808).checkRange()); + Assertions.assertTrue(new TimeStampNsLiteral( + 2024, 13, 1, 0, 0, 0, 0).checkRange()); + + DateLiteral maxDatetimeV2 = new DateLiteral( + 9999, 12, 31, 23, 59, 59, 999999, ScalarType.createDatetimeV2Type(6)); + Assertions.assertFalse(maxDatetimeV2.checkRange()); + } + + @Test + public void testFractionalSecondRangeIsTypeSpecific() throws AnalysisException { + LocalDateTime lastMicrosecond = LocalDateTime.of(2024, 1, 1, 0, 0, 0, 999999000); + LocalDateTime afterLastMicrosecond = LocalDateTime.of(2024, 1, 1, 0, 0, 0, 999999001); + + for (Type type : new Type[] { + ScalarType.createDatetimeV2Type(6), ScalarType.createTimeStampTzType(6)}) { + DateLiteral valid = new DateLiteral(lastMicrosecond, type); + Assertions.assertFalse(valid.checkRange()); + Assertions.assertDoesNotThrow(valid::checkValueValid); + + DateLiteral canonicalized = new DateLiteral(afterLastMicrosecond, type); + Assertions.assertEquals(999999, canonicalized.getMicrosecond()); + Assertions.assertFalse(canonicalized.checkRange()); + Assertions.assertDoesNotThrow(canonicalized::checkValueValid); + } + + TimeStampNsLiteral timestampNs = new TimeStampNsLiteral( + LocalDateTime.of(2024, 1, 1, 0, 0, 0, 999999999)); + Assertions.assertFalse(timestampNs.checkRange()); + Assertions.assertDoesNotThrow(timestampNs::checkValueValid); + + TimeStampNsLiteral invalidTimestampNs = new TimeStampNsLiteral( + 2024, 1, 1, 0, 0, 0, 1000000000L); + Assertions.assertTrue(invalidTimestampNs.checkRange()); + Assertions.assertThrows(AnalysisException.class, invalidTimestampNs::checkValueValid); + } + + @Test + public void testTimestampNsTimezoneOffset() throws AnalysisException { + TimeStampNsLiteral utc = createTimeStampNsLiteral("2023-06-15 12:00:00.123456789+00:00"); + TimeStampNsLiteral plus8 = createTimeStampNsLiteral("2023-06-15 20:00:00.123456789+08:00"); + TimeStampNsLiteral minus5 = createTimeStampNsLiteral("2023-06-15 07:00:00.123456789-05:00"); + + Assertions.assertEquals(utc.getStringValue(), plus8.getStringValue()); + Assertions.assertEquals(utc.getStringValue(), minus5.getStringValue()); + Assertions.assertEquals(123456789, utc.getNanosecond()); + Assertions.assertEquals(123456789, plus8.getNanosecond()); + Assertions.assertEquals(123456789, minus5.getNanosecond()); + } + + @Test + public void testInvalidTimestampNsseconds() { + Assertions.assertThrows(AnalysisException.class, () -> DateLiteralUtils.createLiteral( + "2023-02-29 14:30:45.123456789", ScalarType.createTimeStampNsType())); + Assertions.assertThrows(AnalysisException.class, () -> DateLiteralUtils.createLiteral( + "1677-09-21 00:12:43.145224191", ScalarType.createTimeStampNsType())); + Assertions.assertThrows(AnalysisException.class, () -> DateLiteralUtils.createLiteral( + "2262-04-11 23:47:16.854775808", ScalarType.createTimeStampNsType())); + } + @Test public void testCompactDateFormats() throws AnalysisException { // YYYYMMDD compact date diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprGsonSerializationTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprGsonSerializationTest.java index 83b057759ed5a5..5eda48b0d0251b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprGsonSerializationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprGsonSerializationTest.java @@ -146,6 +146,8 @@ private Map, Expr> createExprSamples() throws Exception { samples.put(StringLiteral.class, new StringLiteral("expr-gson")); samples.put(StructLiteral.class, new StructLiteral(new StructType(), new IntLiteral(7L), new StringLiteral("field"))); + samples.put(TimeStampNsLiteral.class, + new TimeStampNsLiteral(2024, 4, 27, 12, 34, 56, 123456789)); samples.put(TimeV2Literal.class, new TimeV2Literal(12, 34, 56, 123456, 6, false)); samples.put(VarBinaryLiteral.class, new VarBinaryLiteral("bin".getBytes(StandardCharsets.UTF_8))); samples.put(BetweenPredicate.class, createBetweenPredicate()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprToStringValueVisitorTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprToStringValueVisitorTest.java index 9391d622cc6226..7b3d741b60b089 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprToStringValueVisitorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprToStringValueVisitorTest.java @@ -319,6 +319,17 @@ public void testDateLiteralInComplexType() throws Exception { V.visitDateLiteral(d, StringValueContext.forQuery(FormatOptions.getDefault()).asComplexType())); } + @Test + public void testTimeStampNsLiteral() { + TimeStampNsLiteral timestampNs = new TimeStampNsLiteral( + 2024, 1, 15, 10, 30, 0, 123456789); + Assertions.assertEquals("2024-01-15 10:30:00.123456789", + timestampNs.accept(V, StringValueContext.forQuery(FormatOptions.getDefault()))); + Assertions.assertEquals("\"2024-01-15 10:30:00.123456789\"", + timestampNs.accept(V, + StringValueContext.forQuery(FormatOptions.getDefault()).asComplexType())); + } + // ======================== TimeStampTz ======================== @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java index 977d11c24a9141..90421e81053c6a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java @@ -155,6 +155,18 @@ public void testCreatePythonFunctionRejectsObjectTypes() throws Exception { assertCreateFunctionAnalysisException(ctx, "create function py_obj_type_db.j_bitmap_arg(bitmap) returns int " + "properties('type'='JAVA_UDF', 'symbol'='evaluate');", "JAVA_UDF does not support argument 1 type bitmap"); + assertCreateFunctionAnalysisException(ctx, + "create function py_obj_type_db.j_timestamp_ns_arg(timestamp_ns) returns int " + + "properties('type'='JAVA_UDF', 'symbol'='evaluate');", + "JAVA_UDF does not support argument 1 type timestamp_ns"); + assertCreateFunctionAnalysisException(ctx, + "create function py_obj_type_db.j_timestamp_ns_ret(int) returns timestamp_ns " + + "properties('type'='JAVA_UDF', 'symbol'='evaluate');", + "JAVA_UDF does not support return type timestamp_ns"); + assertCreateFunctionAnalysisException(ctx, + "create function py_obj_type_db.j_timestamp_ns_array(array) returns int " + + "properties('type'='JAVA_UDF', 'symbol'='evaluate');", + "JAVA_UDF does not support argument 1 element type timestamp_ns"); assertCreateFunctionAnalysisException(ctx, "create function py_obj_type_db.py_hll_ret(int) returns hll " + "properties('type'='PYTHON_UDF', 'symbol'='evaluate', 'runtime_version'='3.10.2');", "PYTHON_UDF does not support return type hll"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java index b5148f982eacf7..65d56ec8a95946 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java @@ -27,6 +27,8 @@ import org.junit.BeforeClass; import org.junit.Test; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; import java.nio.file.Files; @@ -292,6 +294,60 @@ public void testMaxValueToSql() throws Exception { Assert.assertEquals("(MAXVALUE, MAXVALUE, MAXVALUE, MAXVALUE, MAXVALUE, MAXVALUE, MAXVALUE)", key.toSql()); } + @Test + public void testTimeStampNsMinValue() throws Exception { + assertTimeStampNsMinValue( + "1677-09-21 00:12:43.145224192", "1677-09-21 00:12:43.145224193"); + } + + @Test + public void testTimeStampNsSuccessor() throws Exception { + Column column = new Column("timestamp_ns", ScalarType.createTimeStampNsType()); + PartitionKey key = PartitionKey.createPartitionKey( + Arrays.asList(new PartitionValue("1970-01-01 00:00:00.123456789")), + Arrays.asList(column)); + + PartitionKey successor = key.successor(); + + Assert.assertTrue(successor.getKeys().get(0) + instanceof org.apache.doris.analysis.TimeStampNsLiteral); + Assert.assertEquals("1970-01-01 00:00:00.123456790", + successor.getKeys().get(0).getStringValue()); + } + + @Test + public void testTimeStampNsSerialization() throws Exception { + Column column = new Column("timestamp_ns", ScalarType.createTimeStampNsType()); + PartitionKey key = PartitionKey.createPartitionKey( + Arrays.asList(new PartitionValue("1970-01-01 00:00:00.123456789")), + Arrays.asList(column)); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + key.write(new DataOutputStream(bytes)); + PartitionKey restored = PartitionKey.read( + new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))); + + Assert.assertEquals(key, restored); + Assert.assertTrue(restored.getKeys().get(0) + instanceof org.apache.doris.analysis.TimeStampNsLiteral); + Assert.assertEquals("1970-01-01 00:00:00.123456789", + restored.getKeys().get(0).getStringValue()); + } + + private void assertTimeStampNsMinValue(String minValue, String nextValue) throws Exception { + Column column = new Column("timestamp_ns", ScalarType.createTimeStampNsType()); + PartitionKey infinityMin = PartitionKey.createInfinityPartitionKey(Arrays.asList(column), false); + PartitionKey literalMin = PartitionKey.createPartitionKey( + Arrays.asList(new PartitionValue(minValue)), Arrays.asList(column)); + PartitionKey literalNext = PartitionKey.createPartitionKey( + Arrays.asList(new PartitionValue(nextValue)), Arrays.asList(column)); + + Assert.assertTrue(infinityMin.isMinValue()); + Assert.assertTrue(literalMin.isMinValue()); + Assert.assertEquals(infinityMin, literalMin); + Assert.assertFalse(literalNext.isMinValue()); + } + @Test public void testTimestampTzPartitionKeyKeepsExplicitOffset() throws Exception { boolean originalRunningUnitTest = FeConstants.runningUnitTest; diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java index 37a66edfc1f51d..f36268d61fe0f5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java @@ -246,5 +246,19 @@ public void testDatetimeV2ScaleMatching() { Assert.assertFalse(Type.matchExactType(dtv2s3, dtv2s6, true)); // Same scale -> match Assert.assertTrue(Type.matchExactType(dtv2s6, ScalarType.createDatetimeV2Type(6), false)); + + ScalarType timestampNs = ScalarType.createTimeStampNsType(); + Assert.assertNotEquals(timestampNs, dtv2s6); + for (int invalidScale = 7; invalidScale <= 9; invalidScale++) { + ScalarType invalidDatetimeV2 = ScalarType.createDatetimeV2Type(invalidScale); + Assert.assertTrue(invalidDatetimeV2.isDatetimeV2()); + Assert.assertFalse(invalidDatetimeV2.isTimeStampNs()); + Assert.assertNotEquals(timestampNs, invalidDatetimeV2); + } + ScalarType wildcardDatetimeV2 = ScalarType.createDatetimeV2Type(-1); + Assert.assertFalse(timestampNs.matchesType(wildcardDatetimeV2)); + Assert.assertFalse(timestampNs.matchesType(dtv2s6)); + Assert.assertFalse(dtv2s6.matchesType(timestampNs)); + Assert.assertFalse(Type.matchExactType(timestampNs, wildcardDatetimeV2, false)); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/connector/converter/NereidsToConnectorExpressionConverterTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/connector/converter/NereidsToConnectorExpressionConverterTest.java index 3d2e9e96eead7d..187eddf9e3d938 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/connector/converter/NereidsToConnectorExpressionConverterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/connector/converter/NereidsToConnectorExpressionConverterTest.java @@ -52,6 +52,7 @@ import org.apache.doris.nereids.trees.expressions.literal.DecimalV3Literal; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral; +import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; import org.apache.doris.nereids.types.BigIntType; @@ -145,6 +146,11 @@ public void dateAndDatetimeLiteralsBecomeJavaTimeValues() { new EqualTo(slot("ts", ScalarType.createDatetimeV2Type(0)), new DateTimeV2Literal(2024, 1, 2, 12, 34, 56)))); Assertions.assertEquals(LocalDateTime.of(2024, 1, 2, 12, 34, 56), ts.getValue()); + + ConnectorLiteral nanoTs = rightLiteralOf(NereidsToConnectorExpressionConverter.convert( + new EqualTo(slot("timestamp_ns", ScalarType.createTimeStampNsType()), + new TimeStampNsLiteral("2024-01-02 12:34:56.123456789")))); + Assertions.assertEquals(LocalDateTime.of(2024, 1, 2, 12, 34, 56, 123456789), nanoTs.getValue()); } // ---- node shape mapping ---- diff --git a/fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlSerializerDateTimeTest.java b/fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlSerializerDateTimeTest.java new file mode 100644 index 00000000000000..924e07755f66de --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlSerializerDateTimeTest.java @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mysql; + +import org.apache.doris.catalog.MysqlColType; +import org.apache.doris.catalog.ScalarType; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class MysqlSerializerDateTimeTest { + private static int skipLenEncodedString(byte[] buffer, int offset) { + return offset + 1 + (buffer[offset] & 0xFF); + } + + private static int fieldMetadataOffset(byte[] buffer) { + int offset = 0; + for (int i = 0; i < 6; i++) { + offset = skipLenEncodedString(buffer, offset); + } + return offset + 1; + } + + @Test + public void testTimestampNsMetadata() { + MysqlSerializer serializer = MysqlSerializer.newInstance(); + ScalarType type = ScalarType.createTimeStampNsType(); + Assertions.assertTrue(type.isTimeStampNs()); + Assertions.assertEquals(29, serializer.getMysqlTypeLength(type)); + Assertions.assertEquals(0, serializer.getMysqlDecimals(type)); + + serializer.writeField("dt", type); + byte[] field = serializer.toArray(); + int metadataOffset = fieldMetadataOffset(field); + int mysqlTypeOffset = metadataOffset + 2 + 4; + Assertions.assertEquals(MysqlColType.MYSQL_TYPE_STRING.getCode(), + field[mysqlTypeOffset] & 0xFF); + int decimalsOffset = mysqlTypeOffset + 1 + 2; + Assertions.assertEquals(0, field[decimalsOffset] & 0xFF); + } + + @Test + public void testDatetimeV2MicrosecondMetadata() { + for (int scale = 0; scale <= 6; scale++) { + MysqlSerializer serializer = MysqlSerializer.newInstance(); + ScalarType type = ScalarType.createDatetimeV2Type(scale); + Assertions.assertEquals(scale, serializer.getMysqlDecimals(type)); + + serializer.writeField("dt", type); + byte[] field = serializer.toArray(); + int metadataOffset = fieldMetadataOffset(field); + int mysqlTypeOffset = metadataOffset + 2 + 4; + Assertions.assertEquals(MysqlColType.MYSQL_TYPE_DATETIME.getCode(), + field[mysqlTypeOffset] & 0xFF); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java index 9f46a7970a2c72..6dbc84ae24cf7b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java @@ -331,7 +331,8 @@ void testFoldDate() { MilliSecondsAdd millisecondsAdd = new MilliSecondsAdd( DateTimeV2Literal.fromJavaDateType(LocalDateTime.of(1, 1, 1, 1, 1, 1)), new BigIntLiteral(1)); rewritten = executor.rewrite(millisecondsAdd, context); - Assertions.assertEquals(new DateTimeV2Literal(DateTimeV2Type.MAX, "0001-01-01 01:01:01.001000"), rewritten); + Assertions.assertEquals( + new DateTimeV2Literal(DateTimeV2Type.of(6), "0001-01-01 01:01:01.001000"), rewritten); // fail to fold, because the result is out of range millisecondsAdd = new MilliSecondsAdd( DateTimeV2Literal.fromJavaDateType(LocalDateTime.of(9999, 12, 31, 23, 59, 59)), @@ -1313,6 +1314,7 @@ void testDateTimeV2TypeDateTimeArithmeticFunctions() { .secondCeil(new DateTimeV2Literal("2021-01-01 12:12:12.123"), new IntegerLiteral(2))) == 0); Assertions.assertTrue(new DateTimeV2Literal("2021-01-01 12:12:12.000000").compareTo((ComparableLiteral) TimeRoundSeries .secondFloor(new DateTimeV2Literal("2021-01-01 12:12:12.123"), new IntegerLiteral(2))) == 0); + } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/check/CheckCastTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/check/CheckCastTest.java index b946bd5749a6c3..7969727e14ea28 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/check/CheckCastTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/check/CheckCastTest.java @@ -42,6 +42,7 @@ import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeV2Type; import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.types.VarcharType; @@ -56,6 +57,20 @@ import java.util.List; public class CheckCastTest { + @Test + public void testTimeStampNsOnlySupportsStringCasts() { + Assertions.assertTrue(CheckCast.check(StringType.INSTANCE, TimeStampNsType.INSTANCE, true)); + Assertions.assertTrue(CheckCast.check(TimeStampNsType.INSTANCE, StringType.INSTANCE, true)); + Assertions.assertTrue(CheckCast.check(StringType.INSTANCE, TimeStampNsType.INSTANCE, false)); + Assertions.assertTrue(CheckCast.check(TimeStampNsType.INSTANCE, StringType.INSTANCE, false)); + Assertions.assertFalse(CheckCast.check(DateTimeV2Type.MAX, TimeStampNsType.INSTANCE, true)); + Assertions.assertFalse(CheckCast.check(TimeStampNsType.INSTANCE, DateTimeV2Type.MAX, true)); + Assertions.assertFalse(CheckCast.check(DateV2Type.INSTANCE, TimeStampNsType.INSTANCE, true)); + Assertions.assertFalse(CheckCast.check(TimeStampNsType.INSTANCE, DateV2Type.INSTANCE, true)); + Assertions.assertFalse(CheckCast.check(DateTimeV2Type.MAX, TimeStampNsType.INSTANCE, false)); + Assertions.assertFalse(CheckCast.check(TimeStampNsType.INSTANCE, DateTimeV2Type.MAX, false)); + } + @Test public void testCastBetweenVariantTypes() { VariantType v1Source = new VariantType(100); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyCastRuleTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyCastRuleTest.java index 0daf90ca5abf65..36281d7b75a5af 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyCastRuleTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyCastRuleTest.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.rules.expression.ExpressionRuleExecutor; import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; import org.apache.doris.nereids.trees.expressions.literal.CharLiteral; import org.apache.doris.nereids.trees.expressions.literal.DecimalLiteral; @@ -37,6 +38,7 @@ import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.SmallIntType; import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.types.VarcharType; @@ -67,6 +69,9 @@ public void testSimplify() { Expression tinyIntLiteral = new TinyIntLiteral((byte) 12); // cast tinyint as tinyint assertRewrite(new Cast(tinyIntLiteral, TinyIntType.INSTANCE), tinyIntLiteral); + + Expression timestampNs = new SlotReference("timestamp_ns", TimeStampNsType.INSTANCE, true); + assertRewrite(new Cast(timestampNs, TimeStampNsType.INSTANCE), timestampNs); // cast tinyint as decimalv2(3,0) assertRewrite(new Cast(tinyIntLiteral, DecimalV2Type.forType(TinyIntType.INSTANCE)), new DecimalLiteral(DecimalV2Type.forType(TinyIntType.INSTANCE), new BigDecimal(12))); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateTest.java index 7d6967e6b3bd10..3e6a5882ff1ef9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateTest.java @@ -46,6 +46,7 @@ import org.apache.doris.nereids.trees.expressions.literal.LargeIntLiteral; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral; +import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.BooleanType; @@ -60,6 +61,7 @@ import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.LargeIntType; import org.apache.doris.nereids.types.SmallIntType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.util.ExpressionUtils; @@ -460,6 +462,71 @@ void testDateTimeV2CmpDateTimeV2() { ExpressionUtils.falseOrNull(datetime2)); } + @Test + void testTimeStampNsBoundaryDateComparison() { + executor = new ExpressionRuleExecutor(ImmutableList.of( + bottomUp(SimplifyComparisonPredicate.INSTANCE) + )); + + Expression datetime9 = new SlotReference("timestampNs", TimeStampNsType.INSTANCE, true); + Expression castToDate = new Cast(datetime9, DateV2Type.INSTANCE); + DateV2Literal minDate = new DateV2Literal("1677-09-21"); + DateV2Literal maxDate = new DateV2Literal("2262-04-11"); + TimeStampNsLiteral minValue = TimeStampNsLiteral.getMinValue(); + TimeStampNsLiteral maxValue = TimeStampNsLiteral.getMaxValue(); + TimeStampNsLiteral minDayEnd = new TimeStampNsLiteral("1677-09-21 23:59:59.999999999"); + TimeStampNsLiteral maxDayStart = new TimeStampNsLiteral("2262-04-11 00:00:00.000000000"); + + assertRewrite(new EqualTo(castToDate, minDate), new And( + new GreaterThanEqual(datetime9, minValue), new LessThanEqual(datetime9, minDayEnd))); + assertRewrite(new EqualTo(castToDate, maxDate), new And( + new GreaterThanEqual(datetime9, maxDayStart), new LessThanEqual(datetime9, maxValue))); + assertRewrite(new GreaterThanEqual(castToDate, minDate), new GreaterThanEqual(datetime9, minValue)); + assertRewrite(new LessThan(castToDate, minDate), new LessThan(datetime9, minValue)); + assertRewrite(new GreaterThan(castToDate, maxDate), new GreaterThan(datetime9, maxValue)); + assertRewrite(new LessThanEqual(castToDate, maxDate), new LessThanEqual(datetime9, maxValue)); + + DateV2Literal beforeMinDate = new DateV2Literal("1677-09-20"); + DateV2Literal afterMaxDate = new DateV2Literal("2262-04-12"); + assertRewrite(new EqualTo(castToDate, beforeMinDate), ExpressionUtils.falseOrNull(datetime9)); + assertRewrite(new GreaterThan(castToDate, beforeMinDate), ExpressionUtils.trueOrNull(datetime9)); + assertRewrite(new EqualTo(castToDate, afterMaxDate), ExpressionUtils.falseOrNull(datetime9)); + assertRewrite(new LessThan(castToDate, afterMaxDate), ExpressionUtils.trueOrNull(datetime9)); + } + + @Test + void testDateToTimeStampNsSubMicrosecondComparison() { + executor = new ExpressionRuleExecutor(ImmutableList.of( + bottomUp(SimplifyComparisonPredicate.INSTANCE) + )); + + Expression date = new SlotReference("date", DateV2Type.INSTANCE, true); + Expression castToNano = new Cast(date, TimeStampNsType.INSTANCE); + TimeStampNsLiteral nanoAfterMidnight = new TimeStampNsLiteral( + "2024-01-01 00:00:00.000000001"); + DateV2Literal currentDate = new DateV2Literal("2024-01-01"); + DateV2Literal nextDate = new DateV2Literal("2024-01-02"); + + assertRewrite(new EqualTo(castToNano, nanoAfterMidnight), ExpressionUtils.falseOrNull(date)); + assertRewrite(new EqualTo(nanoAfterMidnight, castToNano), ExpressionUtils.falseOrNull(date)); + assertRewrite(new NullSafeEqual(castToNano, nanoAfterMidnight), BooleanLiteral.FALSE); + assertRewrite(new NullSafeEqual(nanoAfterMidnight, castToNano), BooleanLiteral.FALSE); + + assertRewrite(new LessThan(castToNano, nanoAfterMidnight), new LessThan(date, nextDate)); + assertRewrite(new LessThan(nanoAfterMidnight, castToNano), new GreaterThan(date, currentDate)); + assertRewrite(new LessThanEqual(castToNano, nanoAfterMidnight), new LessThanEqual(date, currentDate)); + assertRewrite(new LessThanEqual(nanoAfterMidnight, castToNano), new GreaterThanEqual(date, nextDate)); + assertRewrite(new GreaterThan(castToNano, nanoAfterMidnight), new GreaterThan(date, currentDate)); + assertRewrite(new GreaterThan(nanoAfterMidnight, castToNano), new LessThan(date, nextDate)); + assertRewrite(new GreaterThanEqual(castToNano, nanoAfterMidnight), new GreaterThanEqual(date, nextDate)); + assertRewrite(new GreaterThanEqual(nanoAfterMidnight, castToNano), new LessThanEqual(date, currentDate)); + + assertRewrite(new Not(new EqualTo(castToNano, nanoAfterMidnight)), + new Not(ExpressionUtils.falseOrNull(date))); + assertRewrite(new Not(new EqualTo(nanoAfterMidnight, castToNano)), + new Not(ExpressionUtils.falseOrNull(date))); + } + @Test void testRound() { executor = new ExpressionRuleExecutor(ImmutableList.of( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java index 8533adb3061ad3..033ba1c8fc4938 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java @@ -1406,6 +1406,34 @@ public void testStringRangeColToDateLiteral() { Assertions.assertEquals(4.24, filter2021.getRowCount(), 0.01); } + @Test + public void testStringRangeColToTimeStampNsLiteral() { + SlotReference timestamp = new SlotReference("timestamp", new VarcharType(30)); + ColumnStatistic timestampStats = new ColumnStatisticBuilder(100) + .setNdv(100) + .setAvgSizeByte(30) + .setNumNulls(0) + .setMaxExpr(new StringLiteral( + "2020-2-01 00:00:00.000000002").toLegacyLiteral()) + .setMaxValue(new VarcharLiteral( + "2020-2-01 00:00:00.000000002").getDouble()) + .setMinExpr(new StringLiteral( + "2020-1-01 00:00:00.000000001").toLegacyLiteral()) + .setMinValue(new VarcharLiteral( + "2020-1-01 00:00:00.000000001").getDouble()) + .build(); + Statistics baseStats = new StatisticsBuilder() + .setRowCount(100) + .putColumnStatistics(timestamp, timestampStats) + .build(); + + VarcharLiteral october = new VarcharLiteral("2020-10-01 00:00:00.000000003"); + Statistics result = new FilterEstimation().estimate( + new LessThan(timestamp, october), baseStats); + + Assertions.assertEquals(100, result.getRowCount()); + } + @Test public void testStringRangeColToCol() { SlotReference a = new SlotReference("a", new VarcharType(25)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java index 4964414e6b3db5..104db5c21118f6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java @@ -23,12 +23,14 @@ import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.proto.Types.PGenericType; import org.apache.doris.proto.Types.PGenericType.TypeId; import org.apache.doris.proto.Types.PValues; @@ -57,6 +59,54 @@ public void testEqual() { Assertions.assertTrue(Literal.of(false) instanceof BooleanLiteral); } + @Test + public void testGetResultExpressionTimeStampNs() { + assertTimeStampNsResult(123456800L, "1970-01-01 00:00:00.123456800"); + assertTimeStampNsResult(-10L, "1969-12-31 23:59:59.999999990"); + assertTimeStampNsResult(-1L, "1969-12-31 23:59:59.999999999"); + } + + @Test + public void testGetResultExpressionNestedTimeStampNs() { + TimeStampNsType nanoType = TimeStampNsType.INSTANCE; + PGenericType int64Type = PGenericType.newBuilder().setId(TypeId.INT64).build(); + PValues nanoValues = PValues.newBuilder() + .setType(int64Type) + .addInt64Value(-1L) + .addInt64Value(123456789L) + .build(); + PValues arrayValues = PValues.newBuilder() + .setType(PGenericType.newBuilder().setId(TypeId.LIST).build()) + .addChildElement(nanoValues) + .addChildOffset(2) + .build(); + + List result = org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnBE + .getResultExpression(ArrayType.of(nanoType), arrayValues); + Assertions.assertEquals(1, result.size()); + Assertions.assertTrue(result.get(0) instanceof ArrayLiteral); + List items = ((ArrayLiteral) result.get(0)).getValue(); + Assertions.assertEquals("1969-12-31 23:59:59.999999999", + ((TimeStampNsLiteral) items.get(0)).getStringValue()); + Assertions.assertEquals("1970-01-01 00:00:00.123456789", + ((TimeStampNsLiteral) items.get(1)).getStringValue()); + } + + private void assertTimeStampNsResult(long epochNanos, String expected) { + PValues values = PValues.newBuilder() + .setType(PGenericType.newBuilder().setId(TypeId.INT64).build()) + .addInt64Value(epochNanos) + .build(); + List result = org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnBE + .getResultExpression(TimeStampNsType.INSTANCE, values); + + Assertions.assertEquals(1, result.size()); + Assertions.assertInstanceOf(TimeStampNsLiteral.class, result.get(0)); + TimeStampNsLiteral literal = (TimeStampNsLiteral) result.get(0); + Assertions.assertSame(TimeStampNsType.INSTANCE, literal.getDataType()); + Assertions.assertEquals(expected, literal.getStringValue()); + } + @Test public void testGetResultExpressionArrayInt() { int num = 10; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java index 8b2db5b3511e8b..115d10e59888e5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.types.LargeIntType; import org.apache.doris.nereids.types.SmallIntType; import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; @@ -59,11 +60,13 @@ void testNormalize() { s = DateLiteral.normalize("2021-5-01 0:0:0.001").get(); Assertions.assertEquals("2021-05-01 00:00:00.001", s); s = DateLiteral.normalize("2021-5-01 0:0:0.12345678").get(); - Assertions.assertEquals("2021-05-01 00:00:00.1234567", s); + Assertions.assertEquals("2021-05-01 00:00:00.12345678", s); + s = DateLiteral.normalize("2021-5-01 0:0:0.12345678951").get(); + Assertions.assertEquals("2021-05-01 00:00:00.1234567895", s); s = DateLiteral.normalize("2021-5-1 Asia/Shanghai").get(); Assertions.assertEquals("2021-05-01Asia/Shanghai", s); s = DateLiteral.normalize("2021-5-1 0:0:0.12345678 Asia/Shanghai").get(); - Assertions.assertEquals("2021-05-01 00:00:00.1234567Asia/Shanghai", s); + Assertions.assertEquals("2021-05-01 00:00:00.12345678Asia/Shanghai", s); } @Test @@ -224,6 +227,64 @@ void testPoint() { new DateTimeV2Literal("2020.02.01 00.00.00.0000001"); } + @Test + void testTimeStampNsseconds() { + Assertions.assertFalse(DateTimeLiteral.class.isAssignableFrom(TimeStampNsLiteral.class)); + TimeStampNsLiteral nanoseconds = + new TimeStampNsLiteral("1970-01-01 00:00:00.123456789"); + Assertions.assertEquals(TimeStampNsType.INSTANCE, nanoseconds.getDataType()); + Assertions.assertEquals(123456789, nanoseconds.getNanoSecond()); + Assertions.assertEquals("1970-01-01 00:00:00.123456789", + nanoseconds.getStringValue()); + Assertions.assertEquals("1970-01-01 00:00:00.123456789", + nanoseconds.toLegacyLiteral().getStringValue()); + Assertions.assertInstanceOf(org.apache.doris.analysis.TimeStampNsLiteral.class, + nanoseconds.toLegacyLiteral()); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new DateTimeV2Literal(TimeStampNsType.INSTANCE, + "1970-01-01 00:00:00.123456789")); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new DateTimeV2Literal( + TimeStampNsType.INSTANCE, "1970-01-01 00:00:00.123456789")); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new DateTimeV2Literal( + TimeStampNsType.INSTANCE, 1970, 1, 1, 0, 0, 0, 123456789)); + + TimeStampNsLiteral rounded = new TimeStampNsLiteral("1970-01-01 00:00:00.12345675"); + Assertions.assertEquals("1970-01-01 00:00:00.123456750", + rounded.getStringValue()); + + TimeStampNsLiteral scale7 = new TimeStampNsLiteral("1969-12-31 23:59:59.99999994"); + Assertions.assertEquals("1969-12-31 23:59:59.999999940", + scale7.getStringValue()); + TimeStampNsLiteral scale8 = new TimeStampNsLiteral("1969-12-31 23:59:59.999999995"); + Assertions.assertEquals("1969-12-31 23:59:59.999999995", + scale8.getStringValue()); + for (String value : new String[] { + "2024-02-29 12:34:56.1234567", + "2024-02-29 12:34:56.12345678", + "2024-02-29 12:34:56.123456789"}) { + DateTimeV2Type inferredType = DateTimeV2Type.forTypeFromString(value); + Assertions.assertEquals(DateTimeV2Type.MAX_SCALE, inferredType.getScale()); + Assertions.assertFalse(inferredType instanceof TimeStampNsType); + } + + Assertions.assertDoesNotThrow(() -> + new TimeStampNsLiteral("1677-09-21 00:12:43.145224192")); + Assertions.assertDoesNotThrow(() -> + new TimeStampNsLiteral("2262-04-11 23:47:16.854775807")); + Assertions.assertEquals(Long.MIN_VALUE, + new TimeStampNsLiteral("1677-09-21 00:12:43.145224192") + .toLegacyLiteral().getRealValue()); + Assertions.assertEquals(Long.MAX_VALUE, + new TimeStampNsLiteral("2262-04-11 23:47:16.854775807") + .toLegacyLiteral().getRealValue()); + Assertions.assertThrows(AnalysisException.class, () -> + new TimeStampNsLiteral("1677-09-21 00:12:43.145224191")); + Assertions.assertThrows(AnalysisException.class, () -> + new TimeStampNsLiteral("2262-04-11 23:47:16.854775808")); + } + @Test void testSuffixSpace() { new DateLiteral("2016-07-02 "); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java index c3556f1bef4bcb..d7827722f858c9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java @@ -17,6 +17,8 @@ package org.apache.doris.nereids.trees.expressions.literal; +import org.apache.doris.catalog.MysqlColType; +import org.apache.doris.common.Config; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.exceptions.CastException; import org.apache.doris.nereids.trees.expressions.Expression; @@ -36,10 +38,39 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.function.Consumer; import java.util.function.Function; class DateTimeLiteralTest { + @Test + void testMysqlTimestampKeepsMicrosecondRange() { + boolean previousEnableDateConversion = Config.enable_date_conversion; + try { + Config.enable_date_conversion = true; + ByteBuffer data = ByteBuffer.allocate(12).order(ByteOrder.LITTLE_ENDIAN); + data.put((byte) 11) + .putChar((char) 3022) + .put((byte) 1) + .put((byte) 2) + .put((byte) 3) + .put((byte) 4) + .put((byte) 5) + .putInt(123456) + .flip(); + + Literal literal = Literal.getLiteralByMysqlType( + MysqlColType.MYSQL_TYPE_DATETIME, false, data); + + DateTimeV2Literal dateTime = Assertions.assertInstanceOf(DateTimeV2Literal.class, literal); + Assertions.assertEquals(DateTimeV2Type.MAX, dateTime.getDataType()); + Assertions.assertEquals("3022-01-02 03:04:05.123456", dateTime.getStringValue()); + } finally { + Config.enable_date_conversion = previousEnableDateConversion; + } + } + @Test void reject() { // Assertions.assertThrows(IllegalArgumentException.class, () -> { @@ -402,28 +433,34 @@ void testIrregularDateTimeHourMinuteSecondMicrosecond() { check("2016-07-02 01:01:01.123456", DateTimeV2Literal::new); check("2016-7-02 01:01:01.123456", DateTimeV2Literal::new); - // Testing with microsecond of length 7 - DateTimeV2Literal literal = check("2016-07-02 01:01:01.12345678", DateTimeV2Literal::new); - Assertions.assertEquals(123457, literal.microSecond); - - literal = check("2016-07-02 01:01:01.44444444", DateTimeV2Literal::new); - Assertions.assertEquals(444444, literal.microSecond); - - literal = check("2016-07-02 01:01:01.44444445", DateTimeV2Literal::new); - Assertions.assertEquals(444444, literal.microSecond); - - literal = check("2016-07-02 01:01:01.4444445", DateTimeV2Literal::new); - Assertions.assertEquals(444445, literal.microSecond); - - literal = check("2016-07-02 01:01:01.9999995", DateTimeV2Literal::new); - Assertions.assertEquals(0, literal.microSecond); - Assertions.assertEquals(2, literal.second); - - literal = check("2021-01-01 23:59:59.9999995", DateTimeV2Literal::new); - Assertions.assertEquals(0, literal.microSecond); - Assertions.assertEquals(0, literal.second); - Assertions.assertEquals(0, literal.minute); - Assertions.assertEquals(0, literal.hour); + // Fractional digits above microsecond precision are preserved by TIMESTAMP_NS literals. + TimeStampNsLiteral literal = check("2016-07-02 01:01:01.12345678", TimeStampNsLiteral::new); + Assertions.assertEquals(123456, literal.getMicroSecond()); + Assertions.assertEquals(123456780, literal.getNanoSecond()); + + literal = check("2016-07-02 01:01:01.44444444", TimeStampNsLiteral::new); + Assertions.assertEquals(444444, literal.getMicroSecond()); + Assertions.assertEquals(444444440, literal.getNanoSecond()); + + literal = check("2016-07-02 01:01:01.44444445", TimeStampNsLiteral::new); + Assertions.assertEquals(444444, literal.getMicroSecond()); + Assertions.assertEquals(444444450, literal.getNanoSecond()); + + literal = check("2016-07-02 01:01:01.4444445", TimeStampNsLiteral::new); + Assertions.assertEquals(444444, literal.getMicroSecond()); + Assertions.assertEquals(444444500, literal.getNanoSecond()); + + literal = check("2016-07-02 01:01:01.9999995", TimeStampNsLiteral::new); + Assertions.assertEquals(999999, literal.getMicroSecond()); + Assertions.assertEquals(999999500, literal.getNanoSecond()); + Assertions.assertEquals(1, literal.getSecond()); + + literal = check("2021-01-01 23:59:59.9999995", TimeStampNsLiteral::new); + Assertions.assertEquals(999999, literal.getMicroSecond()); + Assertions.assertEquals(999999500, literal.getNanoSecond()); + Assertions.assertEquals(59, literal.getSecond()); + Assertions.assertEquals(59, literal.getMinute()); + Assertions.assertEquals(23, literal.getHour()); } @Test @@ -458,6 +495,25 @@ void testDateTimeV2Scale() { check("2017-01-01 00:00:00.00000", s -> new DateTimeV2Literal(DateTimeV2Type.of(5), s))); } + @Test + void testTimeStampNsGuardDigitRounding() { + TimeStampNsLiteral literal = new TimeStampNsLiteral("1970-01-01 00:00:00.1234567894"); + Assertions.assertEquals("1970-01-01 00:00:00.123456789", literal.getStringValue()); + + literal = new TimeStampNsLiteral("1970-01-01 00:00:00.1234567895"); + Assertions.assertEquals("1970-01-01 00:00:00.123456790", literal.getStringValue()); + + literal = new TimeStampNsLiteral("1970-01-01 00:00:00.9999999995"); + Assertions.assertEquals("1970-01-01 00:00:01.000000000", literal.getStringValue()); + + literal = new TimeStampNsLiteral("1970-01-01 00:00:00.1234567845"); + Assertions.assertEquals("1970-01-01 00:00:00.123456785", literal.getStringValue()); + + DateTimeLiteral parsed = DateTimeLiteral.parseDateTimeLiteral( + "1970-01-01 00:00:00.9999999995", true).get(); + Assertions.assertEquals("1970-01-01 00:00:01.000000", parsed.getStringValue()); + } + @Test void testRoundFloor() { DateTimeV2Literal literal; @@ -625,7 +681,8 @@ void testDateTimeV1UncheckedCastTo() { @Test void testDateTimeV2UncheckedCastTo() { - DateTimeV2Literal v2 = new DateTimeV2Literal(DateTimeV2Type.MAX, 2025, 7, 23, 13, 25, 59, 999999); + DateTimeV2Literal v2 = new DateTimeV2Literal( + DateTimeV2Type.of(6), 2025, 7, 23, 13, 25, 59, 999999); Expression expression = v2.uncheckedCastTo(BigIntType.INSTANCE); Assertions.assertInstanceOf(BigIntLiteral.class, expression); @@ -689,9 +746,17 @@ void testDateTimeV2UncheckedCastTo() { Assertions.assertInstanceOf(StringLiteral.class, expression); Assertions.assertEquals("2025-07-23 13:25:59", ((StringLiteral) expression).value); - v2 = new DateTimeV2Literal(DateTimeV2Type.MAX, 2025, 7, 23, 13, 25, 59, 0); + v2 = new DateTimeV2Literal(DateTimeV2Type.of(6), 2025, 7, 23, 13, 25, 59, 0); expression = v2.uncheckedCastTo(StringType.INSTANCE); Assertions.assertInstanceOf(StringLiteral.class, expression); Assertions.assertEquals("2025-07-23 13:25:59.000000", ((StringLiteral) expression).value); } + + @Test + void testTimestampNsEpochNanoBoundaries() { + Assertions.assertEquals("1677-09-21 00:12:43.145224192", + TimeStampNsLiteral.getMinValue().getStringValue()); + Assertions.assertEquals("2262-04-11 23:47:16.854775807", + TimeStampNsLiteral.getMaxValue().getStringValue()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/IntegerLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/IntegerLiteralTest.java index 0a9a34f669418b..a1af7a9647b9bc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/IntegerLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/IntegerLiteralTest.java @@ -187,6 +187,10 @@ void testUncheckedCastTo() { Assertions.assertEquals(0, ((DateTimeV2Literal) expression).second); Assertions.assertEquals(0, ((DateTimeV2Literal) expression).microSecond); + IntegerLiteral invalidDate = new IntegerLiteral(1000); + Assertions.assertThrows(CastException.class, + () -> invalidDate.uncheckedCastTo(DateTimeV2Type.SYSTEM_DEFAULT)); + // to string d1 = new IntegerLiteral(701231); expression = d1.uncheckedCastTo(StringType.INSTANCE); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/OutFileTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/OutFileTest.java index 6a10110cf90a92..e6a13b9db4e91a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/OutFileTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/OutFileTest.java @@ -45,7 +45,8 @@ protected void runBeforeAll() throws Exception { createTables( "CREATE TABLE IF NOT EXISTS T1 (\n" + " id bigint,\n" - + " score bigint\n" + + " score bigint,\n" + + " ts timestamp_ns\n" + ")\n" + "DUPLICATE KEY(id)\n" + "DISTRIBUTED BY HASH(id) BUCKETS 1\n" @@ -78,6 +79,15 @@ public void testWriteOutFile() throws Exception { .contains("FILE SINK")); } + @Test + public void testTimestampNsOrcOutfileIsRejected() { + String sql = "select ts from T1 into outfile 'file://~/timestamp_ns.orc' format as orc " + + "properties ('broker.name' = 'my_broker')"; + Exception exception = Assertions.assertThrows(Exception.class, () -> getOutputFragment(sql)); + Assertions.assertTrue(exception.getMessage().contains( + "currently orc do not support column type: TIMESTAMP_NS"), exception.getMessage()); + } + private PlanFragment getOutputFragment(String sql) throws Exception { StatementScopeIdGenerator.clear(); StatementContext statementContext = MemoTestUtils.createStatementContext(connectContext, sql); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/types/DataTypeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/DataTypeTest.java index e8e1eff4e4f170..8af1d320585cc2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/types/DataTypeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/DataTypeTest.java @@ -74,6 +74,32 @@ void testFromPrimitiveType() { Assertions.assertEquals(DataType.fromCatalogType(Type.STRING), StringType.INSTANCE); } + @Test + void testTimeStampNsValidation() { + Assertions.assertDoesNotThrow(TimeStampNsType.INSTANCE::validateDataType); + } + + @Test + void testTimeV2PromotionKeepsMicrosecondPrecision() { + Assertions.assertEquals( + ImmutableList.of(DateTimeV2Type.MAX, StringType.INSTANCE), + TimeV2Type.of(6).getAllPromotions()); + } + + @Test + void testImplicitDateTimeV2PrecisionKeepsMicrosecondCompatibility() { + Assertions.assertEquals(DateTimeV2Type.MAX, + DateTimeV2Type.forType(StringType.INSTANCE)); + Assertions.assertEquals(DateTimeV2Type.MAX, + DateTimeV2Type.forType(DecimalV3Type.createDecimalV3Type(18, 9))); + Assertions.assertEquals(DateTimeV2Type.MAX, + DateTimeV2Type.forTypeFromString("1st Jun 2007 09:45:30")); + Assertions.assertEquals(DateTimeV2Type.MAX, + DateTimeV2Type.forType(DateTimeV2Type.MAX)); + Assertions.assertEquals(DateTimeV2Type.MAX, + DateTimeV2Type.forTypeFromString("2024-01-01 00:00:00.123456789")); + } + @Test void testConvertFromString() { // boolean @@ -121,6 +147,16 @@ void testConvertFromString() { // datetimev2 Assertions.assertEquals(DateTimeV2Type.of(3), DataType.convertFromString("datetimev2(3)")); + Assertions.assertSame(TimeStampNsType.INSTANCE, DataType.convertFromString("timestamp_ns")); + for (String datetimeType : ImmutableList.of("datetime", "datetimev2")) { + for (int invalidScale = 7; invalidScale <= 9; invalidScale++) { + int scale = invalidScale; + Assertions.assertThrows(AnalysisException.class, + () -> DataType.convertFromString(datetimeType + "(" + scale + ")")); + } + } + Assertions.assertThrows(AnalysisException.class, + () -> DataType.convertFromString("timestamp_ns(9)")); // hll Assertions.assertEquals(HllType.INSTANCE, DataType.convertFromString("hll")); // bitmap diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionMatrixTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionMatrixTest.java index e654a2794bd237..3ae70c57fee20b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionMatrixTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionMatrixTest.java @@ -45,6 +45,7 @@ import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeV2Type; import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.types.VarcharType; @@ -695,6 +696,11 @@ public void testProcessComparisonPredicateForDateTimeV2Type() { testProcessComparisonPredicate(DateTimeV2Type.of(4), CharType.createCharType(5), DateTimeV2Type.of(6)); testProcessComparisonPredicate(DateTimeV2Type.of(4), VarcharType.createVarcharType(5), DateTimeV2Type.of(6)); testProcessComparisonPredicate(DateTimeV2Type.of(4), StringType.INSTANCE, DateTimeV2Type.of(6)); + testProcessComparisonPredicate(TimeStampNsType.INSTANCE, DecimalV2Type.SYSTEM_DEFAULT, null); + testProcessComparisonPredicate(TimeStampNsType.INSTANCE, DateV2Type.INSTANCE, null); + testProcessComparisonPredicate(TimeStampNsType.INSTANCE, DateTimeV2Type.MAX, null); + testProcessComparisonPredicate(TimeStampNsType.INSTANCE, TimeV2Type.MAX, null); + testProcessComparisonPredicate(TimeStampNsType.INSTANCE, StringType.INSTANCE, TimeStampNsType.INSTANCE); testProcessComparisonPredicate(DateTimeV2Type.of(4), ArrayType.of(StringType.INSTANCE), null); testProcessComparisonPredicate(DateTimeV2Type.of(4), MapType.of(StringType.INSTANCE, StringType.INSTANCE), null); testProcessComparisonPredicate(DateTimeV2Type.of(4), new StructType(ImmutableList.of(new StructField("c1", StringType.INSTANCE, true, ""))), null); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java index 7255e6530b2747..05f4879bad5164 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.util; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer; import org.apache.doris.nereids.rules.expression.check.CheckCast; import org.apache.doris.nereids.trees.expressions.Add; import org.apache.doris.nereids.trees.expressions.Cast; @@ -27,6 +28,7 @@ import org.apache.doris.nereids.trees.expressions.GreaterThan; import org.apache.doris.nereids.trees.expressions.InPredicate; import org.apache.doris.nereids.trees.expressions.Multiply; +import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.NullSafeEqual; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.Subtract; @@ -73,11 +75,13 @@ import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.types.TimeV2Type; import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.types.VarcharType; import org.apache.doris.nereids.types.VariantType; +import org.apache.doris.nereids.types.coercion.AnyDataType; import org.apache.doris.nereids.types.coercion.IntegralType; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.GlobalVariable; @@ -87,8 +91,107 @@ import org.junit.jupiter.api.Test; import java.math.BigDecimal; +import java.util.Optional; public class TypeCoercionUtilsTest { + @Test + public void testReplaceDateTimeV2WithMaxKeepsTimestampNsIndependent() { + Assertions.assertEquals(DateTimeV2Type.MAX, + TypeCoercionUtils.replaceDateTimeV2WithMax(DateTimeV2Type.of(3))); + Assertions.assertSame(TimeStampNsType.INSTANCE, + TypeCoercionUtils.replaceDateTimeV2WithMax(TimeStampNsType.INSTANCE)); + Assertions.assertEquals(ArrayType.of(TimeStampNsType.INSTANCE), + TypeCoercionUtils.replaceDateTimeV2WithMax(ArrayType.of(TimeStampNsType.INSTANCE))); + } + + @Test + public void testVariantWiderTypeKeepsTimestampNsPrecision() { + Assertions.assertEquals(Optional.of(TimeStampNsType.INSTANCE), + TypeCoercionUtils.findWiderTypeForTwo( + VariantType.INSTANCE, TimeStampNsType.INSTANCE, false, false)); + Assertions.assertEquals(Optional.of(ArrayType.of(TimeStampNsType.INSTANCE)), + TypeCoercionUtils.findWiderTypeForTwo( + VariantType.INSTANCE, ArrayType.of(TimeStampNsType.INSTANCE), false, false)); + + StructType mixedDateLikeType = new StructType(ImmutableList.of( + new StructField("ts", TimeStampNsType.INSTANCE, false, ""), + new StructField("date", DateV2Type.INSTANCE, true, ""))); + StructType normalizedMixedDateLikeType = new StructType(ImmutableList.of( + new StructField("ts", TimeStampNsType.INSTANCE, false, ""), + new StructField("date", DateTimeV2Type.MAX, true, ""))); + Assertions.assertEquals(Optional.of(normalizedMixedDateLikeType), + TypeCoercionUtils.findWiderTypeForTwo( + VariantType.INSTANCE, mixedDateLikeType, false, false)); + } + + @Test + public void testTimestampNsIsNotInDateTimeV2PrecisionFamily() { + Assertions.assertTrue(TypeCoercionUtils.hasTimeStampNsType(TimeStampNsType.INSTANCE)); + Assertions.assertFalse(TypeCoercionUtils.hasDateTimeV2Type(TimeStampNsType.INSTANCE)); + Assertions.assertFalse(TypeCoercionUtils.hasDateTimeV2Type( + ArrayType.of(TimeStampNsType.INSTANCE))); + Assertions.assertTrue(TypeCoercionUtils.hasDateTimeV2Type(DateTimeV2Type.MAX)); + + Assertions.assertThrows(IllegalArgumentException.class, + () -> DateTimeV2Type.getWiderDatetimeV2Type( + DateTimeV2Type.MAX, TimeStampNsType.INSTANCE)); + } + + @Test + public void testTimestampNsOnlyCoercesWithTimestampNsAndStrings() { + Assertions.assertEquals(Optional.of(TimeStampNsType.INSTANCE), + TypeCoercionUtils.findWiderTypeForTwo( + TimeStampNsType.INSTANCE, StringType.INSTANCE, false, false)); + Assertions.assertEquals(Optional.of(StringType.INSTANCE), + TypeCoercionUtils.findWiderTypeForTwo( + TimeStampNsType.INSTANCE, StringType.INSTANCE, false, true)); + Assertions.assertEquals(Optional.empty(), TypeCoercionUtils.findWiderTypeForTwo( + TimeStampNsType.INSTANCE, DateTimeV2Type.MAX, false, false)); + Assertions.assertEquals(Optional.empty(), TypeCoercionUtils.findWiderTypeForTwo( + DateV2Type.INSTANCE, TimeStampNsType.INSTANCE, false, false)); + Assertions.assertEquals(Optional.empty(), TypeCoercionUtils.findWiderTypeForTwo( + TimeStampNsType.INSTANCE, DecimalV2Type.SYSTEM_DEFAULT, false, false)); + + Assertions.assertEquals(Optional.of(TimeStampNsType.INSTANCE), + TypeCoercionUtils.implicitCast(StringType.INSTANCE, TimeStampNsType.INSTANCE)); + Assertions.assertEquals(Optional.of(StringType.INSTANCE), + TypeCoercionUtils.implicitCast(TimeStampNsType.INSTANCE, StringType.INSTANCE)); + Assertions.assertEquals(Optional.of(TimeStampNsType.INSTANCE), + TypeCoercionUtils.implicitCast(NullType.INSTANCE, TimeStampNsType.INSTANCE)); + Assertions.assertEquals(Optional.of(TimeStampNsType.INSTANCE), + TypeCoercionUtils.implicitCast( + TimeStampNsType.INSTANCE, AnyDataType.INSTANCE_WITHOUT_INDEX)); + Assertions.assertEquals(Optional.empty(), + TypeCoercionUtils.implicitCast(TimeStampNsType.INSTANCE, DateTimeV2Type.MAX)); + Assertions.assertEquals(Optional.empty(), + TypeCoercionUtils.implicitCast(DateTimeV2Type.MAX, TimeStampNsType.INSTANCE)); + } + + @Test + public void testTimestampNsComparisonCoercionInBothModes() { + boolean oldBehavior = GlobalVariable.enableNewTypeCoercionBehavior; + try { + for (boolean newBehavior : ImmutableList.of(false, true)) { + GlobalVariable.enableNewTypeCoercionBehavior = newBehavior; + + EqualTo stringComparison = (EqualTo) TypeCoercionUtils.processComparisonPredicate( + new EqualTo(new SlotReference("ts", TimeStampNsType.INSTANCE), + new SlotReference("s", StringType.INSTANCE))); + Assertions.assertEquals(TimeStampNsType.INSTANCE, + stringComparison.left().getDataType()); + Assertions.assertEquals(TimeStampNsType.INSTANCE, + stringComparison.right().getDataType()); + + Assertions.assertThrows(AnalysisException.class, + () -> TypeCoercionUtils.processComparisonPredicate( + new EqualTo(new SlotReference("ts", TimeStampNsType.INSTANCE), + new SlotReference("dt", DateTimeV2Type.MAX)))); + } + } finally { + GlobalVariable.enableNewTypeCoercionBehavior = oldBehavior; + } + } + @Test public void testImplicitCastAccept() { IntegerType integerType = IntegerType.INSTANCE; @@ -488,6 +591,32 @@ public void testProcessInStringCoercion() { Assertions.assertEquals(DecimalV3Type.createDecimalV3Type(23, 3), smallIntString.getOptions().get(0).getDataType()); } + @Test + public void testDateStringSubMicrosecondComparisonCoercion() { + Expression date = new SlotReference("date", DateV2Type.INSTANCE, true); + Expression nanoString = new StringLiteral("2024-01-01 00:00:00.000000001"); + + // Preserve the nanosecond literal instead of silently truncating it to DATETIMEV2(6). + // DATEV2/TIMESTAMP_NS casts are intentionally unsupported until cross-type casts land. + Assertions.assertThrows(AnalysisException.class, + () -> TypeCoercionUtils.processComparisonPredicate(new EqualTo(date, nanoString))); + Assertions.assertThrows(AnalysisException.class, + () -> TypeCoercionUtils.processComparisonPredicate(new EqualTo(nanoString, date))); + } + + @Test + public void testDateStringSubMicrosecondInCoercion() { + Expression date = new SlotReference("date", DateV2Type.INSTANCE, true); + Expression nanoString = new StringLiteral("2024-01-01 00:00:00.000000001"); + Expression dateString = new StringLiteral("2024-01-02"); + + Assertions.assertThrows(AnalysisException.class, () -> TypeCoercionUtils.processInPredicate( + new InPredicate(date, ImmutableList.of(nanoString, dateString)))); + Assertions.assertThrows(AnalysisException.class, + () -> ExpressionAnalyzer.FUNCTION_ANALYZER_RULE.rewrite( + new Not(new InPredicate(date, ImmutableList.of(dateString, nanoString))), null)); + } + @Test public void testCharacterLiteralTypeCoercion() { // datev2 diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java index 607cea3b40a302..67999830275f31 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java @@ -17,9 +17,14 @@ package org.apache.doris.qe; +import org.apache.doris.analysis.Expr; +import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.InternalSchemaInitializer; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; import org.apache.doris.mysql.MysqlChannel; @@ -45,6 +50,7 @@ import java.lang.reflect.Field; import java.lang.reflect.Method; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; @@ -279,6 +285,21 @@ public Void answer(InvocationOnMock invocation) { executor.sendTextResultRow(resultSet); } + @Test + public void testArrowFlightSqlRejectsTimestampNsOutput() { + Expr timestampNs = Mockito.mock(Expr.class); + Mockito.when(timestampNs.getType()).thenReturn(Type.TIMESTAMP_NS); + AnalysisException scalarException = Assertions.assertThrows(AnalysisException.class, + () -> StmtExecutor.checkArrowFlightSqlOutput(Lists.newArrayList(timestampNs))); + Assertions.assertTrue(scalarException.getMessage().contains( + "TIMESTAMP_NS is not supported by Arrow Flight SQL"), scalarException.getMessage()); + + Expr nestedTimestampNs = Mockito.mock(Expr.class); + Mockito.when(nestedTimestampNs.getType()).thenReturn(ArrayType.create(Type.TIMESTAMP_NS, true)); + Assertions.assertThrows(AnalysisException.class, + () -> StmtExecutor.checkArrowFlightSqlOutput(Lists.newArrayList(nestedTimestampNs))); + } + @Test public void testSendBinaryResultRow() throws IOException { ConnectContext mockCtx = Mockito.mock(ConnectContext.class); @@ -327,6 +348,37 @@ public Void answer(InvocationOnMock invocation) { executor.sendBinaryResultRow(resultSet); } + @Test + public void testSendBinaryTimestampNsResultRow() throws IOException { + ConnectContext mockCtx = Mockito.mock(ConnectContext.class); + MysqlChannel channel = Mockito.mock(MysqlChannel.class); + Mockito.when(mockCtx.getConnectType()).thenReturn(ConnectType.MYSQL); + Mockito.when(mockCtx.getMysqlChannel()).thenReturn(channel); + MysqlSerializer mysqlSerializer = MysqlSerializer.newInstance(); + Mockito.when(channel.getSerializer()).thenReturn(mysqlSerializer); + Mockito.when(mockCtx.getSessionVariable()).thenReturn(VariableMgr.newSessionVariable()); + + String value = "2025-01-01 01:02:03.123456789"; + List> rows = Lists.newArrayList(); + rows.add(Lists.newArrayList(value)); + ResultSet resultSet = new CommonResultSet( + new CommonResultSetMetaData(Lists.newArrayList( + new Column("timestamp_ns", ScalarType.createTimeStampNsType()))), + rows); + Mockito.doAnswer(invocation -> { + byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8); + byte[] expected = new byte[valueBytes.length + 3]; + expected[2] = (byte) valueBytes.length; + System.arraycopy(valueBytes, 0, expected, 3, valueBytes.length); + ByteBuffer buffer = invocation.getArgument(0); + Assertions.assertArrayEquals(expected, buffer.array()); + return null; + }).when(channel).sendOnePacket(Mockito.any(ByteBuffer.class)); + + StmtExecutor executor = new StmtExecutor(mockCtx, new OriginStatement("", 1), false); + executor.sendBinaryResultRow(resultSet); + } + @Test public void testSendBinaryBooleanResultRow() throws IOException { ConnectContext mockCtx = Mockito.mock(ConnectContext.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelperArrowTypeTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelperArrowTypeTest.java index 147a52aa55ed98..78db26687affdc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelperArrowTypeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelperArrowTypeTest.java @@ -89,6 +89,13 @@ public void dateV2IsDescribedAsDate32() { buildField(desc("d", TPrimitiveType.DATEV2)).getType()); } + @Test + public void timestampNsIsRejectedUntilArrowOutputIsImplemented() { + IllegalArgumentException exception = Assertions.assertThrows(IllegalArgumentException.class, + () -> buildField(desc("ts", TPrimitiveType.TIMESTAMP_NS))); + Assertions.assertEquals("TIMESTAMP_NS is not supported by Arrow Flight SQL", exception.getMessage()); + } + /** * An Arrow list carries its element type in its child and nowhere else, so the placeholder child this * replaced ({@code ZeroVector}'s Null type) described every array in the catalog as an array OF diff --git a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 446685892295e4..d2ba7a1203b786 100644 --- a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -573,6 +573,7 @@ THEN: 'THEN'; TIME: 'TIME'; TIMESTAMP: 'TIMESTAMP'; TIMESTAMPTZ: 'TIMESTAMPTZ'; +TIMESTAMP_NS: 'TIMESTAMP_NS'; TINYINT: 'TINYINT'; TO: 'TO'; TOKENIZER: 'TOKENIZER'; diff --git a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 9dfd4630ed8890..961a107585291e 100644 --- a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -1900,6 +1900,7 @@ primitiveColType | type=DATEV1 | type=DATETIMEV1 | type=TIMESTAMPTZ + | type=TIMESTAMP_NS | type=BITMAP | type=QUANTILE_STATE | type=HLL @@ -2364,6 +2365,7 @@ nonReserved | TIME | TIMESTAMP | TIMESTAMPTZ + | TIMESTAMP_NS | TRANSACTION | TREE | TRIGGERS diff --git a/fe/fe-type/src/main/java/org/apache/doris/catalog/PrimitiveType.java b/fe/fe-type/src/main/java/org/apache/doris/catalog/PrimitiveType.java index 216db92e21698a..36b640831772f1 100644 --- a/fe/fe-type/src/main/java/org/apache/doris/catalog/PrimitiveType.java +++ b/fe/fe-type/src/main/java/org/apache/doris/catalog/PrimitiveType.java @@ -66,6 +66,7 @@ public enum PrimitiveType { AGG_STATE("AGG_STATE", 16, TPrimitiveType.AGG_STATE, true), DATEV2("DATEV2", 4, TPrimitiveType.DATEV2, true), DATETIMEV2("DATETIMEV2", 8, TPrimitiveType.DATETIMEV2, true), + TIMESTAMP_NS("TIMESTAMP_NS", 8, TPrimitiveType.TIMESTAMP_NS, true), TIMEV2("TIMEV2", 8, TPrimitiveType.TIMEV2, false), TIMESTAMPTZ("TIMESTAMPTZ", 8, TPrimitiveType.TIMESTAMPTZ, false), LAMBDA_FUNCTION("LAMBDA_FUNCTION", 16, TPrimitiveType.LAMBDA_FUNCTION, false), @@ -135,6 +136,7 @@ public enum PrimitiveType { supportedTypes.add(DATETIME); supportedTypes.add(DATEV2); supportedTypes.add(DATETIMEV2); + supportedTypes.add(TIMESTAMP_NS); supportedTypes.add(TIMEV2); supportedTypes.add(TIMESTAMPTZ); supportedTypes.add(IPV4); @@ -213,6 +215,8 @@ public static PrimitiveType fromThrift(TPrimitiveType tPrimitiveType) { return DATEV2; case DATETIMEV2: return DATETIMEV2; + case TIMESTAMP_NS: + return TIMESTAMP_NS; case IPV4: return IPV4; case IPV6: @@ -328,11 +332,12 @@ public boolean isNull() { } public boolean isDateLikeType() { - return (this == DATE || this == DATETIME || this == DATEV2 || this == DATETIMEV2 || this == TIMESTAMPTZ); + return this == DATE || this == DATETIME || this == DATEV2 || this == DATETIMEV2 + || this == TIMESTAMP_NS || this == TIMESTAMPTZ; } public boolean isDateV2LikeType() { - return (this == DATEV2 || this == DATETIMEV2 || this == TIMESTAMPTZ); + return this == DATEV2 || this == DATETIMEV2 || this == TIMESTAMP_NS || this == TIMESTAMPTZ; } public boolean isTimeStampTzType() { @@ -428,6 +433,7 @@ public MysqlColType toMysqlType() { case DATEV2: return MysqlColType.MYSQL_TYPE_DATE; case TIMESTAMPTZ: + case TIMESTAMP_NS: return MysqlColType.MYSQL_TYPE_STRING; case DATETIME: case DATETIMEV2: { @@ -463,6 +469,7 @@ public int getOlapColumnIndexSize() { return DATEV2_INDEX_LEN; case DATETIME: case DATETIMEV2: + case TIMESTAMP_NS: case TIMESTAMPTZ: return DATETIME_INDEX_LEN; case VARCHAR: diff --git a/fe/fe-type/src/main/java/org/apache/doris/catalog/ScalarType.java b/fe/fe-type/src/main/java/org/apache/doris/catalog/ScalarType.java index 219a7c2384fc6c..b39bfbbd69f14c 100644 --- a/fe/fe-type/src/main/java/org/apache/doris/catalog/ScalarType.java +++ b/fe/fe-type/src/main/java/org/apache/doris/catalog/ScalarType.java @@ -81,6 +81,7 @@ public class ScalarType extends Type { public static final int MAX_DECIMAL256_PRECISION = 76; public static final int DEFAULT_MIN_AVG_DECIMAL128_SCALE = 4; public static final int MAX_DATETIMEV2_SCALE = 6; + public static final int TIMESTAMP_NS_SCALE = 9; public static final int MAX_PRECISION = MAX_DECIMAL256_PRECISION; private long byteSize = -1; @@ -146,6 +147,8 @@ public static ScalarType createType(PrimitiveType type, int len, int precision, return createDecimalType(precision, scale); case DATETIMEV2: return createDatetimeV2Type(scale); + case TIMESTAMP_NS: + return createTimeStampNsType(); case TIMEV2: return createTimeV2Type(scale); case TIMESTAMPTZ: @@ -205,6 +208,8 @@ public static ScalarType createType(PrimitiveType type) { return DATEV2; case DATETIMEV2: return DEFAULT_DATETIMEV2; + case TIMESTAMP_NS: + return Type.TIMESTAMP_NS; case TIMEV2: return TIMEV2; case TIMESTAMPTZ: @@ -281,6 +286,8 @@ public static ScalarType createType(String type) { return DATEV2; case "DATETIMEV2": return DATETIMEV2; + case "TIMESTAMP_NS": + return TIMESTAMP_NS; case "TIME": case "TIMEV2": return TIMEV2; @@ -451,6 +458,11 @@ public static ScalarType createDatetimeV2Type(int scale) { return type; } + @SuppressWarnings("checkstyle:MissingJavadocMethod") + public static ScalarType createTimeStampNsType() { + return new ScalarType(PrimitiveType.TIMESTAMP_NS); + } + @SuppressWarnings("checkstyle:MissingJavadocMethod") public static ScalarType createTimeV2Type(int scale) { ScalarType type = new ScalarType(PrimitiveType.TIMEV2); @@ -605,13 +617,15 @@ public String toString() { return "decimal(*,*)"; } return "decimal(" + precision + "," + scale + ")"; - } else if (type.isDecimalV3Type()) { + } else if (type.isDecimalV3Type()) { if (isWildcardDecimal()) { return "decimalv3(*,*)"; } return "decimalv3(" + precision + "," + scale + ")"; - } else if (type == PrimitiveType.DATETIMEV2) { + } else if (type == PrimitiveType.DATETIMEV2) { return "datetimev2(" + scale + ")"; + } else if (type == PrimitiveType.TIMESTAMP_NS) { + return "timestamp_ns"; } else if (type == PrimitiveType.TIMEV2) { return "timev2(" + scale + ")"; } else if (type == PrimitiveType.TIMESTAMPTZ) { @@ -680,6 +694,9 @@ public String toSql(int depth) { case DATETIMEV2: stringBuilder.append("datetimev2").append("(").append(scale).append(")"); break; + case TIMESTAMP_NS: + stringBuilder.append("timestamp_ns"); + break; case TIMEV2: stringBuilder.append("time").append("(").append(scale).append(")"); break; diff --git a/fe/fe-type/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-type/src/main/java/org/apache/doris/catalog/Type.java index 3c8632e1a7be1d..6f9383ee0d6b31 100644 --- a/fe/fe-type/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-type/src/main/java/org/apache/doris/catalog/Type.java @@ -71,6 +71,7 @@ public abstract class Type { public static final ScalarType DATEV2 = new ScalarType(PrimitiveType.DATEV2); public static final ScalarType TIMEV2 = new ScalarType(PrimitiveType.TIMEV2); public static final ScalarType TIMESTAMPTZ = new ScalarType(PrimitiveType.TIMESTAMPTZ); + public static final ScalarType TIMESTAMP_NS = ScalarType.createTimeStampNsType(); public static final ScalarType STRING = ScalarType.createStringType(); public static final ScalarType VARBINARY = ScalarType.createVarbinaryType(-1); public static final ScalarType DEFAULT_DECIMALV2 = ScalarType.createDecimalType(PrimitiveType.DECIMALV2, @@ -97,7 +98,8 @@ public abstract class Type { public static final ScalarType DEFAULT_DECIMALV3 = DEFAULT_DECIMAL32; public static final ScalarType DEFAULT_DATETIMEV2 = ScalarType.createDatetimeV2Type(0); public static final ScalarType DATETIMEV2 = DEFAULT_DATETIMEV2; - public static final ScalarType DATETIMEV2_WITH_MAX_SCALAR = ScalarType.createDatetimeV2Type(6); + public static final ScalarType DATETIMEV2_WITH_MAX_SCALAR + = ScalarType.createDatetimeV2Type(ScalarType.MAX_DATETIMEV2_SCALE); public static final ScalarType DEFAULT_TIMESTAMP_TZ = ScalarType.createTimeStampTzType(0); public static final ScalarType TIMESTAMP_TZ = DEFAULT_TIMESTAMP_TZ; public static final ScalarType TIMESTAMP_TZ_WITH_MAX_SCALAR = ScalarType.createTimeStampTzType(6); @@ -165,6 +167,7 @@ public abstract class Type { numericDateTimeTypes.add(DATETIME); numericDateTimeTypes.add(DATEV2); numericDateTimeTypes.add(DATETIMEV2); + numericDateTimeTypes.add(TIMESTAMP_NS); numericDateTimeTypes.add(TIMEV2); numericDateTimeTypes.addAll(numericTypes); @@ -178,6 +181,7 @@ public abstract class Type { trivialTypes.add(DATETIME); trivialTypes.add(DATEV2); trivialTypes.add(DATETIMEV2); + trivialTypes.add(TIMESTAMP_NS); trivialTypes.add(IPV4); trivialTypes.add(IPV6); trivialTypes.add(TIMEV2); @@ -202,6 +206,7 @@ public abstract class Type { arraySubTypes.add(DATETIME); arraySubTypes.add(DATEV2); arraySubTypes.add(DATETIMEV2); + arraySubTypes.add(TIMESTAMP_NS); arraySubTypes.add(TIMESTAMP_TZ); arraySubTypes.add(IPV4); arraySubTypes.add(IPV6); @@ -232,6 +237,7 @@ public abstract class Type { mapSubTypes.add(DATETIME); mapSubTypes.add(DATEV2); mapSubTypes.add(DATETIMEV2); + mapSubTypes.add(TIMESTAMP_NS); mapSubTypes.add(TIMESTAMP_TZ); mapSubTypes.add(IPV4); mapSubTypes.add(IPV6); @@ -258,6 +264,7 @@ public abstract class Type { structSubTypes.add(DATETIME); structSubTypes.add(DATEV2); structSubTypes.add(DATETIMEV2); + structSubTypes.add(TIMESTAMP_NS); structSubTypes.add(TIMESTAMP_TZ); structSubTypes.add(IPV4); structSubTypes.add(IPV6); @@ -470,6 +477,8 @@ public String hideVersionForVersionColumn( typeStr.append("(").append(((ScalarType) this).getScalarScale()).append(")"); } return typeStr.toString(); + } else if (isTimeStampNs()) { + return "timestamp_ns"; } else if (isTimeStampTz()) { StringBuilder typeStr = new StringBuilder("timestamptz"); if (((ScalarType) this).getScalarScale() > 0) { @@ -538,6 +547,10 @@ public boolean isDatetimeV2() { return isScalarType(PrimitiveType.DATETIMEV2); } + public boolean isTimeStampNs() { + return isScalarType(PrimitiveType.TIMESTAMP_NS); + } + public boolean isTimeV2() { return isScalarType(PrimitiveType.TIMEV2); } @@ -687,6 +700,7 @@ public boolean isNativeType() { public boolean isDateType() { return isScalarType(PrimitiveType.DATE) || isScalarType(PrimitiveType.DATETIME) || isScalarType(PrimitiveType.DATEV2) || isScalarType(PrimitiveType.DATETIMEV2) + || isScalarType(PrimitiveType.TIMESTAMP_NS) || isScalarType(PrimitiveType.TIMESTAMPTZ); } @@ -870,6 +884,8 @@ public static Type fromPrimitiveType(PrimitiveType type) { return Type.DATEV2; case DATETIMEV2: return Type.DATETIMEV2; + case TIMESTAMP_NS: + return Type.TIMESTAMP_NS; case TIMEV2: return Type.TIMEV2; case TIMESTAMPTZ: @@ -978,6 +994,8 @@ protected static Pair fromThrift(TTypeDesc col, int nodeIdx) { Preconditions.checkState(scalarType.isSetPrecision() && scalarType.isSetScale()); type = ScalarType.createDatetimeV2Type(scalarType.getScale()); + } else if (scalarType.getType() == TPrimitiveType.TIMESTAMP_NS) { + type = ScalarType.createTimeStampNsType(); } else if (scalarType.getType() == TPrimitiveType.TIMEV2) { Preconditions.checkState(scalarType.isSetPrecision() && scalarType.isSetScale()); @@ -1119,6 +1137,8 @@ public Integer getPrecision() { case TIMESTAMPTZ: case TIMEV2: return t.decimalPrecision(); + case TIMESTAMP_NS: + return 29; default: return null; } @@ -1155,6 +1175,8 @@ public Integer getDecimalDigits() { case DECIMAL128: case DECIMAL256: return t.decimalScale(); + case TIMESTAMP_NS: + return 9; default: return null; } diff --git a/gensrc/proto/types.proto b/gensrc/proto/types.proto index 1cb49602e60a76..51acb7cb8ca324 100644 --- a/gensrc/proto/types.proto +++ b/gensrc/proto/types.proto @@ -122,6 +122,7 @@ message PGenericType { IPV6 = 40; VARBINARY = 41; TIMESTAMPTZ = 42; + TIMESTAMP_NS = 43; UNKNOWN = 999; } required TypeId id = 2; diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index f911d17346f1f9..daef6845d5c871 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -102,7 +102,8 @@ enum TPrimitiveType { UINT64 = 41, // only used in BE to represent offsets FIXED_LENGTH_OBJECT = 42 // only used in BE to represent fixed-length object VARBINARY = 43, // represent varbinary type - TIMESTAMPTZ = 44 // timestamp with time zone + TIMESTAMPTZ = 44, // timestamp with time zone + TIMESTAMP_NS = 45 // signed nanoseconds since the Unix epoch } enum TTypeNodeType { diff --git a/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns.out b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns.out new file mode 100644 index 00000000000000..24fe3a3935b62b --- /dev/null +++ b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns.out @@ -0,0 +1,99 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !storage -- +1 1677-09-21 00:12:43.145224200 1677-09-21 00:12:43.145224200 1677-09-21 00:12:43.145224192 +2 1969-12-31 23:59:59.999999900 1969-12-31 23:59:59.999999990 1969-12-31 23:59:59.999999999 +3 1970-01-01 00:00:00.000000000 1970-01-01 00:00:00.000000000 1970-01-01 00:00:00.000000000 +4 1970-01-01 00:00:00.000000100 1970-01-01 00:00:00.000000010 1970-01-01 00:00:00.000000001 +5 2262-04-11 23:47:16.854775800 2262-04-11 23:47:16.854775800 2262-04-11 23:47:16.854775807 + +-- !row_store -- +1 1677-09-21 00:12:43.145224192 +2 1970-01-01 00:00:00.123456789 +3 2262-04-11 23:47:16.854775807 + +-- !rounding -- +1677-09-21 00:12:43.145224192 1970-01-01 00:00:00.123456750 1970-01-01 00:00:00.999999995 1970-01-01 00:00:00.123456789 2262-04-11 23:47:16.854775807 + +-- !cast_and_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 5 + +-- !filter -- +1 1677-09-21 00:12:43.145224192 +3 1970-01-01 00:00:00.000000000 +5 2262-04-11 23:47:16.854775807 + +-- !out_of_range -- +1677-09-21 00:12:43.145224192 \N 1970-01-01 00:00:00.000000000 \N 2262-04-11 23:47:16.854775807 + +-- !all_comparisons -- +true true true true true true true true true true + +-- !sort_limit -- +1 1677-09-21 00:12:43.145224192 +2 1969-12-31 23:59:59.999999999 +3 1970-01-01 00:00:00.000000000 +6 1970-01-01 00:00:00.000000000 +4 1970-01-01 00:00:00.000000001 +5 2262-04-11 23:47:16.854775807 +7 \N + +-- !group_by -- +\N 1 7 7 +1677-09-21 00:12:43.145224192 1 1 1 +1969-12-31 23:59:59.999999999 1 2 2 +1970-01-01 00:00:00.000000000 2 3 6 +1970-01-01 00:00:00.000000001 1 4 4 +2262-04-11 23:47:16.854775807 1 5 5 + +-- !hash_join -- +1 1 1677-09-21 00:12:43.145224192 +2 2 1969-12-31 23:59:59.999999999 +3 3 1970-01-01 00:00:00.000000000 +3 6 1970-01-01 00:00:00.000000000 +4 4 1970-01-01 00:00:00.000000001 +5 5 2262-04-11 23:47:16.854775807 +6 3 1970-01-01 00:00:00.000000000 +6 6 1970-01-01 00:00:00.000000000 + +-- !relational_aggregates -- +6 5 1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 5 + +-- !unique_key -- +1677-09-21 00:12:43.145224192 1 +1969-12-31 23:59:59.999999999 2 +1970-01-01 00:00:00.000000000 20 +1970-01-01 00:00:00.000000001 4 +2262-04-11 23:47:16.854775807 5 + +-- !aggregate_key -- +1677-09-21 00:12:43.145224192 1 +1969-12-31 23:59:59.999999999 2 +1970-01-01 00:00:00.000000000 7 +1970-01-01 00:00:00.000000001 5 +2262-04-11 23:47:16.854775807 6 + +-- !range_partition -- +p_after_epoch [('2262-04-11 23:47:16.854775807'), (MAXVALUE)) +p_before_epoch [('1677-09-21 00:12:43.145224193'), ('1970-01-01 00:00:00.000000000')) +p_before_maximum [('1970-01-01 00:00:00.000000002'), ('2262-04-11 23:47:16.854775807')) +p_epoch [('1970-01-01 00:00:00.000000000'), ('1970-01-01 00:00:00.000000002')) +p_minimum [('1677-09-21 00:12:43.145224192'), ('1677-09-21 00:12:43.145224193')) + +-- !array_functions -- +1 ["1677-09-21 00:12:43.145224192", "1970-01-01 00:00:00.000000000", "2262-04-11 23:47:16.854775807"] +2 \N + +-- !index_predicates -- +1 1677-09-21 00:12:43.145224192 +2 1970-01-01 00:00:00.000000000 +3 2262-04-11 23:47:16.854775807 + +-- !cast_with_constant_folding -- +1677-09-21 00:12:43.145224192 1970-01-01 00:00:00.123456789 1970-01-01 00:00:00.123456789 1970-01-01 00:00:00.123456789 1970-01-01 00:00:00.123456789 1970-01-01 00:00:00.123456790 1970-01-01 00:00:01.000000000 2262-04-11 23:47:16.854775807 + +-- !cast_without_constant_folding -- +1677-09-21 00:12:43.145224192 1970-01-01 00:00:00.123456789 1970-01-01 00:00:00.123456789 1970-01-01 00:00:00.123456789 1970-01-01 00:00:00.123456789 1970-01-01 00:00:00.123456790 1970-01-01 00:00:01.000000000 2262-04-11 23:47:16.854775807 + +-- !nonconstant_permissive_cast -- +1 2024-01-01 00:00:00.123456789 +2 \N diff --git a/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_binary_output.out b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_binary_output.out new file mode 100644 index 00000000000000..6a2d4e04a126cf --- /dev/null +++ b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_binary_output.out @@ -0,0 +1,15 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !text_protocol -- +1 1677-09-21 00:12:43.145224192 +2 1969-12-31 23:59:59.999999999 +3 1970-01-01 00:00:00.000000000 +4 1970-01-01 00:00:00.000000001 +5 2262-04-11 23:47:16.854775807 +6 \N +-- !binary_protocol -- +1 1677-09-21 00:12:43.145224192 +2 1969-12-31 23:59:59.999999999 +3 1970-01-01 00:00:00.000000000 +4 1970-01-01 00:00:00.000000001 +5 2262-04-11 23:47:16.854775807 +6 \N diff --git a/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_ddl_predicate.out b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_ddl_predicate.out new file mode 100644 index 00000000000000..7be5664f5f7143 --- /dev/null +++ b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_ddl_predicate.out @@ -0,0 +1,60 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !timestamp_ns_alias_values -- +1 1970-01-01 00:00:00.123456789 1970-01-01 00:00:00.123456789 1970-01-01 00:00:00.123456789 1970-01-01 00:00:00.123456789 true true true true true + +-- !timestamp_ns_alias_schema -- +1 id int YES +2 ts timestamp_ns YES +3 dt7 timestamp_ns YES +4 dt8 timestamp_ns YES +5 dt9 timestamp_ns YES + +-- !datetime_alias_scales -- +\N \N \N 6 +1677-09-21 00:12:43.145224200 1677-09-21 00:12:43.145224200 1677-09-21 00:12:43.145224192 1 +1969-12-31 23:59:59.999999900 1969-12-31 23:59:59.999999990 1969-12-31 23:59:59.999999999 2 +1970-01-01 00:00:00.000000000 1970-01-01 00:00:00.000000000 1970-01-01 00:00:00.000000000 3 +1970-01-01 00:00:00.123456800 1970-01-01 00:00:00.123456790 1970-01-01 00:00:00.123456789 4 +2262-04-11 23:47:16.854775800 2262-04-11 23:47:16.854775800 2262-04-11 23:47:16.854775807 5 + +-- !aggregate_key_and_values -- +1677-09-21 00:12:43.145224192 1677-09-21 00:12:43.145224192 1677-09-21 00:12:43.145224200 +1970-01-01 00:00:00.000000000 2262-04-11 23:47:16.854775807 1969-12-31 23:59:59.999999900 +2262-04-11 23:47:16.854775807 2262-04-11 23:47:16.854775807 2262-04-11 23:47:16.854775800 + +-- !unique_mow_primary_semantics -- +1677-09-21 00:12:43.145224192 1677-09-21 00:12:43.145224200 1 +1970-01-01 00:00:00.000000000 1970-01-01 00:00:00.000000010 20 +2262-04-11 23:47:16.854775807 2262-04-11 23:47:16.854775800 3 + +-- !invalid_and_overflow_values_become_null -- +100 \N \N \N +101 \N \N \N +102 \N \N \N + +-- !list_partition_scale_rounding -- +1970-01-01 00:00:00.123456750 1970-01-01 00:00:00.123456785 1970-01-01 00:00:00.123456790 1 +1970-01-01 00:00:00.999999950 1970-01-01 00:00:00.999999995 1970-01-01 00:00:01.000000000 2 + +-- !nullable_first_range -- +7 \N + +-- !all_storage_predicates -- +1 false true false false true true false true true false +2 false true false false true true false true false true +3 true false false true false true false true true false +4 false true true true false false false true false true +5 false true true true false false false true false true +6 false true true true false false false true true false +7 \N \N \N \N \N \N true false \N \N + +-- !topn -- +4 1970-01-01 00:00:00.000000001 +5 1970-01-01 00:00:00.123456789 +6 2262-04-11 23:47:16.854775807 + +-- !json_round_trip -- +1677-09-21 00:12:43.145224192 1970-01-01 00:00:00.000000000 2262-04-11 23:47:16.854775807 + +-- !aggregate_state -- +1 2262-04-11 23:47:16.854775807 diff --git a/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_partial_default.csv b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_partial_default.csv new file mode 100644 index 00000000000000..f599e28b8ab0d8 --- /dev/null +++ b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_partial_default.csv @@ -0,0 +1 @@ +10 diff --git a/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_stream_load.csv b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_stream_load.csv new file mode 100644 index 00000000000000..3664e0573775f9 --- /dev/null +++ b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_stream_load.csv @@ -0,0 +1,9 @@ +1|1677-09-21 00:12:43.145224192 +2|1969-12-31 23:59:59.999999999 +3|1970-01-01 00:00:00.000000000 +4|1970-01-01 00:00:00.000000001 +5|2262-04-11 23:47:16.854775807 +6|2262-04-11 23:47:16.854775808 +7|1970-02-30 00:00:00.000000000 +8|2023-08-17T01:41:18.123456789Z +9|2023-08-17T01:41:18.123456789America/Los_Angeles diff --git a/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_stream_load.out b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_stream_load.out new file mode 100644 index 00000000000000..10d48d32611768 --- /dev/null +++ b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_stream_load.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !stream_load -- +1 1677-09-21 00:12:43.145224192 +2 1969-12-31 23:59:59.999999999 +3 1970-01-01 00:00:00.000000000 +4 1970-01-01 00:00:00.000000001 +5 2262-04-11 23:47:16.854775807 +8 2023-08-17 09:41:18.123456789 +9 2023-08-17 16:41:18.123456789 diff --git a/regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns.groovy b/regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns.groovy new file mode 100644 index 00000000000000..cb79df4e57e31d --- /dev/null +++ b/regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns.groovy @@ -0,0 +1,416 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_timestamp_ns") { + sql "drop table if exists test_timestamp_ns" + // dt7/dt8/dt9 describe the inserted fractional widths; every column is TIMESTAMP_NS. + sql """ + create table test_timestamp_ns ( + id int, + dt7 timestamp_ns, + dt8 timestamp_ns, + dt9 timestamp_ns + ) + duplicate key(id) + distributed by hash(id) buckets 1 + properties("replication_num" = "1") + """ + + sql """ + insert into test_timestamp_ns values + (1, '1677-09-21 00:12:43.1452242', + '1677-09-21 00:12:43.14522420', + '1677-09-21 00:12:43.145224192'), + (2, '1969-12-31 23:59:59.9999999', + '1969-12-31 23:59:59.99999999', + '1969-12-31 23:59:59.999999999'), + (3, '1970-01-01 00:00:00.0000000', + '1970-01-01 00:00:00.00000000', + '1970-01-01 00:00:00.000000000'), + (4, '1970-01-01 00:00:00.0000001', + '1970-01-01 00:00:00.00000001', + '1970-01-01 00:00:00.000000001'), + (5, '2262-04-11 23:47:16.8547758', + '2262-04-11 23:47:16.85477580', + '2262-04-11 23:47:16.854775807') + """ + + order_qt_storage """ + select id, dt7, dt8, dt9 + from test_timestamp_ns + order by dt9 + """ + + sql "drop table if exists test_timestamp_ns_row_store" + sql """ + create table test_timestamp_ns_row_store ( + id int, + dt timestamp_ns + ) + duplicate key(id) + distributed by hash(id) buckets 1 + properties( + "replication_num" = "1", + "store_row_column" = "true" + ) + """ + sql """ + insert into test_timestamp_ns_row_store values + (1, '1677-09-21 00:12:43.145224192'), + (2, '1970-01-01 00:00:00.123456789'), + (3, '2262-04-11 23:47:16.854775807') + """ + order_qt_row_store """ + select id, dt + from test_timestamp_ns_row_store + order by id + """ + + qt_rounding """ + select + cast('1677-09-21 00:12:43.145224192' as timestamp_ns), + cast('1970-01-01 00:00:00.12345675' as timestamp_ns), + cast('1970-01-01 00:00:00.999999995' as timestamp_ns), + cast('1970-01-01 00:00:00.123456789' as timestamp_ns), + cast('2262-04-11 23:47:16.854775807' as timestamp_ns) + """ + + qt_cast_and_aggregate """ + select + cast(cast('1677-09-21 00:12:43.145224192' as timestamp_ns) as string), + cast(cast('2262-04-11 23:47:16.854775807' as timestamp_ns) as string), + min(dt9), max(dt9), count(distinct dt9) + from test_timestamp_ns + """ + + // Cross-family casts and datetime scalar functions are implemented in follow-up changes. + test { + sql "select cast(dt9 as datetimev2(6)) from test_timestamp_ns" + exception "cannot cast" + } + test { + sql "select seconds_add(dt9, 1) from test_timestamp_ns" + exception "Can not find the compatibility function signature: seconds_add" + } + + order_qt_filter """ + select id, dt9 + from test_timestamp_ns + where dt9 in ( + cast('1677-09-21 00:12:43.145224192' as timestamp_ns), + cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + cast('2262-04-11 23:47:16.854775807' as timestamp_ns)) + order by id + """ + + qt_out_of_range """ + select + cast('1677-09-21 00:12:43.145224192' as timestamp_ns), + cast('1677-09-21 00:12:43.145224191' as timestamp_ns), + cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + cast('2262-04-11 23:47:16.854775808' as timestamp_ns), + cast('2262-04-11 23:47:16.854775807' as timestamp_ns) + """ + + qt_all_comparisons """ + select + cast('1677-09-21 00:12:43.145224192' as timestamp_ns) + = cast('1677-09-21 00:12:43.145224192' as timestamp_ns), + cast('1677-09-21 00:12:43.145224192' as timestamp_ns) + < cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + cast('1969-12-31 23:59:59.999999999' as timestamp_ns) + = cast('1969-12-31 23:59:59.999999999' as timestamp_ns), + cast('1969-12-31 23:59:59.999999999' as timestamp_ns) + <> cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + cast('1969-12-31 23:59:59.999999999' as timestamp_ns) + < cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + cast('1970-01-01 00:00:00.000000001' as timestamp_ns) + > cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + cast('1970-01-01 00:00:00.000000001' as timestamp_ns) + >= cast('1970-01-01 00:00:00.000000001' as timestamp_ns), + cast('1970-01-01 00:00:00.000000001' as timestamp_ns) + <= cast('1970-01-01 00:00:00.000000001' as timestamp_ns), + cast('1970-01-01 00:00:00.000000000' as timestamp_ns) + < cast('2262-04-11 23:47:16.854775807' as timestamp_ns), + cast('2262-04-11 23:47:16.854775807' as timestamp_ns) + = cast('2262-04-11 23:47:16.854775807' as timestamp_ns) + """ + + sql "drop table if exists test_timestamp_ns_relational" + sql """ + create table test_timestamp_ns_relational ( + id int, + dt timestamp_ns, + payload varchar(16) + ) + duplicate key(id) + distributed by hash(id) buckets 1 + properties("replication_num" = "1") + """ + sql """ + insert into test_timestamp_ns_relational values + (1, '1677-09-21 00:12:43.145224192', 'minimum'), + (2, '1969-12-31 23:59:59.999999999', 'before'), + (3, '1970-01-01 00:00:00.000000000', 'epoch-a'), + (4, '1970-01-01 00:00:00.000000001', 'after'), + (5, '2262-04-11 23:47:16.854775807', 'maximum'), + (6, '1970-01-01 00:00:00.000000000', 'epoch-b'), + (7, null, 'null') + """ + + qt_sort_limit """ + select id, dt + from test_timestamp_ns_relational + order by dt asc nulls last, id + limit 7 + """ + + order_qt_group_by """ + select dt, count(*), min(id), max(id) + from test_timestamp_ns_relational + group by dt + order by dt nulls first + """ + + order_qt_hash_join """ + select l.id, r.id, l.dt + from test_timestamp_ns_relational l + join test_timestamp_ns_relational r on l.dt = r.dt + order by l.id, r.id + """ + + qt_relational_aggregates """ + select + count(dt), + count(distinct dt), + min(dt), + max(dt), + approx_count_distinct(dt) + from test_timestamp_ns_relational + """ + + sql "drop table if exists test_timestamp_ns_unique" + sql """ + create table test_timestamp_ns_unique ( + dt timestamp_ns, + value int + ) + unique key(dt) + distributed by hash(dt) buckets 1 + properties( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ) + """ + sql """ + insert into test_timestamp_ns_unique values + ('1677-09-21 00:12:43.145224192', 1), + ('1969-12-31 23:59:59.999999999', 2), + ('1970-01-01 00:00:00.000000000', 3), + ('1970-01-01 00:00:00.000000001', 4), + ('2262-04-11 23:47:16.854775807', 5) + """ + sql """ + insert into test_timestamp_ns_unique values + ('1970-01-01 00:00:00.000000000', 20) + """ + order_qt_unique_key """ + select dt, value + from test_timestamp_ns_unique + order by dt + """ + + sql "drop table if exists test_timestamp_ns_aggregate" + sql """ + create table test_timestamp_ns_aggregate ( + dt timestamp_ns, + amount bigint sum + ) + aggregate key(dt) + distributed by hash(dt) buckets 1 + properties("replication_num" = "1") + """ + sql """ + insert into test_timestamp_ns_aggregate values + ('1677-09-21 00:12:43.145224192', 1), + ('1969-12-31 23:59:59.999999999', 2), + ('1970-01-01 00:00:00.000000000', 3), + ('1970-01-01 00:00:00.000000000', 4), + ('1970-01-01 00:00:00.000000001', 5), + ('2262-04-11 23:47:16.854775807', 6) + """ + order_qt_aggregate_key """ + select dt, amount + from test_timestamp_ns_aggregate + order by dt + """ + + sql "drop table if exists test_timestamp_ns_partition" + sql """ + create table test_timestamp_ns_partition ( + dt timestamp_ns, + value int + ) + duplicate key(dt) + partition by range(dt) ( + partition p_minimum values less than ('1677-09-21 00:12:43.145224193'), + partition p_before_epoch values less than ('1970-01-01 00:00:00.000000000'), + partition p_epoch values less than ('1970-01-01 00:00:00.000000002'), + partition p_before_maximum values less than ('2262-04-11 23:47:16.854775807'), + partition p_after_epoch values less than MAXVALUE + ) + distributed by hash(dt) buckets 1 + properties("replication_num" = "1") + """ + order_qt_range_partition """ + select partition_name, partition_description + from information_schema.partitions + where table_schema = database() + and table_name = 'test_timestamp_ns_partition' + order by partition_name + """ + sql "drop table if exists test_timestamp_ns_complex" + sql """ + create table test_timestamp_ns_complex ( + id int, + values_array array + ) + duplicate key(id) + distributed by hash(id) buckets 1 + properties("replication_num" = "1") + """ + sql """ + insert into test_timestamp_ns_complex values + (1, array( + cast('2262-04-11 23:47:16.854775807' as timestamp_ns), + cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + cast('1677-09-21 00:12:43.145224192' as timestamp_ns) + )), + (2, null) + """ + order_qt_array_functions """ + select id, + array_sort(values_array) + from test_timestamp_ns_complex + order by id + """ + + sql "drop table if exists test_timestamp_ns_index" + sql """ + create table test_timestamp_ns_index ( + id int, + dt timestamp_ns, + index idx_dt(dt) using inverted + ) + duplicate key(id) + distributed by hash(id) buckets 1 + properties( + "replication_num" = "1", + "bloom_filter_columns" = "dt" + ) + """ + sql """ + insert into test_timestamp_ns_index values + (1, '1677-09-21 00:12:43.145224192'), + (2, '1970-01-01 00:00:00.000000000'), + (3, '2262-04-11 23:47:16.854775807') + """ + order_qt_index_predicates """ + select id, dt + from test_timestamp_ns_index + where dt in ( + cast('1677-09-21 00:12:43.145224192' as timestamp_ns), + cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + cast('2262-04-11 23:47:16.854775807' as timestamp_ns)) + order by id + """ + + sql "set debug_skip_fold_constant = false" + qt_cast_with_constant_folding """ + select + cast('1677-09-21 00:12:43.145224192' as timestamp_ns), + cast('1970-01-01 00:00:00.123456789' as timestamp_ns), + cast('1970-01-01 00:00:00.123456789' as timestamp_ns), + cast('1970-01-01 00:00:00.123456789' as timestamp_ns), + cast('1970-01-01 00:00:00.1234567894' as timestamp_ns), + cast('1970-01-01 00:00:00.1234567895' as timestamp_ns), + cast('1970-01-01 00:00:00.9999999995' as timestamp_ns), + cast('2262-04-11 23:47:16.854775807' as timestamp_ns) + """ + sql "set debug_skip_fold_constant = true" + qt_cast_without_constant_folding """ + select + cast('1677-09-21 00:12:43.145224192' as timestamp_ns), + cast('1970-01-01 00:00:00.123456789' as timestamp_ns), + cast('1970-01-01 00:00:00.123456789' as timestamp_ns), + cast('1970-01-01 00:00:00.123456789' as timestamp_ns), + cast('1970-01-01 00:00:00.1234567894' as timestamp_ns), + cast('1970-01-01 00:00:00.1234567895' as timestamp_ns), + cast('1970-01-01 00:00:00.9999999995' as timestamp_ns), + cast('2262-04-11 23:47:16.854775807' as timestamp_ns) + """ + sql "set debug_skip_fold_constant = false" + + sql "drop table if exists test_timestamp_ns_cast_input" + sql """ + create table test_timestamp_ns_cast_input ( + id int, + value varchar(64) + ) + duplicate key(id) + distributed by hash(id) buckets 1 + properties("replication_num" = "1") + """ + sql """ + insert into test_timestamp_ns_cast_input values + (1, '2024-01-01 00:00:00.123456789'), + (2, '2024-01-01 00:00:00.123.456') + """ + def originalEnableStrictCast = sql("select @@enable_strict_cast")[0][0] + try { + sql "set enable_strict_cast = false" + order_qt_nonconstant_permissive_cast """ + select id, cast(value as timestamp_ns) + from test_timestamp_ns_cast_input + order by id + """ + + sql "set enable_strict_cast = true" + test { + sql """ + select cast(value as timestamp_ns) + from test_timestamp_ns_cast_input + where id = 2 + """ + exception "2024-01-01 00:00:00.123.456" + } + } finally { + sql "set enable_strict_cast = ${originalEnableStrictCast}" + } + + test { + sql """ + create table test_timestamp_ns_invalid_scale ( + id int, + dt datetimev2(10) + ) + distributed by hash(id) buckets 1 + properties("replication_num" = "1") + """ + exception "between 0 and 6" + } +} diff --git a/regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns_binary_output.groovy b/regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns_binary_output.groovy new file mode 100644 index 00000000000000..406c90d93524dc --- /dev/null +++ b/regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns_binary_output.groovy @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import com.mysql.cj.jdbc.ServerPreparedStatement + +import java.sql.PreparedStatement +import java.sql.Types + +suite("test_timestamp_ns_binary_output") { + def user = context.config.jdbcUser + def password = context.config.jdbcPassword + + sql "drop table if exists test_timestamp_ns_binary_output" + sql """ + create table test_timestamp_ns_binary_output ( + id int, + dt timestamp_ns + ) + duplicate key(id) + distributed by hash(id) buckets 1 + properties("replication_num" = "1") + """ + sql """ + insert into test_timestamp_ns_binary_output values + (1, '1677-09-21 00:12:43.145224192'), + (2, '1969-12-31 23:59:59.999999999'), + (3, '1970-01-01 00:00:00.000000000'), + (4, '1970-01-01 00:00:00.000000001'), + (5, '2262-04-11 23:47:16.854775807'), + (6, null) + """ + + order_qt_text_protocol """ + select id, dt from test_timestamp_ns_binary_output order by id + """ + + String url = getServerPrepareJdbcUrl( + context.config.jdbcUrl, "regression_test_datatype_p0_datetimev2") + connect(user, password, url) { + PreparedStatement stmt = prepareStatement(""" + select id, dt from test_timestamp_ns_binary_output where id >= ? order by id + """) + assertEquals(ServerPreparedStatement, stmt.class) + assertEquals(Types.CHAR, stmt.metaData.getColumnType(2)) + stmt.setInt(1, 1) + qe_binary_protocol stmt + stmt.close() + } +} diff --git a/regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns_ddl_predicate.groovy b/regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns_ddl_predicate.groovy new file mode 100644 index 00000000000000..8c5ab6a73041b0 --- /dev/null +++ b/regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns_ddl_predicate.groovy @@ -0,0 +1,399 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_timestamp_ns_ddl_predicate") { + for (def datetimeType : ["datetime", "datetimev2"]) { + for (def invalidScale : [7, 8, 9]) { + test { + sql "select cast('1970-01-01 00:00:00.123456789' as ${datetimeType}(${invalidScale}))" + exception "between 0 and 6" + } + } + } + + sql "drop table if exists test_datetime_scale_above_six" + test { + sql """ + create table test_datetime_scale_above_six ( + id int, + dt datetimev2(7) + ) + distributed by hash(id) buckets 1 + properties("replication_num" = "1") + """ + exception "between 0 and 6" + } + + sql "drop table if exists test_timestamp_ns_values" + // The legacy golden tags and dt7/dt8/dt9 names refer to input widths, not DATETIMEV2 scales. + sql """ + create table test_timestamp_ns_values ( + id int, + ts timestamp_ns, + dt7 timestamp_ns, + dt8 timestamp_ns, + dt9 timestamp_ns + ) + duplicate key(id) + distributed by hash(id) buckets 1 + properties("replication_num" = "1") + """ + sql """ + insert into test_timestamp_ns_values values + (1, + '1970-01-01 00:00:00.123456789', + '1970-01-01 00:00:00.123456789', + '1970-01-01 00:00:00.123456789', + '1970-01-01 00:00:00.123456789') + """ + order_qt_timestamp_ns_alias_values """ + select id, ts, dt7, dt8, dt9, + ts = dt7, dt7 = dt8, dt8 = dt9, + cast(dt7 as timestamp_ns) = dt7, + cast(dt8 as timestamp_ns) = dt8 + from test_timestamp_ns_values + order by id + """ + order_qt_timestamp_ns_alias_schema """ + select ordinal_position, column_name, data_type, is_nullable + from information_schema.columns + where table_schema = database() + and table_name = 'test_timestamp_ns_values' + order by ordinal_position + """ + + test { + sql "select cast('1970-01-01 00:00:00.000000001' as timestamp_ns(9))" + exception "timestamp_ns does not support precision" + } + + sql "drop table if exists test_timestamp_ns_ddl_duplicate" + sql """ + create table test_timestamp_ns_ddl_duplicate ( + dt7 timestamp_ns, + dt8 timestamp_ns, + dt9 timestamp_ns, + id int + ) + duplicate key(dt7, dt8) + distributed by hash(dt9) buckets 4 + properties("replication_num" = "1") + """ + sql """ + insert into test_timestamp_ns_ddl_duplicate values + ('1677-09-21 00:12:43.1452242', + '1677-09-21 00:12:43.14522420', + '1677-09-21 00:12:43.145224192', 1), + ('1969-12-31 23:59:59.9999999', + '1969-12-31 23:59:59.99999999', + '1969-12-31 23:59:59.999999999', 2), + ('1970-01-01 00:00:00.0000000', + '1970-01-01 00:00:00.00000000', + '1970-01-01 00:00:00.000000000', 3), + ('1970-01-01 00:00:00.1234568', + '1970-01-01 00:00:00.12345679', + '1970-01-01 00:00:00.123456789', 4), + ('2262-04-11 23:47:16.8547758', + '2262-04-11 23:47:16.85477580', + '2262-04-11 23:47:16.854775807', 5), + (null, null, null, 6) + """ + order_qt_datetime_alias_scales """ + select dt7, dt8, dt9, id + from test_timestamp_ns_ddl_duplicate + order by id + """ + sql "drop table if exists test_timestamp_ns_ddl_aggregate" + sql """ + create table test_timestamp_ns_ddl_aggregate ( + dt timestamp_ns, + max_dt timestamp_ns max, + min_dt timestamp_ns min + ) + aggregate key(dt) + distributed by hash(dt) buckets 1 + properties("replication_num" = "1") + """ + sql """ + insert into test_timestamp_ns_ddl_aggregate values + ('1677-09-21 00:12:43.145224192', + '1677-09-21 00:12:43.145224192', + '1677-09-21 00:12:43.1452242'), + ('1970-01-01 00:00:00.000000000', + '1970-01-01 00:00:00.000000001', + '1969-12-31 23:59:59.9999999'), + ('1970-01-01 00:00:00.000000000', + '2262-04-11 23:47:16.854775807', + '1970-01-01 00:00:00.0000000'), + ('2262-04-11 23:47:16.854775807', + '2262-04-11 23:47:16.854775807', + '2262-04-11 23:47:16.8547758') + """ + order_qt_aggregate_key_and_values """ + select dt, max_dt, min_dt + from test_timestamp_ns_ddl_aggregate + order by dt + """ + + sql "drop table if exists test_timestamp_ns_ddl_unique" + sql """ + create table test_timestamp_ns_ddl_unique ( + dt timestamp_ns, + value_dt timestamp_ns, + value int + ) + unique key(dt) + distributed by hash(dt) buckets 1 + properties( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ) + """ + sql """ + insert into test_timestamp_ns_ddl_unique values + ('1677-09-21 00:12:43.145224192', '1677-09-21 00:12:43.14522420', 1), + ('1970-01-01 00:00:00.000000000', '1970-01-01 00:00:00.00000000', 2), + ('2262-04-11 23:47:16.854775807', '2262-04-11 23:47:16.85477580', 3) + """ + sql """ + insert into test_timestamp_ns_ddl_unique values + ('1970-01-01 00:00:00.000000000', '1970-01-01 00:00:00.00000001', 20) + """ + order_qt_unique_mow_primary_semantics """ + select dt, value_dt, value + from test_timestamp_ns_ddl_unique + order by dt + """ + + test { + sql """ + create table test_timestamp_ns_scale_overflow ( + id int, + dt datetime(10) + ) + distributed by hash(id) buckets 1 + properties("replication_num" = "1") + """ + exception "between 0 and 6" + } + + sql """ + insert into test_timestamp_ns_ddl_duplicate values + ('1970-02-30 00:00:00.0000000', + '1970-02-30 00:00:00.00000000', + '1970-02-30 00:00:00.000000000', 100) + """ + sql """ + insert into test_timestamp_ns_ddl_duplicate values + ('1677-09-21 00:12:43.1452241', + '1677-09-21 00:12:43.14522419', + '1677-09-21 00:12:43.145224191', 101) + """ + sql """ + insert into test_timestamp_ns_ddl_duplicate values + ('2262-04-11 23:47:16.8547759', + '2262-04-11 23:47:16.85477581', + '2262-04-11 23:47:16.854775808', 102) + """ + order_qt_invalid_and_overflow_values_become_null """ + select id, dt7, dt8, dt9 + from test_timestamp_ns_ddl_duplicate + where id between 100 and 102 + order by id + """ + + sql "drop table if exists test_timestamp_ns_partition_bucket" + sql """ + create table test_timestamp_ns_partition_bucket ( + dt timestamp_ns, + id int + ) + duplicate key(dt) + partition by range(dt) ( + partition p_before_epoch values less than + ('1970-01-01 00:00:00.000000000'), + partition p_epoch values less than + ('1970-01-01 00:00:00.000000002'), + partition p_after_epoch values less than MAXVALUE + ) + distributed by hash(dt) buckets 4 + properties("replication_num" = "1") + """ + sql """ + insert into test_timestamp_ns_partition_bucket values + ('1677-09-21 00:12:43.145224192', 1), + ('1969-12-31 23:59:59.999999999', 2), + ('1970-01-01 00:00:00.000000000', 3), + ('1970-01-01 00:00:00.000000001', 4), + ('1970-01-01 00:00:00.123456789', 5), + ('2262-04-11 23:47:16.854775807', 6), + (null, 7) + """ + explain { + sql """ + select * + from test_timestamp_ns_partition_bucket + where dt = cast('1970-01-01 00:00:00.000000000' as timestamp_ns) + """ + contains "partitions=1/3 (p_epoch)" + contains "tablets=1/4" + } + explain { + sql """ + select * + from test_timestamp_ns_partition_bucket + where dt >= cast('2262-04-11 23:47:16.854775807' as timestamp_ns) + """ + contains "partitions=1/3 (p_after_epoch)" + } + explain { + sql """ + select * + from test_timestamp_ns_partition_bucket + where dt is null + """ + contains "partitions=1/3 (p_before_epoch)" + } + + sql "drop table if exists test_timestamp_ns_list_partition_rounding" + sql """ + create table test_timestamp_ns_list_partition_rounding ( + dt7 timestamp_ns, + dt8 timestamp_ns, + dt9 timestamp_ns, + id int + ) + duplicate key(dt7, dt8, dt9) + partition by list(dt7, dt8, dt9) ( + partition p_round values in + (('1970-01-01 00:00:00.12345675', + '1970-01-01 00:00:00.123456785', + '1970-01-01 00:00:00.1234567895')), + partition p_carry values in + (('1970-01-01 00:00:00.99999995', + '1970-01-01 00:00:00.999999995', + '1970-01-01 00:00:00.9999999995')) + ) + distributed by hash(id) buckets 1 + properties("replication_num" = "1") + """ + sql """ + insert into test_timestamp_ns_list_partition_rounding values + ('1970-01-01 00:00:00.12345675', + '1970-01-01 00:00:00.123456785', + '1970-01-01 00:00:00.1234567895', 1), + ('1970-01-01 00:00:00.99999995', + '1970-01-01 00:00:00.999999995', + '1970-01-01 00:00:00.9999999995', 2) + """ + order_qt_list_partition_scale_rounding """ + select dt7, dt8, dt9, id + from test_timestamp_ns_list_partition_rounding + order by id + """ + + order_qt_nullable_first_range """ + select id, dt + from test_timestamp_ns_partition_bucket + where dt is null + order by id + """ + + order_qt_all_storage_predicates """ + select id, + dt = cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + dt != cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + dt > cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + dt >= cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + dt < cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + dt <= cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + dt is null, + dt is not null, + dt in ( + cast('1677-09-21 00:12:43.145224192' as timestamp_ns), + cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + cast('2262-04-11 23:47:16.854775807' as timestamp_ns)), + dt not in ( + cast('1677-09-21 00:12:43.145224192' as timestamp_ns), + cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + cast('2262-04-11 23:47:16.854775807' as timestamp_ns)) + from test_timestamp_ns_partition_bucket + order by id + """ + + order_qt_topn """ + select id, dt + from test_timestamp_ns_partition_bucket + order by dt desc nulls last + limit 3 + """ + + order_qt_json_round_trip """ + select + cast( + json_extract_string( + json_object( + 'dt', + cast( + cast('1677-09-21 00:12:43.145224192' as timestamp_ns) + as string)), + '\$.dt') + as timestamp_ns), + cast( + json_extract_string( + json_object( + 'dt', + cast( + cast('1970-01-01 00:00:00.000000000' as timestamp_ns) + as string)), + '\$.dt') + as timestamp_ns), + cast( + json_extract_string( + json_object( + 'dt', + cast( + cast('2262-04-11 23:47:16.854775807' as timestamp_ns) + as string)), + '\$.dt') + as timestamp_ns) + """ + + sql "set enable_agg_state = true" + sql "drop table if exists test_timestamp_ns_agg_state" + sql """ + create table test_timestamp_ns_agg_state ( + id int, + dt_state agg_state generic + ) + aggregate key(id) + distributed by hash(id) buckets 1 + properties("replication_num" = "1") + """ + sql """ + insert into test_timestamp_ns_agg_state values + (1, max_state(cast('1677-09-21 00:12:43.145224192' as timestamp_ns))), + (1, max_state(cast('1970-01-01 00:00:00.000000000' as timestamp_ns))), + (1, max_state(cast('2262-04-11 23:47:16.854775807' as timestamp_ns))) + """ + order_qt_aggregate_state """ + select id, max_merge(dt_state) + from test_timestamp_ns_agg_state + group by id + order by id + """ +} diff --git a/regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns_stream_load.groovy b/regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns_stream_load.groovy new file mode 100644 index 00000000000000..ff2786afd585d6 --- /dev/null +++ b/regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns_stream_load.groovy @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_timestamp_ns_stream_load") { + sql "set time_zone = '+08:00'" + sql "drop table if exists test_timestamp_ns_stream_load" + sql """ + create table test_timestamp_ns_stream_load ( + id int, + dt timestamp_ns + ) + duplicate key(id) + distributed by hash(id) buckets 1 + properties("replication_num" = "1") + """ + + streamLoad { + table "test_timestamp_ns_stream_load" + file "test_timestamp_ns_stream_load.csv" + set "column_separator", "|" + set "strict_mode", "true" + set "max_filter_ratio", "0.3" + + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(9, json.NumberTotalRows) + assertEquals(7, json.NumberLoadedRows) + assertEquals(2, json.NumberFilteredRows) + } + } + + order_qt_stream_load """ + select id, dt + from test_timestamp_ns_stream_load + order by id + """ + +}