[feature](timestamp_ns) Add basic TIMESTAMP_NS type support

be/src/core/call_on_type_index.h

@@ -37,6 +37,7 @@ struct TypePair {
 class DataTypeDate;
 class DataTypeDateV2;
 class DataTypeDateTimeV2;
+class DataTypeTimeStampNs;
 class DataTypeDateTime;
 class DataTypeIPv4;
 class DataTypeIPv6;
@@ -83,6 +84,8 @@ bool call_on_index_and_data_type(PrimitiveType number, F&& f) {
         return f(TypePair<DataTypeDateV2, T>());
     case PrimitiveType::TYPE_DATETIMEV2:
         return f(TypePair<DataTypeDateTimeV2, T>());
+    case PrimitiveType::TYPE_TIMESTAMP_NS:
+        return f(TypePair<DataTypeTimeStampNs, T>());
     case PrimitiveType::TYPE_DATETIME:
         return f(TypePair<DataTypeDateTime, T>());
     case PrimitiveType::TYPE_TIMEV2:
@@ -236,6 +239,8 @@ bool dispatch_type_base(PrimitiveType number, F&& f) {
             return f(DispatchDataType<TYPE_DATEV2>());
         case PrimitiveType::TYPE_DATETIMEV2:
             return f(DispatchDataType<TYPE_DATETIMEV2>());
+        case PrimitiveType::TYPE_TIMESTAMP_NS:
+            return f(DispatchDataType<TYPE_TIMESTAMP_NS>());
         case PrimitiveType::TYPE_DATETIME:
             return f(DispatchDataType<TYPE_DATETIME>());
         case PrimitiveType::TYPE_TIMEV2:

be/src/core/column/column_array.cpp

@@ -874,6 +874,9 @@ ColumnArrayDataOffsets filter_return_new_dispatch(const Filter& filt, ssize_t re
         return filter_number_return_new<TYPE_DATETIME>(filt, result_size_hint, data, offsets);
     if (check_and_get_column<ColumnDateTimeV2>(data.get()))
         return filter_number_return_new<TYPE_DATETIMEV2>(filt, result_size_hint, data, offsets);
+    if (check_and_get_column<ColumnTimeStampNs>(data.get())) {
+        return filter_number_return_new<TYPE_TIMESTAMP_NS>(filt, result_size_hint, data, offsets);
+    }
     if (check_and_get_column<ColumnTimeStampTz>(data.get()))
         return filter_number_return_new<TYPE_TIMESTAMPTZ>(filt, result_size_hint, data, offsets);
     if (check_and_get_column<ColumnTimeV2>(data.get()))
@@ -940,6 +943,9 @@ size_t filter_inplace_dispatch(const Filter& filter, IColumn& src_data,
         return filter_number_inplace<TYPE_DATETIME>(filter, src_data, src_offsets);
     if (check_and_get_column<ColumnDateTimeV2>(&src_data))
         return filter_number_inplace<TYPE_DATETIMEV2>(filter, src_data, src_offsets);
+    if (check_and_get_column<ColumnTimeStampNs>(&src_data)) {
+        return filter_number_inplace<TYPE_TIMESTAMP_NS>(filter, src_data, src_offsets);
+    }
     if (check_and_get_column<ColumnTimeStampTz>(&src_data))
         return filter_number_inplace<TYPE_TIMESTAMPTZ>(filter, src_data, src_offsets);
     if (check_and_get_column<ColumnTimeV2>(&src_data))

be/src/core/column/column_vector.cpp

@@ -559,6 +559,7 @@ template class ColumnVector<TYPE_DATE>;
 template class ColumnVector<TYPE_DATEV2>;
 template class ColumnVector<TYPE_DATETIME>;
 template class ColumnVector<TYPE_DATETIMEV2>;
+template class ColumnVector<TYPE_TIMESTAMP_NS>;
 template class ColumnVector<TYPE_TIMEV2>;
 template class ColumnVector<TYPE_TIMESTAMPTZ>;
 template class ColumnVector<TYPE_UINT32>;

be/src/core/column/column_vector.h

@@ -412,7 +412,8 @@ class ColumnVector final : public COWHelper<IColumn, ColumnVector<T>> {
     // when run function which need_replace_null_data_to_default, use the value far from 0 to avoid
     // raise errors for null cell.
     static value_type default_value() {
-        if constexpr (is_date_type(T) || T == PrimitiveType::TYPE_TIMESTAMPTZ) {
+        if constexpr ((is_date_type(T) && T != PrimitiveType::TYPE_TIMESTAMP_NS) ||
+                      T == PrimitiveType::TYPE_TIMESTAMPTZ) {
             return PrimitiveTypeTraits<T>::CppType::DEFAULT_VALUE;
         } else {
             return value_type();

be/src/core/column/columns_common.cpp

@@ -359,6 +359,8 @@ INSTANTIATE(DateV2Value<DateV2ValueType>, IColumn::Offset)
 INSTANTIATE(DateV2Value<DateV2ValueType>, ColumnArray::Offset64)
 INSTANTIATE(DateV2Value<DateTimeV2ValueType>, IColumn::Offset)
 INSTANTIATE(DateV2Value<DateTimeV2ValueType>, ColumnArray::Offset64)
+INSTANTIATE(TimeStampNsValue, IColumn::Offset)
+INSTANTIATE(TimeStampNsValue, ColumnArray::Offset64)
 INSTANTIATE(TimestampTzValue, IColumn::Offset)
 INSTANTIATE(TimestampTzValue, ColumnArray::Offset64)
 INSTANTIATE(UInt32, IColumn::Offset)

be/src/core/data_type/convert_field_to_type.cpp

@@ -58,6 +58,9 @@ void dispatch(F&& f, const Field& field) {
     case PrimitiveType::TYPE_DATETIMEV2:
         f(field.template get<TYPE_DATETIMEV2>());
         return;
+    case PrimitiveType::TYPE_TIMESTAMP_NS:
+        f(field.template get<TYPE_TIMESTAMP_NS>());
+        return;
     case PrimitiveType::TYPE_TIMESTAMPTZ:
         f(field.template get<TYPE_TIMESTAMPTZ>());
         return;
@@ -180,6 +183,9 @@ class FieldVisitorToJsonb : public StaticVisitor<void> {
     void operator()(const DateV2Value<DateTimeV2ValueType>& x, JsonbWriter* writer) const {
         writer->writeInt64(*(UInt64*)&x);
     }
+    void operator()(const TimeStampNsValue& x, JsonbWriter* writer) const {
+        writer->writeInt64(x.epoch_nanos());
+    }
     void operator()(const TimestampTzValue& x, JsonbWriter* writer) const {
         writer->writeInt64(*(UInt64*)&x);
     }
@@ -828,4 +834,4 @@ void convert_field_to_type(const Field& from_value, const IDataType& to_type, Fi
         return convert_field_to_typeImpl(from_value, to_type, from_type_hint, to);
     }
 }
-} // namespace doris
+} // namespace doris

be/src/core/data_type/data_type.cpp

@@ -126,6 +126,8 @@ PGenericType_TypeId IDataType::get_pdata_type(const IDataType* data_type) {
         return PGenericType::VARIANT;
     case PrimitiveType::TYPE_DATETIMEV2:
         return PGenericType::DATETIMEV2;
+    case PrimitiveType::TYPE_TIMESTAMP_NS:
+        return PGenericType::TIMESTAMP_NS;
     case PrimitiveType::TYPE_TIMESTAMPTZ:
         return PGenericType::TIMESTAMPTZ;
     case PrimitiveType::TYPE_BITMAP:

be/src/core/data_type/data_type_array.cpp

@@ -162,7 +162,8 @@ FieldWithDataType DataTypeArray::get_field_with_data_type(const IColumn& column,
     if (is_decimal(nested_type_id)) {
         precision = nested_type->get_precision();
         scale = nested_type->get_scale();
-    } else if (nested_type_id == TYPE_DATETIMEV2 || nested_type_id == TYPE_TIMESTAMPTZ) {
+    } else if (nested_type_id == TYPE_DATETIMEV2 || nested_type_id == TYPE_TIMESTAMP_NS ||
+               nested_type_id == TYPE_TIMESTAMPTZ) {
         scale = nested_type->get_scale();
     } else if (nested_type_id == TYPE_JSONB) {
         // Array<Jsonb> should return JsonbField as element

be/src/core/data_type/data_type_factory.cpp

@@ -59,6 +59,7 @@
 #include "core/data_type/data_type_string.h"
 #include "core/data_type/data_type_struct.h"
 #include "core/data_type/data_type_time.h"
+#include "core/data_type/data_type_timestamp_ns.h"
 #include "core/data_type/data_type_timestamptz.h"
 #include "core/data_type/data_type_varbinary.h"
 #include "core/data_type/data_type_variant.h"
@@ -152,6 +153,9 @@ DataTypePtr DataTypeFactory::_create_primitive_data_type(const FieldType& type,
     case FieldType::OLAP_FIELD_TYPE_DATETIMEV2:
         result = create_datetimev2(scale);
         break;
+    case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS:
+        result = std::make_shared<DataTypeTimeStampNs>();
+        break;
     case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ:
         result = std::make_shared<DataTypeTimeStampTz>(scale);
         break;
@@ -258,6 +262,9 @@ DataTypePtr DataTypeFactory::create_data_type(const PColumnMeta& pcolumn) {
     case PGenericType::DATETIMEV2:
         nested = std::make_shared<DataTypeDateTimeV2>(pcolumn.decimal_param().scale());
         break;
+    case PGenericType::TIMESTAMP_NS:
+        nested = std::make_shared<DataTypeTimeStampNs>();
+        break;
     case PGenericType::DATETIME:
         nested = std::make_shared<DataTypeDateTime>();
         break;
@@ -436,6 +443,9 @@ DataTypePtr DataTypeFactory::create_data_type(const PrimitiveType primitive_type
     case TYPE_DATETIMEV2:
         nested = create_datetimev2(scale);
         break;
+    case TYPE_TIMESTAMP_NS:
+        nested = std::make_shared<DataTypeTimeStampNs>();
+        break;
     case TYPE_DATETIME:
         nested = std::make_shared<DataTypeDateTime>();
         break;

be/src/core/data_type/data_type_number_base.cpp

@@ -203,6 +203,7 @@ template class DataTypeNumberBase<TYPE_DATE>;
 template class DataTypeNumberBase<TYPE_DATEV2>;
 template class DataTypeNumberBase<TYPE_DATETIME>;
 template class DataTypeNumberBase<TYPE_DATETIMEV2>;
+template class DataTypeNumberBase<TYPE_TIMESTAMP_NS>;
 template class DataTypeNumberBase<TYPE_IPV4>;
 template class DataTypeNumberBase<TYPE_IPV6>;
 template class DataTypeNumberBase<TYPE_TIMEV2>;

be/src/core/data_type/data_type_time.h

@@ -26,6 +26,7 @@
 #include <cstddef>
 #include <memory>
 #include <string>
+#include <utility>
 
 #include "core/data_type/data_type.h"
 #include "core/data_type/data_type_number_base.h"
@@ -40,8 +41,10 @@ class IColumn;
 
 class DataTypeTimeV2 final : public DataTypeNumberBase<PrimitiveType::TYPE_TIMEV2> {
 public:
+    static constexpr UInt32 MAX_SCALE = 6;
+
     DataTypeTimeV2(int scale = 0) : _scale(scale) {
-        if (UNLIKELY(scale > 6)) {
+        if (UNLIKELY(std::cmp_greater(scale, MAX_SCALE))) {
             throw doris::Exception(ErrorCode::INTERNAL_ERROR, "Scale {} is out of bounds", scale);
         }
         if (scale == -1) {

be/src/core/data_type/data_type_timestamp_ns.cpp

@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "core/data_type/data_type_timestamp_ns.h"
+
+#include <gen_cpp/data.pb.h>
+
+#include <cstdint>
+#include <typeinfo>
+#include <utility>
+
+#include "common/exception.h"
+#include "core/assert_cast.h"
+#include "core/column/column_vector.h"
+#include "core/string_ref.h"
+#include "core/value/timestamp_ns_value.h"
+
+namespace doris {
+
+Field DataTypeTimeStampNs::get_field(const TExprNode& node) const {
+    int64_t value = 0;
+    const StringRef string_value(node.date_literal.value.data(), node.date_literal.value.size());
+    const auto status = parse_timestamp_ns(string_value, &value);
+    if (!status.ok()) {
+        throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT,
+                               "Invalid value: {} for type TimeStampNs: {}",
+                               node.date_literal.value, status.to_string());
+    }
+    return Field::create_field<TYPE_TIMESTAMP_NS>(TimeStampNsValue(value));
+}
+
+bool DataTypeTimeStampNs::equals(const IDataType& rhs) const {
+    return typeid(rhs) == typeid(*this);
+}
+
+void DataTypeTimeStampNs::to_pb_column_meta(PColumnMeta* col_meta) const {
+    IDataType::to_pb_column_meta(col_meta);
+}
+
+FieldWithDataType DataTypeTimeStampNs::get_field_with_data_type(const IColumn& column,
+                                                                size_t row_num) const {
+    const auto& column_data =
+            assert_cast<const ColumnTimeStampNs&, TypeCheckOnRelease::DISABLE>(column);
+    Field field;
+    column_data.get(row_num, field);
+    return FieldWithDataType {.field = std::move(field),
+                              .base_scalar_type_id = get_primitive_type(),
+                              .precision = -1,
+                              .scale = static_cast<int>(get_scale())};
+}
+
+} // namespace doris

be/src/core/data_type/data_type_timestamp_ns.h

@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include "core/data_type/data_type_number_base.h"
+#include "core/data_type_serde/data_type_timestamp_ns_serde.h"
+
+namespace doris {
+
+class DataTypeTimeStampNs final : public DataTypeNumberBase<PrimitiveType::TYPE_TIMESTAMP_NS> {
+public:
+    static constexpr PrimitiveType PType = TYPE_TIMESTAMP_NS;
+    static constexpr bool is_parametric = false;
+    static constexpr UInt32 SCALE = 9;
+
+    PrimitiveType get_primitive_type() const override { return TYPE_TIMESTAMP_NS; }
+    doris::FieldType get_storage_field_type() const override {
+        return doris::FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS;
+    }
+    const std::string get_family_name() const override { return "TimeStampNs"; }
+    std::string do_get_name() const override { return "TimeStampNs"; }
+
+    bool equals(const IDataType& rhs) const override;
+    bool equals_ignore_precision(const IDataType& rhs) const override {
+        return rhs.get_primitive_type() == TYPE_TIMESTAMP_NS;
+    }
+
+    using SerDeType = DataTypeTimeStampNsSerDe;
+    DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+        return std::make_shared<SerDeType>(nesting_level);
+    }
+
+    Field get_field(const TExprNode& node) const override;
+    UInt32 get_scale() const override { return SCALE; }
+    void to_pb_column_meta(PColumnMeta* col_meta) const override;
+    FieldWithDataType get_field_with_data_type(const IColumn& column,
+                                               size_t row_num) const override;
+};
+
+} // namespace doris

be/src/core/data_type/define_primitive_type.h

@@ -73,7 +73,8 @@ enum PrimitiveType : PrimitiveNative {
     TYPE_UINT64,                         /* 39, used as offset */
     TYPE_FIXED_LENGTH_OBJECT,            /* 40, represent fixed-length object on BE */
     TYPE_VARBINARY,                      /* 41, varbinary */
-    TYPE_TIMESTAMPTZ                     /* 42, timestamptz */
+    TYPE_TIMESTAMPTZ,                    /* 42, timestamptz */
+    TYPE_TIMESTAMP_NS                    /* 43, signed Int64 epoch nanoseconds */
 };
 
 } // namespace doris

be/src/core/data_type/primitive_type.cpp

@@ -69,6 +69,9 @@ PrimitiveType thrift_to_type(TPrimitiveType::type ttype) {
     case TPrimitiveType::DATETIMEV2:
         return TYPE_DATETIMEV2;
 
+    case TPrimitiveType::TIMESTAMP_NS:
+        return TYPE_TIMESTAMP_NS;
+
     case TPrimitiveType::TIMEV2:
         return TYPE_TIMEV2;
 
@@ -191,6 +194,9 @@ TPrimitiveType::type to_thrift(PrimitiveType ptype) {
     case TYPE_DATETIMEV2:
         return TPrimitiveType::DATETIMEV2;
 
+    case TYPE_TIMESTAMP_NS:
+        return TPrimitiveType::TIMESTAMP_NS;
+
     case TYPE_TIMEV2:
         return TPrimitiveType::TIMEV2;
 
@@ -305,6 +311,9 @@ std::string type_to_string(PrimitiveType t) {
     case TYPE_DATETIMEV2:
         return "DATETIMEV2";
 
+    case TYPE_TIMESTAMP_NS:
+        return "TIMESTAMP_NS";
+
     case TYPE_TIMEV2:
         return "TIMEV2";