Import YDB C++ SDK 39

.github/import_generation.txt

@@ -1 +1 @@
-39
+40

.github/last_commit.txt

@@ -1 +1 @@
-bc2335edd99f161c2f65b36772768babcd9e687c
+fd4d02d6b3ebb3565fec58f5a585b44e09e6899e

CHANGELOG.md

@@ -1,3 +1,7 @@
+* Added interface for export of metrics and spans, supported plugin for OpenTelemetry
+
+* Supported gRPC compression option on client side
+
 ## v3.17.0
 
 * Added support of describe for scheme objects with type 'secret' via new TSecretClient

CMakeLists.txt

@@ -10,6 +10,8 @@ project(YDB-CPP-SDK VERSION ${YDB_SDK_VERSION} LANGUAGES C CXX ASM)
 option(YDB_SDK_INSTALL "Install YDB C++ SDK" Off)
 option(YDB_SDK_TESTS "Build YDB C++ SDK tests" Off)
 option(YDB_SDK_EXAMPLES "Build YDB C++ SDK examples" On)
+option(YDB_SDK_ENABLE_OTEL_METRICS "Build OpenTelemetry metrics plugin" Off)
+option(YDB_SDK_ENABLE_OTEL_TRACE "Build OpenTelemetry trace plugin" Off)
 set(YDB_SDK_GOOGLE_COMMON_PROTOS_TARGET "" CACHE STRING "Name of cmake target preparing google common proto library")
 option(YDB_SDK_USE_RAPID_JSON "Search for rapid json library in system" ON)
 
@@ -58,6 +60,7 @@ add_subdirectory(library/cpp)
 add_subdirectory(include/ydb-cpp-sdk/client)
 add_subdirectory(src)
 add_subdirectory(util)
+add_subdirectory(plugins)
 
 #_ydb_sdk_validate_public_headers()
 

cmake/external_libs.cmake

@@ -14,6 +14,11 @@ find_package(Brotli 1.1.0 REQUIRED)
 find_package(jwt-cpp REQUIRED)
 find_package(double-conversion REQUIRED)
 
+# OpenTelemetry
+if (YDB_SDK_ENABLE_OTEL_METRICS OR YDB_SDK_ENABLE_OTEL_TRACE)
+  find_package(opentelemetry-cpp REQUIRED)
+endif()
+
 # RapidJSON
 if (YDB_SDK_USE_RAPID_JSON)
   find_package(RapidJSON REQUIRED)

include/ydb-cpp-sdk/client/driver/driver.h

@@ -3,6 +3,8 @@
 #include "fwd.h"
 
 #include <ydb-cpp-sdk/client/common_client/settings.h>
+#include <ydb-cpp-sdk/client/metrics/metrics.h>
+#include <ydb-cpp-sdk/client/trace/trace.h>
 #include <ydb-cpp-sdk/client/types/status_codes.h>
 #include <ydb-cpp-sdk/client/types/credentials/credentials.h>
 #include <ydb-cpp-sdk/client/types/fatal_error_handlers/handlers.h>
@@ -12,6 +14,8 @@
 
 #include <library/cpp/logger/backend.h>
 
+#include <grpcpp/grpcpp.h>
+
 ////////////////////////////////////////////////////////////////////////////////
 
 namespace NYdb::inline V3 {
@@ -137,6 +141,11 @@ class TDriverConfig {
     //! default: "round_robin"
     TDriverConfig& SetGRpcLoadBalancingPolicy(const std::string& policy);
 
+    //! Set grpc compression algorithm
+    //! algorithm - EGrpcCompressionAlgorithm enum value, see grpc documentation for available algorithms
+    //! default: EGrpcCompressionAlgorithm::None
+    TDriverConfig& SetGRpcCompressionAlgorithm(EGrpcCompressionAlgorithm algorithm);
+
     //! Set inactive socket timeout.
     //! Used to close connections, that were inactive for given time.
     //! Closes unused connections every 1/10 of timeout, so deletion time is approximate.
@@ -159,13 +168,27 @@ class TDriverConfig {
     //! default: 0
     TDriverConfig& SetMaxMessageSize(uint64_t maxMessageSize);
 
+    //! Append a segment to the SDK build info header (x-ydb-sdk-build-info).
+    //! Do not call this method unless you know exactly what you are doing.
+    //! Segments are joined with ';'. Each segment must match: <name>/<X>.<Y>.<Z>
+    //!   name chars: lowercase latin letters, digits, '-'
+    //!   X, Y, Z chars: lowercase latin letters, digits
+    //! Throws on invalid format or if total extra length exceeds 512 bytes.
+    TDriverConfig& AppendBuildInfo(std::string_view segment);
+
     //! Log backend.
     TDriverConfig& SetLog(std::unique_ptr<TLogBackend>&& log);
 
     //! Set executor for async responses.
     //! If not set, default executor will be used.
     TDriverConfig& SetExecutor(std::shared_ptr<IExecutor> executor);
 
+    //! Set external metrics registry implementation.
+    TDriverConfig& SetMetricRegistry(std::shared_ptr<NMetrics::IMetricRegistry> registry);
+
+    //! Set external trace provider implementation.
+    TDriverConfig& SetTraceProvider(std::shared_ptr<NTrace::ITraceProvider> provider);
+
 private:
     class TImpl;
     std::shared_ptr<TImpl> Impl_;

include/ydb-cpp-sdk/client/import/import.h

@@ -162,10 +162,10 @@ struct TImportFromFsSettings : public TOperationRequestSettings<TImportFromFsSet
         // database path where to import data
         std::string Dst;
 
-        // Source path.
+        // Source path in database.
         // If the export contains the database objects list, you may specify the database object name,
         // and the FS path will be looked up in the database objects list by the import procedure
-        std::string SrcPath = {};
+        std::string SrcPathDb = {};
     };
 
     FLUENT_SETTING(std::string, BasePath);

include/ydb-cpp-sdk/client/metrics/metrics.h

@@ -0,0 +1,57 @@
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace NYdb::inline V3::NMetrics {
+
+using TLabels = std::map<std::string, std::string>;
+
+class ICounter {
+public:
+    virtual ~ICounter() = default;
+    virtual void Inc() = 0;
+};
+
+class IGauge {
+public:
+    virtual ~IGauge() = default;
+    virtual void Add(double delta) = 0;
+    virtual void Set(double value) = 0;
+};
+
+class IHistogram {
+public:
+    virtual ~IHistogram() = default;
+    virtual void Record(double value) = 0;
+};
+
+class IMetricRegistry {
+public:
+    virtual ~IMetricRegistry() = default;
+
+    virtual std::shared_ptr<ICounter> Counter(
+        const std::string& name,
+        const TLabels& labels = {},
+        const std::string& description = {},
+        const std::string& unit = {}
+    ) = 0;
+    virtual std::shared_ptr<IGauge> Gauge(
+        const std::string& name,
+        const TLabels& labels = {},
+        const std::string& description = {},
+        const std::string& unit = {}
+    ) = 0;
+    virtual std::shared_ptr<IHistogram> Histogram(
+        const std::string& name,
+        const std::vector<double>& buckets,
+        const TLabels& labels = {},
+        const std::string& description = {},
+        const std::string& unit = {}
+    ) = 0;
+};
+
+} // namespace NYdb::NMetrics

include/ydb-cpp-sdk/client/table/table.h

@@ -442,6 +442,7 @@ class TIndexDescription {
     const std::vector<std::string>& GetDataColumns() const;
     const std::variant<std::monostate, TKMeansTreeSettings, TFulltextIndexSettings>& GetIndexSettings() const;
     uint64_t GetSizeBytes() const;
+    void SetParallel(uint32_t parallel);
 
     static TIndexDescription CreateGlobalIndex(
         const std::string& name,
@@ -521,6 +522,7 @@ class TIndexDescription {
     std::vector<TGlobalIndexSettings> GlobalIndexSettings_;
     std::variant<std::monostate, TKMeansTreeSettings, TFulltextIndexSettings> SpecializedIndexSettings_;
     uint64_t SizeBytes_ = 0;
+    uint32_t Parallel_ = 0;
 };
 
 struct TRenameIndex {
@@ -618,6 +620,8 @@ class TChangefeedDescription {
     TChangefeedDescription& WithInitialScan();
     // Enable UserSIDs
     TChangefeedDescription& WithUserSIDs();
+    // Enable TraceIds
+    TChangefeedDescription& WithTraceIds();
     // Attributes
     TChangefeedDescription& AddAttribute(const std::string& key, const std::string& value);
     TChangefeedDescription& SetAttributes(const std::unordered_map<std::string, std::string>& attrs);
@@ -634,6 +638,7 @@ class TChangefeedDescription {
     const std::optional<TDuration>& GetResolvedTimestamps() const;
     bool GetInitialScan() const;
     bool GetUserSIDs() const;
+    bool GetTraceIds() const;
     const std::unordered_map<std::string, std::string>& GetAttributes() const;
     const std::string& GetAwsRegion() const;
     const std::optional<TInitialScanProgress>& GetInitialScanProgress() const;
@@ -664,6 +669,7 @@ class TChangefeedDescription {
     std::optional<TDuration> RetentionPeriod_;
     bool InitialScan_ = false;
     bool UserSIDs_ = false;
+    bool TraceIds_ = false;
     std::unordered_map<std::string, std::string> Attributes_;
     std::string AwsRegion_;
     std::optional<TInitialScanProgress> InitialScanProgress_;

include/ydb-cpp-sdk/client/topic/write_session.h

@@ -181,6 +181,8 @@ struct TWriteSessionSettings : public TRequestSettings<TWriteSessionSettings> {
 
     //! Enables validation of SeqNo. If enabled, then writer will check writing with seqNo and without it and throws exception.
     FLUENT_SETTING_DEFAULT(bool, ValidateSeqNo, true);
+
+    TWriteSessionSettings& SetTrackProducerIdInTx(bool value);
 };
 
 template<class T>

include/ydb-cpp-sdk/client/trace/trace.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <string>
+
+namespace NYdb::inline V3::NTrace {
+
+enum class ESpanKind {
+    INTERNAL,
+    SERVER,
+    CLIENT,
+    PRODUCER,
+    CONSUMER
+};
+
+class ISpan {
+public:
+    virtual ~ISpan() = default;
+    virtual void End() = 0;
+    virtual void SetAttribute(const std::string& key, const std::string& value) = 0;
+    virtual void SetAttribute(const std::string& key, int64_t value) = 0;
+    virtual void AddEvent(const std::string& name, const std::map<std::string, std::string>& attributes = {}) = 0;
+};
+
+class ITracer {
+public:
+    virtual ~ITracer() = default;
+    virtual std::shared_ptr<ISpan> StartSpan(const std::string& name, ESpanKind kind = ESpanKind::INTERNAL) = 0;
+};
+
+class ITraceProvider {
+public:
+    virtual ~ITraceProvider() = default;
+    virtual std::shared_ptr<ITracer> GetTracer(const std::string& name) = 0;
+};
+
+} // namespace NYdb::NTrace

include/ydb-cpp-sdk/client/types/ydb.h

@@ -46,6 +46,12 @@ enum EPileState {
     DISCONNECTED = 6 /* "disconnected" */
 };
 
+enum class EGrpcCompressionAlgorithm {
+    None,
+    Deflate,
+    Gzip
+};
+
 class TBalancingPolicy {
     friend class TDriverConfig;
     friend class TDriver;

include/ydb-cpp-sdk/open_telemetry/metrics.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include <ydb-cpp-sdk/client/metrics/metrics.h>
+
+#include <opentelemetry/version.h>
+#include <opentelemetry/nostd/shared_ptr.h>
+
+OPENTELEMETRY_BEGIN_NAMESPACE
+namespace metrics {
+class MeterProvider;
+}
+OPENTELEMETRY_END_NAMESPACE
+
+namespace NYdb::inline V3::NMetrics {
+
+std::shared_ptr<IMetricRegistry> CreateOtelMetricRegistry(
+    opentelemetry::nostd::shared_ptr<opentelemetry::metrics::MeterProvider> meterProvider);
+
+} // namespace NYdb::NMetrics

include/ydb-cpp-sdk/open_telemetry/trace.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include <ydb-cpp-sdk/client/trace/trace.h>
+
+#include <opentelemetry/version.h>
+#include <opentelemetry/nostd/shared_ptr.h>
+
+OPENTELEMETRY_BEGIN_NAMESPACE
+namespace trace {
+class TracerProvider;
+}
+OPENTELEMETRY_END_NAMESPACE
+
+namespace NYdb::inline V3::NTrace {
+
+std::shared_ptr<ITraceProvider> CreateOtelTraceProvider(
+    opentelemetry::nostd::shared_ptr<opentelemetry::trace::TracerProvider> tracerProvider);
+
+} // namespace NYdb::NTrace

plugins/CMakeLists.txt

@@ -0,0 +1,2 @@
+add_subdirectory(metrics)
+add_subdirectory(trace)

plugins/metrics/CMakeLists.txt

@@ -0,0 +1,3 @@
+if (YDB_SDK_ENABLE_OTEL_METRICS)
+  add_subdirectory(otel EXCLUDE_FROM_ALL)
+endif()

plugins/metrics/otel/CMakeLists.txt

@@ -0,0 +1,18 @@
+_ydb_sdk_add_library(open_telemetry_metrics)
+
+target_sources(open_telemetry_metrics PRIVATE
+  src/metrics.cpp
+)
+target_include_directories(open_telemetry_metrics PUBLIC
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  $<INSTALL_INTERFACE:include>
+)
+target_link_libraries(open_telemetry_metrics PUBLIC
+  client-metrics
+  client-resources
+  opentelemetry-cpp::api
+  opentelemetry-cpp::metrics
+)
+_ydb_sdk_make_client_component(OpenTelemetryMetrics open_telemetry_metrics)
+
+_ydb_sdk_install_headers(${CMAKE_INSTALL_INCLUDEDIR} DIRECTORY include/)