diff --git a/proto/collections.proto b/proto/collections.proto index 92bf455..1d52f4c 100644 --- a/proto/collections.proto +++ b/proto/collections.proto @@ -1,11 +1,11 @@ syntax = "proto3"; package qdrant; -option csharp_namespace = "Qdrant.Client.Grpc"; - import "json_with_int.proto"; import "qdrant_common.proto"; +option csharp_namespace = "Qdrant.Client.Grpc"; + enum Datatype { Default = 0; Float32 = 1; @@ -361,11 +361,24 @@ message BinaryQuantization { optional BinaryQuantizationQueryEncoding query_encoding = 3; } +message TurboQuantization{ + optional bool always_ram = 1; + optional TurboQuantBitSize bits = 2; +} + +enum TurboQuantBitSize{ + Bits1 = 0; + Bits1_5 = 1; + Bits2 = 2; + Bits4 = 3; +} + message QuantizationConfig { oneof quantization { ScalarQuantization scalar = 1; ProductQuantization product = 2; BinaryQuantization binary = 3; + TurboQuantization turboquant = 4; } } @@ -377,6 +390,7 @@ message QuantizationConfigDiff { ProductQuantization product = 2; Disabled disabled = 3; BinaryQuantization binary = 4; + TurboQuantization turboquant = 5; } } @@ -406,6 +420,8 @@ message StrictModeConfig { optional float search_max_oversampling = 8; // Max batchsize when upserting optional uint64 upsert_max_batchsize = 9; + // Max batchsize when searching + optional uint64 search_max_batchsize = 20; // Max size of a collections vector storage in bytes, ignoring replicas. optional uint64 max_collection_vector_size_bytes = 10; // Max number of read operations per minute per replica @@ -426,6 +442,9 @@ message StrictModeConfig { optional uint64 max_points_count = 18; // Max number of payload indexes in a collection optional uint64 max_payload_index_count = 19; + // Reject memory-consuming update operations when process resident memory exceeds this percentage of total RAM (cgroup-aware, 1-100). + // Delete-style operations are still allowed so memory can be freed. + optional uint32 max_resident_memory_percent = 21; } message StrictModeSparseConfig { @@ -749,8 +768,11 @@ message PayloadSchemaInfo { } message UpdateQueueInfo { - // Number of elements in the queue - uint64 length = 1; + // Number of elements in the queue + uint64 length = 1; + + // Number of points that are deferred (i.e hidden from search as they're not yet optimized). + optional uint64 deferred_points = 2; } message CollectionInfo { diff --git a/proto/points.proto b/proto/points.proto index b03adc3..767d4cb 100644 --- a/proto/points.proto +++ b/proto/points.proto @@ -352,6 +352,62 @@ message DeleteFieldIndexCollection { optional uint64 timeout = 5; } +// Dense vector creation parameters. +// Only includes immutable properties that define the vector space. +// Storage type, index, and quantization are configured separately. +message DenseVectorCreationConfig { + // Size/dimensionality of the vectors + uint64 size = 1; + // Distance function used for comparing vectors + Distance distance = 2; + // Configuration for multi-vector search (e.g., ColBERT) + optional MultiVectorConfig multivector_config = 3; + // Data type of the vectors (Float32, Float16, Uint8) + optional Datatype datatype = 4; +} + +// Sparse vector creation parameters. +// Only includes immutable properties that define the vector space. +message SparseVectorCreationConfig { + // If set - apply modifier to the vector values (e.g., IDF) + optional Modifier modifier = 1; + // Data type used to store weights in the index + optional Datatype datatype = 2; +} + +message CreateVectorNameRequest { + // Name of the collection + string collection_name = 1; + // Wait until the changes have been applied? + optional bool wait = 2; + // Name of the new vector + string vector_name = 3; + // Configuration for the new vector - either dense or sparse + oneof vector_config { + // Dense vector parameters + DenseVectorCreationConfig dense_config = 4; + // Sparse vector parameters + SparseVectorCreationConfig sparse_config = 5; + } + // If set, overrides global timeout setting for this request. Unit is seconds. + optional uint64 timeout = 6; + // Write ordering guarantees + optional WriteOrdering ordering = 7; +} + +message DeleteVectorNameRequest { + // Name of the collection + string collection_name = 1; + // Wait until the changes have been applied? + optional bool wait = 2; + // Name of the vector to delete + string vector_name = 3; + // If set, overrides global timeout setting for this request. Unit is seconds. + optional uint64 timeout = 4; + // Write ordering guarantees + optional WriteOrdering ordering = 5; +} + message PayloadIncludeSelector { // List of payload keys to include into result repeated string fields = 1; @@ -417,11 +473,11 @@ message QuantizationSearchParams { // Oversampling factor for quantization. // - // Defines how many extra vectors should be pre-selected using quantized index, + // Defines how many extra vectors should be preselected using quantized index, // and then re-scored using original vectors. // // For example, if `oversampling` is 2.4 and `limit` is 100, - // then 240 vectors will be pre-selected using quantized index, + // then 240 vectors will be preselected using quantized index, // and then top-100 will be returned after re-scoring. optional double oversampling = 3; } diff --git a/proto/points_service.proto b/proto/points_service.proto index 860d4c7..162cb2f 100644 --- a/proto/points_service.proto +++ b/proto/points_service.proto @@ -31,6 +31,10 @@ service Points { // Delete field index for collection rpc DeleteFieldIndex(DeleteFieldIndexCollection) returns (PointsOperationResponse) {} + // Create a new named vector on the collection + rpc CreateVectorName(CreateVectorNameRequest) returns (PointsOperationResponse) {} + // Delete a named vector from the collection + rpc DeleteVectorName(DeleteVectorNameRequest) returns (PointsOperationResponse) {} // Retrieve closest points based on vector similarity and given filtering // conditions rpc Search(SearchPoints) returns (SearchResponse) {} diff --git a/src/builder_ext.rs b/src/builder_ext.rs index 873db15..28c31ed 100644 --- a/src/builder_ext.rs +++ b/src/builder_ext.rs @@ -4,10 +4,11 @@ use crate::qdrant::update_collection_cluster_setup_request::Operation; use crate::qdrant::{ shard_key, AbortShardTransferBuilder, BinaryQuantizationBuilder, ClearPayloadPointsBuilder, ContextExamplePair, CountPointsBuilder, CreateAliasBuilder, CreateCollectionBuilder, - CreateFieldIndexCollectionBuilder, CreateShardKeyRequestBuilder, DeleteCollectionBuilder, - DeleteFieldIndexCollectionBuilder, DeletePayloadPointsBuilder, DeletePointVectorsBuilder, - DeletePointsBuilder, DeleteShardKey, DeleteShardKeyRequestBuilder, - DeleteSnapshotRequestBuilder, DiscoverBatchPointsBuilder, DiscoverPoints, + CreateFieldIndexCollectionBuilder, CreateShardKeyRequestBuilder, + CreateVectorNameRequestBuilder, DeleteCollectionBuilder, DeleteFieldIndexCollectionBuilder, + DeletePayloadPointsBuilder, DeletePointVectorsBuilder, DeletePointsBuilder, DeleteShardKey, + DeleteShardKeyRequestBuilder, DeleteSnapshotRequestBuilder, DeleteVectorNameRequestBuilder, + DenseVectorCreationConfigBuilder, DiscoverBatchPointsBuilder, DiscoverPoints, DiscoverPointsBuilder, Distance, FacetCountsBuilder, FieldType, GetPointsBuilder, LookupLocationBuilder, MoveShardBuilder, PayloadExcludeSelector, PayloadIncludeSelector, PointId, PointStruct, PointVectors, PointsUpdateOperation, ProductQuantizationBuilder, @@ -16,9 +17,10 @@ use crate::qdrant::{ RecommendPoints, RecommendPointsBuilder, RenameAliasBuilder, ReplicaBuilder, ReplicateShardBuilder, ScalarQuantizationBuilder, ScrollPointsBuilder, SearchBatchPointsBuilder, SearchMatrixPointsBuilder, SearchPointGroupsBuilder, SearchPoints, - SearchPointsBuilder, SetPayloadPointsBuilder, ShardKey, UpdateBatchPointsBuilder, - UpdateCollectionBuilder, UpdateCollectionClusterSetupRequestBuilder, UpdatePointVectorsBuilder, - UpsertPointsBuilder, Value, VectorParamsBuilder, VectorsSelector, WithLookupBuilder, + SearchPointsBuilder, SetPayloadPointsBuilder, ShardKey, SparseVectorCreationConfigBuilder, + TurboQuantizationBuilder, UpdateBatchPointsBuilder, UpdateCollectionBuilder, + UpdateCollectionClusterSetupRequestBuilder, UpdatePointVectorsBuilder, UpsertPointsBuilder, + Value, VectorParamsBuilder, VectorsSelector, WithLookupBuilder, }; impl VectorParamsBuilder { @@ -54,6 +56,12 @@ impl BinaryQuantizationBuilder { } } +impl TurboQuantizationBuilder { + pub fn new() -> Self { + Self::empty() + } +} + impl SearchPointsBuilder { pub fn new( collection_name: impl Into, @@ -307,6 +315,43 @@ impl DeleteFieldIndexCollectionBuilder { } } +impl DenseVectorCreationConfigBuilder { + pub fn new(size: u64, distance: Distance) -> Self { + let mut builder = Self::empty(); + builder.size = Some(size); + builder.distance = Some(distance.into()); + builder + } +} + +impl SparseVectorCreationConfigBuilder { + pub fn new() -> Self { + Self::empty() + } +} + +impl CreateVectorNameRequestBuilder { + pub fn new( + collection_name: impl Into, + vector_name: impl Into, + vector_config: impl Into, + ) -> Self { + let mut builder = Self::empty(); + builder.collection_name = Some(collection_name.into()); + builder.vector_name = Some(vector_name.into()); + builder.vector_config(vector_config) + } +} + +impl DeleteVectorNameRequestBuilder { + pub fn new(collection_name: impl Into, vector_name: impl Into) -> Self { + let mut builder = Self::empty(); + builder.collection_name = Some(collection_name.into()); + builder.vector_name = Some(vector_name.into()); + builder + } +} + impl UpdateCollectionClusterSetupRequestBuilder { pub fn new(collection_name: impl Into, operation: impl Into) -> Self { let mut builder = Self::empty(); diff --git a/src/builders/create_vector_name_request_builder.rs b/src/builders/create_vector_name_request_builder.rs new file mode 100644 index 0000000..85e2791 --- /dev/null +++ b/src/builders/create_vector_name_request_builder.rs @@ -0,0 +1,173 @@ +use crate::grpc_macros::convert_option; +use crate::qdrant::*; + +#[must_use] +#[derive(Clone)] +pub struct CreateVectorNameRequestBuilder { + /// Name of the collection + pub(crate) collection_name: Option, + /// Wait until the changes have been applied? + pub(crate) wait: Option>, + /// Name of the new vector + pub(crate) vector_name: Option, + /// Configuration for the new vector - either dense or sparse + vector_config: Option, + /// If set, overrides global timeout setting for this request. Unit is seconds. + pub(crate) timeout: Option>, + /// Write ordering guarantees + pub(crate) ordering: Option>, +} + +impl CreateVectorNameRequestBuilder { + /// Name of the collection + pub fn collection_name(self, value: impl Into) -> Self { + let mut new = self; + new.collection_name = Option::Some(value.into()); + new + } + + /// Wait until the changes have been applied? + pub fn wait(self, value: bool) -> Self { + let mut new = self; + new.wait = Option::Some(Option::Some(value)); + new + } + + /// Name of the new vector + pub fn vector_name(self, value: impl Into) -> Self { + let mut new = self; + new.vector_name = Option::Some(value.into()); + new + } + + /// Configuration for the new vector - either dense or sparse. + /// + /// Accepts either a [`DenseVectorCreationConfig`] (or its builder) + /// or a [`SparseVectorCreationConfig`] (or its builder). + pub fn vector_config>( + self, + value: VALUE, + ) -> Self { + let mut new = self; + new.vector_config = Option::Some(value.into()); + new + } + + /// If set, overrides global timeout setting for this request. Unit is seconds. + pub fn timeout(self, value: u64) -> Self { + let mut new = self; + new.timeout = Option::Some(Option::Some(value)); + new + } + + /// Write ordering guarantees + pub fn ordering>(self, value: VALUE) -> Self { + let mut new = self; + new.ordering = Option::Some(Option::Some(value.into())); + new + } + + fn build_inner(self) -> Result { + Ok(CreateVectorNameRequest { + collection_name: match self.collection_name { + Some(value) => value, + None => { + return Result::Err(core::convert::Into::into( + ::derive_builder::UninitializedFieldError::from("collection_name"), + )); + } + }, + wait: self.wait.unwrap_or_default(), + vector_name: match self.vector_name { + Some(value) => value, + None => { + return Result::Err(core::convert::Into::into( + ::derive_builder::UninitializedFieldError::from("vector_name"), + )); + } + }, + timeout: self.timeout.unwrap_or_default(), + ordering: self.ordering.unwrap_or_default(), + vector_config: { convert_option(&self.vector_config) }, + }) + } + /// Create an empty builder, with all fields set to `None` or `PhantomData`. + fn create_empty() -> Self { + Self { + collection_name: core::default::Default::default(), + wait: core::default::Default::default(), + vector_name: core::default::Default::default(), + vector_config: core::default::Default::default(), + timeout: core::default::Default::default(), + ordering: core::default::Default::default(), + } + } +} + +impl From for CreateVectorNameRequest { + fn from(value: CreateVectorNameRequestBuilder) -> Self { + value.build_inner().unwrap_or_else(|_| { + panic!( + "Failed to convert {0} to {1}", + "CreateVectorNameRequestBuilder", "CreateVectorNameRequest" + ) + }) + } +} + +impl CreateVectorNameRequestBuilder { + /// Builds the desired type. Can often be omitted. + pub fn build(self) -> CreateVectorNameRequest { + self.build_inner().unwrap_or_else(|_| { + panic!( + "Failed to build {0} into {1}", + "CreateVectorNameRequestBuilder", "CreateVectorNameRequest" + ) + }) + } +} + +impl CreateVectorNameRequestBuilder { + pub(crate) fn empty() -> Self { + Self::create_empty() + } +} + +/// Error type for CreateVectorNameRequestBuilder +#[non_exhaustive] +#[derive(Debug)] +pub enum CreateVectorNameRequestBuilderError { + /// Uninitialized field + UninitializedField(&'static str), + /// Custom validation error + ValidationError(String), +} + +// Implementing the Display trait for better error messages +impl std::fmt::Display for CreateVectorNameRequestBuilderError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + Self::UninitializedField(field) => { + write!(f, "`{field}` must be initialized") + } + Self::ValidationError(error) => write!(f, "{error}"), + } + } +} + +// Implementing the Error trait +impl std::error::Error for CreateVectorNameRequestBuilderError {} + +// Implementing From trait for conversion from UninitializedFieldError +impl From for CreateVectorNameRequestBuilderError { + fn from(error: derive_builder::UninitializedFieldError) -> Self { + Self::UninitializedField(error.field_name()) + } +} + +// Implementing From trait for conversion from String +impl From for CreateVectorNameRequestBuilderError { + fn from(error: String) -> Self { + Self::ValidationError(error) + } +} diff --git a/src/builders/delete_vector_name_request_builder.rs b/src/builders/delete_vector_name_request_builder.rs new file mode 100644 index 0000000..2c84f9c --- /dev/null +++ b/src/builders/delete_vector_name_request_builder.rs @@ -0,0 +1,155 @@ +use crate::qdrant::*; + +#[must_use] +#[derive(Clone)] +pub struct DeleteVectorNameRequestBuilder { + /// Name of the collection + pub(crate) collection_name: Option, + /// Wait until the changes have been applied? + pub(crate) wait: Option>, + /// Name of the vector to delete + pub(crate) vector_name: Option, + /// If set, overrides global timeout setting for this request. Unit is seconds. + pub(crate) timeout: Option>, + /// Write ordering guarantees + pub(crate) ordering: Option>, +} + +impl DeleteVectorNameRequestBuilder { + /// Name of the collection + pub fn collection_name(self, value: impl Into) -> Self { + let mut new = self; + new.collection_name = Option::Some(value.into()); + new + } + + /// Wait until the changes have been applied? + pub fn wait(self, value: bool) -> Self { + let mut new = self; + new.wait = Option::Some(Option::Some(value)); + new + } + + /// Name of the vector to delete + pub fn vector_name(self, value: impl Into) -> Self { + let mut new = self; + new.vector_name = Option::Some(value.into()); + new + } + + /// If set, overrides global timeout setting for this request. Unit is seconds. + pub fn timeout(self, value: u64) -> Self { + let mut new = self; + new.timeout = Option::Some(Option::Some(value)); + new + } + + /// Write ordering guarantees + pub fn ordering>(self, value: VALUE) -> Self { + let mut new = self; + new.ordering = Option::Some(Option::Some(value.into())); + new + } + + fn build_inner(self) -> Result { + Ok(DeleteVectorNameRequest { + collection_name: match self.collection_name { + Some(value) => value, + None => { + return Result::Err(core::convert::Into::into( + ::derive_builder::UninitializedFieldError::from("collection_name"), + )); + } + }, + wait: self.wait.unwrap_or_default(), + vector_name: match self.vector_name { + Some(value) => value, + None => { + return Result::Err(core::convert::Into::into( + ::derive_builder::UninitializedFieldError::from("vector_name"), + )); + } + }, + timeout: self.timeout.unwrap_or_default(), + ordering: self.ordering.unwrap_or_default(), + }) + } + /// Create an empty builder, with all fields set to `None` or `PhantomData`. + fn create_empty() -> Self { + Self { + collection_name: core::default::Default::default(), + wait: core::default::Default::default(), + vector_name: core::default::Default::default(), + timeout: core::default::Default::default(), + ordering: core::default::Default::default(), + } + } +} + +impl From for DeleteVectorNameRequest { + fn from(value: DeleteVectorNameRequestBuilder) -> Self { + value.build_inner().unwrap_or_else(|_| { + panic!( + "Failed to convert {0} to {1}", + "DeleteVectorNameRequestBuilder", "DeleteVectorNameRequest" + ) + }) + } +} + +impl DeleteVectorNameRequestBuilder { + /// Builds the desired type. Can often be omitted. + pub fn build(self) -> DeleteVectorNameRequest { + self.build_inner().unwrap_or_else(|_| { + panic!( + "Failed to build {0} into {1}", + "DeleteVectorNameRequestBuilder", "DeleteVectorNameRequest" + ) + }) + } +} + +impl DeleteVectorNameRequestBuilder { + pub(crate) fn empty() -> Self { + Self::create_empty() + } +} + +/// Error type for DeleteVectorNameRequestBuilder +#[non_exhaustive] +#[derive(Debug)] +pub enum DeleteVectorNameRequestBuilderError { + /// Uninitialized field + UninitializedField(&'static str), + /// Custom validation error + ValidationError(String), +} + +// Implementing the Display trait for better error messages +impl std::fmt::Display for DeleteVectorNameRequestBuilderError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + Self::UninitializedField(field) => { + write!(f, "`{field}` must be initialized") + } + Self::ValidationError(error) => write!(f, "{error}"), + } + } +} + +// Implementing the Error trait +impl std::error::Error for DeleteVectorNameRequestBuilderError {} + +// Implementing From trait for conversion from UninitializedFieldError +impl From for DeleteVectorNameRequestBuilderError { + fn from(error: derive_builder::UninitializedFieldError) -> Self { + Self::UninitializedField(error.field_name()) + } +} + +// Implementing From trait for conversion from String +impl From for DeleteVectorNameRequestBuilderError { + fn from(error: String) -> Self { + Self::ValidationError(error) + } +} diff --git a/src/builders/dense_vector_creation_config_builder.rs b/src/builders/dense_vector_creation_config_builder.rs new file mode 100644 index 0000000..1648b8f --- /dev/null +++ b/src/builders/dense_vector_creation_config_builder.rs @@ -0,0 +1,152 @@ +use crate::qdrant::*; + +/// Dense vector creation parameters. +/// Only includes immutable properties that define the vector space. +/// Storage type, index, and quantization are configured separately. +#[must_use] +#[derive(Clone)] +pub struct DenseVectorCreationConfigBuilder { + /// Size/dimensionality of the vectors + pub(crate) size: Option, + /// Distance function used for comparing vectors + pub(crate) distance: Option, + /// Configuration for multi-vector search (e.g., ColBERT) + pub(crate) multivector_config: Option>, + /// Data type of the vectors (Float32, Float16, Uint8) + pub(crate) datatype: Option>, +} + +impl DenseVectorCreationConfigBuilder { + /// Size/dimensionality of the vectors + pub fn size(self, value: u64) -> Self { + let mut new = self; + new.size = Option::Some(value); + new + } + + /// Distance function used for comparing vectors + pub fn distance>(self, value: VALUE) -> Self { + let mut new = self; + new.distance = Option::Some(value.into()); + new + } + + /// Configuration for multi-vector search (e.g., ColBERT) + pub fn multivector_config>( + self, + value: VALUE, + ) -> Self { + let mut new = self; + new.multivector_config = Option::Some(Option::Some(value.into())); + new + } + + /// Data type of the vectors (Float32, Float16, Uint8) + pub fn datatype>(self, value: VALUE) -> Self { + let mut new = self; + new.datatype = Option::Some(Option::Some(value.into())); + new + } + + fn build_inner( + self, + ) -> Result { + Ok(DenseVectorCreationConfig { + size: match self.size { + Some(value) => value, + None => { + return Result::Err(core::convert::Into::into( + ::derive_builder::UninitializedFieldError::from("size"), + )); + } + }, + distance: match self.distance { + Some(value) => value, + None => { + return Result::Err(core::convert::Into::into( + ::derive_builder::UninitializedFieldError::from("distance"), + )); + } + }, + multivector_config: self.multivector_config.unwrap_or_default(), + datatype: self.datatype.unwrap_or_default(), + }) + } + /// Create an empty builder, with all fields set to `None` or `PhantomData`. + fn create_empty() -> Self { + Self { + size: core::default::Default::default(), + distance: core::default::Default::default(), + multivector_config: core::default::Default::default(), + datatype: core::default::Default::default(), + } + } +} + +impl From for DenseVectorCreationConfig { + fn from(value: DenseVectorCreationConfigBuilder) -> Self { + value.build_inner().unwrap_or_else(|_| { + panic!( + "Failed to convert {0} to {1}", + "DenseVectorCreationConfigBuilder", "DenseVectorCreationConfig" + ) + }) + } +} + +impl DenseVectorCreationConfigBuilder { + /// Builds the desired type. Can often be omitted. + pub fn build(self) -> DenseVectorCreationConfig { + self.build_inner().unwrap_or_else(|_| { + panic!( + "Failed to build {0} into {1}", + "DenseVectorCreationConfigBuilder", "DenseVectorCreationConfig" + ) + }) + } +} + +impl DenseVectorCreationConfigBuilder { + pub(crate) fn empty() -> Self { + Self::create_empty() + } +} + +/// Error type for DenseVectorCreationConfigBuilder +#[non_exhaustive] +#[derive(Debug)] +pub enum DenseVectorCreationConfigBuilderError { + /// Uninitialized field + UninitializedField(&'static str), + /// Custom validation error + ValidationError(String), +} + +// Implementing the Display trait for better error messages +impl std::fmt::Display for DenseVectorCreationConfigBuilderError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + Self::UninitializedField(field) => { + write!(f, "`{field}` must be initialized") + } + Self::ValidationError(error) => write!(f, "{error}"), + } + } +} + +// Implementing the Error trait +impl std::error::Error for DenseVectorCreationConfigBuilderError {} + +// Implementing From trait for conversion from UninitializedFieldError +impl From for DenseVectorCreationConfigBuilderError { + fn from(error: derive_builder::UninitializedFieldError) -> Self { + Self::UninitializedField(error.field_name()) + } +} + +// Implementing From trait for conversion from String +impl From for DenseVectorCreationConfigBuilderError { + fn from(error: String) -> Self { + Self::ValidationError(error) + } +} diff --git a/src/builders/mod.rs b/src/builders/mod.rs index 38843c7..e7703d6 100644 --- a/src/builders/mod.rs +++ b/src/builders/mod.rs @@ -31,6 +31,9 @@ pub use product_quantization_builder::ProductQuantizationBuilder; mod binary_quantization_builder; pub use binary_quantization_builder::BinaryQuantizationBuilder; +mod turbo_quantization_builder; +pub use turbo_quantization_builder::TurboQuantizationBuilder; + mod strict_mode_config_builder; pub use strict_mode_config_builder::StrictModeConfigBuilder; @@ -142,6 +145,18 @@ pub use create_field_index_collection_builder::CreateFieldIndexCollectionBuilder mod delete_field_index_collection_builder; pub use delete_field_index_collection_builder::DeleteFieldIndexCollectionBuilder; +mod dense_vector_creation_config_builder; +pub use dense_vector_creation_config_builder::DenseVectorCreationConfigBuilder; + +mod sparse_vector_creation_config_builder; +pub use sparse_vector_creation_config_builder::SparseVectorCreationConfigBuilder; + +mod create_vector_name_request_builder; +pub use create_vector_name_request_builder::CreateVectorNameRequestBuilder; + +mod delete_vector_name_request_builder; +pub use delete_vector_name_request_builder::DeleteVectorNameRequestBuilder; + mod quantization_search_params_builder; pub use quantization_search_params_builder::QuantizationSearchParamsBuilder; diff --git a/src/builders/sparse_vector_creation_config_builder.rs b/src/builders/sparse_vector_creation_config_builder.rs new file mode 100644 index 0000000..fd94023 --- /dev/null +++ b/src/builders/sparse_vector_creation_config_builder.rs @@ -0,0 +1,78 @@ +use crate::qdrant::*; + +/// Sparse vector creation parameters. +/// Only includes immutable properties that define the vector space. +#[must_use] +#[derive(Clone)] +pub struct SparseVectorCreationConfigBuilder { + /// If set - apply modifier to the vector values (e.g., IDF) + pub(crate) modifier: Option>, + /// Data type used to store weights in the index + pub(crate) datatype: Option>, +} + +impl SparseVectorCreationConfigBuilder { + /// If set - apply modifier to the vector values (e.g., IDF) + pub fn modifier>(self, value: VALUE) -> Self { + let mut new = self; + new.modifier = Option::Some(Option::Some(value.into())); + new + } + + /// Data type used to store weights in the index + pub fn datatype>(self, value: VALUE) -> Self { + let mut new = self; + new.datatype = Option::Some(Option::Some(value.into())); + new + } + + fn build_inner(self) -> Result { + Ok(SparseVectorCreationConfig { + modifier: self.modifier.unwrap_or_default(), + datatype: self.datatype.unwrap_or_default(), + }) + } + + /// Create an empty builder, with all fields set to `None` or `PhantomData`. + fn create_empty() -> Self { + Self { + modifier: core::default::Default::default(), + datatype: core::default::Default::default(), + } + } +} + +impl From for SparseVectorCreationConfig { + fn from(value: SparseVectorCreationConfigBuilder) -> Self { + value.build_inner().unwrap_or_else(|_| { + panic!( + "Failed to convert {0} to {1}", + "SparseVectorCreationConfigBuilder", "SparseVectorCreationConfig" + ) + }) + } +} + +impl SparseVectorCreationConfigBuilder { + /// Builds the desired type. Can often be omitted. + pub fn build(self) -> SparseVectorCreationConfig { + self.build_inner().unwrap_or_else(|_| { + panic!( + "Failed to build {0} into {1}", + "SparseVectorCreationConfigBuilder", "SparseVectorCreationConfig" + ) + }) + } +} + +impl SparseVectorCreationConfigBuilder { + pub(crate) fn empty() -> Self { + Self::create_empty() + } +} + +impl Default for SparseVectorCreationConfigBuilder { + fn default() -> Self { + Self::create_empty() + } +} diff --git a/src/builders/strict_mode_config_builder.rs b/src/builders/strict_mode_config_builder.rs index 8919beb..d6db77f 100644 --- a/src/builders/strict_mode_config_builder.rs +++ b/src/builders/strict_mode_config_builder.rs @@ -12,6 +12,7 @@ pub struct StrictModeConfigBuilder { pub(crate) search_allow_exact: Option>, pub(crate) search_max_oversampling: Option>, pub(crate) upsert_max_batchsize: Option>, + pub(crate) search_max_batchsize: Option>, pub(crate) max_collection_vector_size_bytes: Option>, pub(crate) read_rate_limit: Option>, pub(crate) write_rate_limit: Option>, @@ -22,6 +23,7 @@ pub struct StrictModeConfigBuilder { pub(crate) sparse_config: Option>, pub(crate) max_points_count: Option>, pub(crate) max_payload_index_count: Option>, + pub(crate) max_resident_memory_percent: Option>, } impl StrictModeConfigBuilder { @@ -79,6 +81,12 @@ impl StrictModeConfigBuilder { new } + pub fn search_max_batchsize(self, value: u64) -> Self { + let mut new = self; + new.search_max_batchsize = Option::Some(Option::Some(value)); + new + } + pub fn max_collection_vector_size_bytes(self, value: u64) -> Self { let mut new = self; new.max_collection_vector_size_bytes = Option::Some(Option::Some(value)); @@ -139,6 +147,12 @@ impl StrictModeConfigBuilder { new } + pub fn max_resident_memory_percent(self, value: u32) -> Self { + let mut new = self; + new.max_resident_memory_percent = Option::Some(Option::Some(value)); + new + } + fn build_inner(self) -> Result { Ok(StrictModeConfig { enabled: self.enabled.unwrap_or_default(), @@ -150,6 +164,7 @@ impl StrictModeConfigBuilder { search_allow_exact: self.search_allow_exact.unwrap_or_default(), search_max_oversampling: self.search_max_oversampling.unwrap_or_default(), upsert_max_batchsize: self.upsert_max_batchsize.unwrap_or_default(), + search_max_batchsize: self.search_max_batchsize.unwrap_or_default(), max_collection_vector_size_bytes: self .max_collection_vector_size_bytes .unwrap_or_default(), @@ -164,6 +179,7 @@ impl StrictModeConfigBuilder { sparse_config: self.sparse_config.unwrap_or_default(), max_points_count: self.max_points_count.unwrap_or_default(), max_payload_index_count: self.max_payload_index_count.unwrap_or_default(), + max_resident_memory_percent: self.max_resident_memory_percent.unwrap_or_default(), }) } /// Create an empty builder, with all fields set to `None` or `PhantomData`. @@ -178,6 +194,7 @@ impl StrictModeConfigBuilder { search_allow_exact: core::default::Default::default(), search_max_oversampling: core::default::Default::default(), upsert_max_batchsize: core::default::Default::default(), + search_max_batchsize: core::default::Default::default(), max_collection_vector_size_bytes: core::default::Default::default(), read_rate_limit: core::default::Default::default(), write_rate_limit: core::default::Default::default(), @@ -188,6 +205,7 @@ impl StrictModeConfigBuilder { sparse_config: core::default::Default::default(), max_points_count: core::default::Default::default(), max_payload_index_count: core::default::Default::default(), + max_resident_memory_percent: core::default::Default::default(), } } } diff --git a/src/builders/turbo_quantization_builder.rs b/src/builders/turbo_quantization_builder.rs new file mode 100644 index 0000000..0773879 --- /dev/null +++ b/src/builders/turbo_quantization_builder.rs @@ -0,0 +1,114 @@ +use crate::qdrant::*; + +#[must_use] +#[derive(Clone)] +pub struct TurboQuantizationBuilder { + /// If true - quantized vectors always will be stored in RAM, ignoring the config of main storage + pub(crate) always_ram: Option>, + pub(crate) bits: Option>, +} + +impl TurboQuantizationBuilder { + /// If true - quantized vectors always will be stored in RAM, ignoring the config of main storage + pub fn always_ram(self, value: bool) -> Self { + let mut new = self; + new.always_ram = Some(Some(value)); + new + } + + /// Number of bits used to encode each component of the quantized vector. + pub fn bits(self, value: impl Into) -> Self { + let mut new = self; + let bits: TurboQuantBitSize = value.into(); + new.bits = Some(Some(bits.into())); + new + } + + fn build_inner(self) -> Result { + Ok(TurboQuantization { + always_ram: self.always_ram.unwrap_or_default(), + bits: self.bits.unwrap_or_default(), + }) + } + + /// Create an empty builder, with all fields set to `None` or `PhantomData`. + fn create_empty() -> Self { + Self { + always_ram: Default::default(), + bits: Default::default(), + } + } +} + +impl From for TurboQuantization { + fn from(value: TurboQuantizationBuilder) -> Self { + value.build_inner().unwrap_or_else(|_| { + panic!( + "Failed to convert {0} to {1}", + "TurboQuantizationBuilder", "TurboQuantization" + ) + }) + } +} + +impl TurboQuantizationBuilder { + /// Builds the desired type. Can often be omitted. + pub fn build(self) -> TurboQuantization { + self.build_inner().unwrap_or_else(|_| { + panic!( + "Failed to build {0} into {1}", + "TurboQuantizationBuilder", "TurboQuantization" + ) + }) + } +} + +impl TurboQuantizationBuilder { + pub(crate) fn empty() -> Self { + Self::create_empty() + } +} + +impl Default for TurboQuantizationBuilder { + fn default() -> Self { + Self::create_empty() + } +} + +#[non_exhaustive] +#[derive(Debug)] +pub enum TurboQuantizationBuilderError { + /// Uninitialized field + UninitializedField(&'static str), + /// Custom validation error + ValidationError(String), +} + +// Implementing the Display trait for better error messages +impl std::fmt::Display for TurboQuantizationBuilderError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + Self::UninitializedField(field) => { + write!(f, "`{field}` must be initialized") + } + Self::ValidationError(error) => write!(f, "{error}"), + } + } +} + +// Implementing the Error trait +impl std::error::Error for TurboQuantizationBuilderError {} + +// Implementing From trait for conversion from UninitializedFieldError +impl From for TurboQuantizationBuilderError { + fn from(error: derive_builder::UninitializedFieldError) -> Self { + Self::UninitializedField(error.field_name()) + } +} + +// Implementing From trait for conversion from String +impl From for TurboQuantizationBuilderError { + fn from(error: String) -> Self { + Self::ValidationError(error) + } +} diff --git a/src/grpc_conversions/mod.rs b/src/grpc_conversions/mod.rs index 43ba09c..8191e3a 100644 --- a/src/grpc_conversions/mod.rs +++ b/src/grpc_conversions/mod.rs @@ -8,20 +8,22 @@ use crate::qdrant::point_id::PointIdOptions; use crate::qdrant::points_selector::PointsSelectorOneOf; use crate::qdrant::value::Kind; use crate::qdrant::{ - alias_operations, condition, group_id, points_update_operation, quantization_config, - quantization_config_diff, r#match, read_consistency, shard_key, start_from, target_vector, - update_collection_cluster_setup_request, vector_example, vectors_config, vectors_config_diff, - with_payload_selector, with_vectors_selector, AbortShardTransfer, AbortShardTransferBuilder, - AliasOperations, BinaryQuantization, BinaryQuantizationBuilder, Condition, CreateAlias, - CreateShardKey, DeleteAlias, DeleteShardKey, Disabled, FieldCondition, Filter, GeoLineString, - GeoPoint, GroupId, HasIdCondition, IsEmptyCondition, IsNullCondition, ListValue, Match, - MoveShard, MoveShardBuilder, NestedCondition, PayloadExcludeSelector, PayloadIncludeSelector, - PointId, PointsIdsList, PointsSelector, PointsUpdateOperation, ProductQuantization, - ProductQuantizationBuilder, QuantizationConfig, QuantizationConfigDiff, ReadConsistency, - RenameAlias, Replica, ReplicateShard, ReplicateShardBuilder, RestartTransfer, + alias_operations, condition, create_vector_name_request, group_id, points_update_operation, + quantization_config, quantization_config_diff, r#match, read_consistency, shard_key, + start_from, target_vector, update_collection_cluster_setup_request, vector_example, + vectors_config, vectors_config_diff, with_payload_selector, with_vectors_selector, + AbortShardTransfer, AbortShardTransferBuilder, AliasOperations, BinaryQuantization, + BinaryQuantizationBuilder, Condition, CreateAlias, CreateShardKey, DeleteAlias, DeleteShardKey, + DenseVectorCreationConfig, DenseVectorCreationConfigBuilder, Disabled, FieldCondition, Filter, + GeoLineString, GeoPoint, GroupId, HasIdCondition, IsEmptyCondition, IsNullCondition, ListValue, + Match, MoveShard, MoveShardBuilder, NestedCondition, PayloadExcludeSelector, + PayloadIncludeSelector, PointId, PointsIdsList, PointsSelector, PointsUpdateOperation, + ProductQuantization, ProductQuantizationBuilder, QuantizationConfig, QuantizationConfigDiff, + ReadConsistency, RenameAlias, Replica, ReplicateShard, ReplicateShardBuilder, RestartTransfer, ScalarQuantization, ScalarQuantizationBuilder, ShardKey, ShardKeySelector, SparseIndexConfig, - SparseVectorParams, StartFrom, Struct, TargetVector, Value, Vector, VectorExample, - VectorParams, VectorParamsBuilder, VectorParamsDiff, VectorParamsDiffBuilder, + SparseVectorCreationConfig, SparseVectorCreationConfigBuilder, SparseVectorParams, StartFrom, + Struct, TargetVector, TurboQuantization, TurboQuantizationBuilder, Value, Vector, + VectorExample, VectorParams, VectorParamsBuilder, VectorParamsDiff, VectorParamsDiffBuilder, VectorParamsDiffMap, VectorParamsMap, VectorsConfig, VectorsConfigDiff, VectorsSelector, WithPayloadSelector, WithVectorsSelector, }; @@ -156,6 +158,12 @@ impl From for quantization_config::Quantization { } } +impl From for quantization_config::Quantization { + fn from(value: TurboQuantization) -> Self { + Self::Turboquant(value) + } +} + impl From for QuantizationConfigDiff { fn from(value: quantization_config_diff::Quantization) -> Self { Self { @@ -188,6 +196,36 @@ impl From for quantization_config_diff::Quantization { } } +impl From for quantization_config_diff::Quantization { + fn from(value: TurboQuantization) -> Self { + Self::Turboquant(value) + } +} + +impl From for create_vector_name_request::VectorConfig { + fn from(value: DenseVectorCreationConfig) -> Self { + Self::DenseConfig(value) + } +} + +impl From for create_vector_name_request::VectorConfig { + fn from(value: SparseVectorCreationConfig) -> Self { + Self::SparseConfig(value) + } +} + +impl From for create_vector_name_request::VectorConfig { + fn from(value: DenseVectorCreationConfigBuilder) -> Self { + Self::DenseConfig(value.build()) + } +} + +impl From for create_vector_name_request::VectorConfig { + fn from(value: SparseVectorCreationConfigBuilder) -> Self { + Self::SparseConfig(value.build()) + } +} + impl From for AliasOperations { fn from(value: alias_operations::Action) -> Self { AliasOperations { @@ -456,6 +494,12 @@ impl From for quantization_config_diff::Quantization } } +impl From for quantization_config_diff::Quantization { + fn from(value: TurboQuantizationBuilder) -> Self { + Self::Turboquant(value.build()) + } +} + impl From for quantization_config::Quantization { fn from(value: ScalarQuantizationBuilder) -> Self { Self::Scalar(value.build()) @@ -474,6 +518,12 @@ impl From for quantization_config::Quantization { } } +impl From for quantization_config::Quantization { + fn from(value: TurboQuantizationBuilder) -> Self { + Self::Turboquant(value.build()) + } +} + impl From for points_update_operation::Operation { fn from(value: points_update_operation::PointStructList) -> Self { Self::Upsert(value) diff --git a/src/qdrant.rs b/src/qdrant.rs index 5bc86f4..08e95c6 100644 --- a/src/qdrant.rs +++ b/src/qdrant.rs @@ -784,8 +784,15 @@ pub struct BinaryQuantization { pub query_encoding: ::core::option::Option, } #[derive(Clone, Copy, PartialEq, ::prost::Message)] +pub struct TurboQuantization { + #[prost(bool, optional, tag = "1")] + pub always_ram: ::core::option::Option, + #[prost(enumeration = "TurboQuantBitSize", optional, tag = "2")] + pub bits: ::core::option::Option, +} +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct QuantizationConfig { - #[prost(oneof = "quantization_config::Quantization", tags = "1, 2, 3")] + #[prost(oneof = "quantization_config::Quantization", tags = "1, 2, 3, 4")] pub quantization: ::core::option::Option, } /// Nested message and enum types in `QuantizationConfig`. @@ -798,13 +805,15 @@ pub mod quantization_config { Product(super::ProductQuantization), #[prost(message, tag = "3")] Binary(super::BinaryQuantization), + #[prost(message, tag = "4")] + Turboquant(super::TurboQuantization), } } #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct Disabled {} #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct QuantizationConfigDiff { - #[prost(oneof = "quantization_config_diff::Quantization", tags = "1, 2, 3, 4")] + #[prost(oneof = "quantization_config_diff::Quantization", tags = "1, 2, 3, 4, 5")] pub quantization: ::core::option::Option, } /// Nested message and enum types in `QuantizationConfigDiff`. @@ -819,6 +828,8 @@ pub mod quantization_config_diff { Disabled(super::Disabled), #[prost(message, tag = "4")] Binary(super::BinaryQuantization), + #[prost(message, tag = "5")] + Turboquant(super::TurboQuantization), } } #[derive(Clone, PartialEq, ::prost::Message)] @@ -850,6 +861,9 @@ pub struct StrictModeConfig { /// Max batchsize when upserting #[prost(uint64, optional, tag = "9")] pub upsert_max_batchsize: ::core::option::Option, + /// Max batchsize when searching + #[prost(uint64, optional, tag = "20")] + pub search_max_batchsize: ::core::option::Option, /// Max size of a collections vector storage in bytes, ignoring replicas. #[prost(uint64, optional, tag = "10")] pub max_collection_vector_size_bytes: ::core::option::Option, @@ -880,6 +894,10 @@ pub struct StrictModeConfig { /// Max number of payload indexes in a collection #[prost(uint64, optional, tag = "19")] pub max_payload_index_count: ::core::option::Option, + /// Reject memory-consuming update operations when process resident memory exceeds this percentage of total RAM (cgroup-aware, 1-100). + /// Delete-style operations are still allowed so memory can be freed. + #[prost(uint32, optional, tag = "21")] + pub max_resident_memory_percent: ::core::option::Option, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct StrictModeSparseConfig { @@ -1309,6 +1327,9 @@ pub struct UpdateQueueInfo { /// Number of elements in the queue #[prost(uint64, tag = "1")] pub length: u64, + /// Number of points that are deferred (i.e hidden from search as they're not yet optimized). + #[prost(uint64, optional, tag = "2")] + pub deferred_points: ::core::option::Option, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct CollectionInfo { @@ -2011,6 +2032,39 @@ impl BinaryQuantizationEncoding { } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] +pub enum TurboQuantBitSize { + Bits1 = 0, + /// 1.5 bit variant (not 15 bits) + Bits15 = 1, + Bits2 = 2, + Bits4 = 3, +} +impl TurboQuantBitSize { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Bits1 => "Bits1", + Self::Bits15 => "Bits1_5", + Self::Bits2 => "Bits2", + Self::Bits4 => "Bits4", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "Bits1" => Some(Self::Bits1), + "Bits1_5" => Some(Self::Bits15), + "Bits2" => Some(Self::Bits2), + "Bits4" => Some(Self::Bits4), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] pub enum ShardingMethod { /// Auto-sharding based on record ids Auto = 0, @@ -3971,6 +4025,87 @@ pub struct DeleteFieldIndexCollection { #[prost(uint64, optional, tag = "5")] pub timeout: ::core::option::Option, } +/// Dense vector creation parameters. +/// Only includes immutable properties that define the vector space. +/// Storage type, index, and quantization are configured separately. +#[derive(Clone, Copy, PartialEq, ::prost::Message)] +pub struct DenseVectorCreationConfig { + /// Size/dimensionality of the vectors + #[prost(uint64, tag = "1")] + pub size: u64, + /// Distance function used for comparing vectors + #[prost(enumeration = "Distance", tag = "2")] + pub distance: i32, + /// Configuration for multi-vector search (e.g., ColBERT) + #[prost(message, optional, tag = "3")] + pub multivector_config: ::core::option::Option, + /// Data type of the vectors (Float32, Float16, Uint8) + #[prost(enumeration = "Datatype", optional, tag = "4")] + pub datatype: ::core::option::Option, +} +/// Sparse vector creation parameters. +/// Only includes immutable properties that define the vector space. +#[derive(Clone, Copy, PartialEq, ::prost::Message)] +pub struct SparseVectorCreationConfig { + /// If set - apply modifier to the vector values (e.g., IDF) + #[prost(enumeration = "Modifier", optional, tag = "1")] + pub modifier: ::core::option::Option, + /// Data type used to store weights in the index + #[prost(enumeration = "Datatype", optional, tag = "2")] + pub datatype: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CreateVectorNameRequest { + /// Name of the collection + #[prost(string, tag = "1")] + pub collection_name: ::prost::alloc::string::String, + /// Wait until the changes have been applied? + #[prost(bool, optional, tag = "2")] + pub wait: ::core::option::Option, + /// Name of the new vector + #[prost(string, tag = "3")] + pub vector_name: ::prost::alloc::string::String, + /// If set, overrides global timeout setting for this request. Unit is seconds. + #[prost(uint64, optional, tag = "6")] + pub timeout: ::core::option::Option, + /// Write ordering guarantees + #[prost(message, optional, tag = "7")] + pub ordering: ::core::option::Option, + /// Configuration for the new vector - either dense or sparse + #[prost(oneof = "create_vector_name_request::VectorConfig", tags = "4, 5")] + pub vector_config: ::core::option::Option, +} +/// Nested message and enum types in `CreateVectorNameRequest`. +pub mod create_vector_name_request { + /// Configuration for the new vector - either dense or sparse + #[derive(Clone, Copy, PartialEq, ::prost::Oneof)] + pub enum VectorConfig { + /// Dense vector parameters + #[prost(message, tag = "4")] + DenseConfig(super::DenseVectorCreationConfig), + /// Sparse vector parameters + #[prost(message, tag = "5")] + SparseConfig(super::SparseVectorCreationConfig), + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteVectorNameRequest { + /// Name of the collection + #[prost(string, tag = "1")] + pub collection_name: ::prost::alloc::string::String, + /// Wait until the changes have been applied? + #[prost(bool, optional, tag = "2")] + pub wait: ::core::option::Option, + /// Name of the vector to delete + #[prost(string, tag = "3")] + pub vector_name: ::prost::alloc::string::String, + /// If set, overrides global timeout setting for this request. Unit is seconds. + #[prost(uint64, optional, tag = "4")] + pub timeout: ::core::option::Option, + /// Write ordering guarantees + #[prost(message, optional, tag = "5")] + pub ordering: ::core::option::Option, +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct PayloadIncludeSelector { /// List of payload keys to include into result @@ -4078,11 +4213,11 @@ pub struct QuantizationSearchParams { pub rescore: ::core::option::Option, /// Oversampling factor for quantization. /// - /// Defines how many extra vectors should be pre-selected using quantized index, + /// Defines how many extra vectors should be preselected using quantized index, /// and then re-scored using original vectors. /// /// For example, if `oversampling` is 2.4 and `limit` is 100, - /// then 240 vectors will be pre-selected using quantized index, + /// then 240 vectors will be preselected using quantized index, /// and then top-100 will be returned after re-scoring. #[prost(double, optional, tag = "3")] pub oversampling: ::core::option::Option, @@ -6319,6 +6454,56 @@ pub mod points_client { .insert(GrpcMethod::new("qdrant.Points", "DeleteFieldIndex")); self.inner.unary(req, path, codec).await } + /// Create a new named vector on the collection + pub async fn create_vector_name( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/qdrant.Points/CreateVectorName", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("qdrant.Points", "CreateVectorName")); + self.inner.unary(req, path, codec).await + } + /// Delete a named vector from the collection + pub async fn delete_vector_name( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/qdrant.Points/DeleteVectorName", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("qdrant.Points", "DeleteVectorName")); + self.inner.unary(req, path, codec).await + } /// Retrieve closest points based on vector similarity and given filtering /// conditions pub async fn search( @@ -6835,6 +7020,22 @@ pub mod points_server { tonic::Response, tonic::Status, >; + /// Create a new named vector on the collection + async fn create_vector_name( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + /// Delete a named vector from the collection + async fn delete_vector_name( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; /// Retrieve closest points based on vector similarity and given filtering /// conditions async fn search( @@ -7550,6 +7751,96 @@ pub mod points_server { }; Box::pin(fut) } + "/qdrant.Points/CreateVectorName" => { + #[allow(non_camel_case_types)] + struct CreateVectorNameSvc(pub Arc); + impl< + T: Points, + > tonic::server::UnaryService + for CreateVectorNameSvc { + type Response = super::PointsOperationResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::create_vector_name(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = CreateVectorNameSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/qdrant.Points/DeleteVectorName" => { + #[allow(non_camel_case_types)] + struct DeleteVectorNameSvc(pub Arc); + impl< + T: Points, + > tonic::server::UnaryService + for DeleteVectorNameSvc { + type Response = super::PointsOperationResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::delete_vector_name(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = DeleteVectorNameSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } "/qdrant.Points/Search" => { #[allow(non_camel_case_types)] struct SearchSvc(pub Arc); diff --git a/src/qdrant_client/points.rs b/src/qdrant_client/points.rs index 914fcf0..eda5a82 100644 --- a/src/qdrant_client/points.rs +++ b/src/qdrant_client/points.rs @@ -7,10 +7,11 @@ use tonic::Status; use crate::auth::MetadataInterceptor; use crate::qdrant::points_client::PointsClient; use crate::qdrant::{ - CountPoints, CountResponse, DeletePointVectors, DeletePoints, FacetCounts, FacetResponse, - GetPoints, GetResponse, PointsOperationResponse, ScrollPoints, ScrollResponse, - SearchMatrixOffsetsResponse, SearchMatrixPairsResponse, SearchMatrixPoints, UpdateBatchPoints, - UpdateBatchResponse, UpdatePointVectors, UpsertPoints, Usage, + CountPoints, CountResponse, CreateVectorNameRequest, DeletePointVectors, DeletePoints, + DeleteVectorNameRequest, FacetCounts, FacetResponse, GetPoints, GetResponse, + PointsOperationResponse, ScrollPoints, ScrollResponse, SearchMatrixOffsetsResponse, + SearchMatrixPairsResponse, SearchMatrixPoints, UpdateBatchPoints, UpdateBatchResponse, + UpdatePointVectors, UpsertPoints, Usage, }; use crate::qdrant_client::{Qdrant, QdrantResult}; @@ -507,6 +508,78 @@ impl Qdrant { .await } + /// Create a new named vector on an existing collection. + /// + /// Adds a new dense or sparse vector to the collection's vector configuration. + /// Existing points will not have data for the new vector until it is set via + /// [`update_vectors`](Self::update_vectors) or another upsert. + /// + /// ```no_run + ///# use qdrant_client::{Qdrant, QdrantError}; + /// use qdrant_client::qdrant::{ + /// CreateVectorNameRequestBuilder, DenseVectorCreationConfigBuilder, Distance, + /// }; + /// + ///# async fn create_vector_name(client: &Qdrant) + ///# -> Result<(), QdrantError> { + /// client + /// .create_vector_name( + /// CreateVectorNameRequestBuilder::new( + /// "my_collection", + /// "image", + /// DenseVectorCreationConfigBuilder::new(512, Distance::Cosine), + /// ) + /// .wait(true), + /// ) + /// .await?; + ///# Ok(()) + ///# } + /// ``` + pub async fn create_vector_name( + &self, + request: impl Into, + ) -> QdrantResult { + let request = &request.into(); + + self.with_points_client(|mut points_api| async move { + let result = points_api.create_vector_name(request.clone()).await?; + Ok(result.into_inner()) + }) + .await + } + + /// Delete a named vector from an existing collection. + /// + /// Removes the named vector and all its data across all points. This operation + /// cannot be undone. + /// + /// ```no_run + ///# use qdrant_client::{Qdrant, QdrantError}; + /// use qdrant_client::qdrant::DeleteVectorNameRequestBuilder; + /// + ///# async fn delete_vector_name(client: &Qdrant) + ///# -> Result<(), QdrantError> { + /// client + /// .delete_vector_name( + /// DeleteVectorNameRequestBuilder::new("my_collection", "image").wait(true), + /// ) + /// .await?; + ///# Ok(()) + ///# } + /// ``` + pub async fn delete_vector_name( + &self, + request: impl Into, + ) -> QdrantResult { + let request = &request.into(); + + self.with_points_client(|mut points_api| async move { + let result = points_api.delete_vector_name(request.clone()).await?; + Ok(result.into_inner()) + }) + .await + } + /// Get the amount of records for each unique value of a field. /// /// ```no_run diff --git a/tests/builder_coverage.rs b/tests/builder_coverage.rs index 7916641..9d1a052 100644 --- a/tests/builder_coverage.rs +++ b/tests/builder_coverage.rs @@ -4,9 +4,10 @@ use qdrant_client::qdrant::{ AbortShardTransferBuilder, AcornSearchParamsBuilder, BinaryQuantizationBuilder, ClearPayloadPointsBuilder, ContextInputBuilder, ContextInputPairBuilder, CountPointsBuilder, CreateAliasBuilder, CreateFieldIndexCollectionBuilder, CreateShardKeyRequestBuilder, - DeleteCollectionBuilder, DeleteFieldIndexCollectionBuilder, DeletePayloadPointsBuilder, - DeletePointVectorsBuilder, DeletePointsBuilder, DeleteShardKeyRequestBuilder, - DeleteSnapshotRequestBuilder, DiscoverBatchPointsBuilder, DiscoverInputBuilder, + CreateVectorNameRequestBuilder, DeleteCollectionBuilder, DeleteFieldIndexCollectionBuilder, + DeletePayloadPointsBuilder, DeletePointVectorsBuilder, DeletePointsBuilder, + DeleteShardKeyRequestBuilder, DeleteSnapshotRequestBuilder, DeleteVectorNameRequestBuilder, + DenseVectorCreationConfigBuilder, DiscoverBatchPointsBuilder, DiscoverInputBuilder, DiscoverPointsBuilder, Distance, FacetCountsBuilder, FieldType, GetPointsBuilder, LookupLocationBuilder, MoveShardBuilder, MultiVectorComparator, MultiVectorConfigBuilder, OrderByBuilder, ProductQuantizationBuilder, QueryBatchPointsBuilder, QueryPointGroupsBuilder, @@ -14,10 +15,10 @@ use qdrant_client::qdrant::{ RecommendPointsBuilder, RenameAliasBuilder, ReplicaBuilder, ReplicatePointsBuilder, ReplicateShardBuilder, RrfBuilder, ScrollPointsBuilder, SearchBatchPointsBuilder, SearchMatrixPointsBuilder, SearchPointGroupsBuilder, SearchPointsBuilder, - SetPayloadPointsBuilder, ShardKey, ShardKeySelectorBuilder, TextIndexParamsBuilder, - TokenizerType, UpdateBatchPointsBuilder, UpdateCollectionBuilder, - UpdateCollectionClusterSetupRequestBuilder, UpdatePointVectorsBuilder, UpsertPointsBuilder, - VectorParamsBuilder, WithLookupBuilder, + SetPayloadPointsBuilder, ShardKey, ShardKeySelectorBuilder, SparseVectorCreationConfigBuilder, + TextIndexParamsBuilder, TokenizerType, TurboQuantizationBuilder, UpdateBatchPointsBuilder, + UpdateCollectionBuilder, UpdateCollectionClusterSetupRequestBuilder, UpdatePointVectorsBuilder, + UpsertPointsBuilder, VectorParamsBuilder, WithLookupBuilder, }; /// TLDR; Ensures new fields introduced in protobuf updates won't cause a panic at runtime due to missing derive_builder attributes. @@ -36,6 +37,16 @@ fn builder_coverage() { VectorParamsBuilder::new(1, Distance::Cosine).build(); ProductQuantizationBuilder::new(1).build(); BinaryQuantizationBuilder::new(true).build(); + TurboQuantizationBuilder::new().build(); + DenseVectorCreationConfigBuilder::new(1, Distance::Cosine).build(); + CreateVectorNameRequestBuilder::new( + "my_collection", + "my_vector", + DenseVectorCreationConfigBuilder::new(1, Distance::Cosine), + ) + .build(); + SparseVectorCreationConfigBuilder::new().build(); + DeleteVectorNameRequestBuilder::new("my_collection", "my_vector").build(); SearchPointsBuilder::new("my_collection", [11.; 5], 3).build(); UpdateCollectionBuilder::new("my_collection").build(); SetPayloadPointsBuilder::new("my_collection", HashMap::default()).build(); diff --git a/tests/protos.rs b/tests/protos.rs index 0dada1b..48efc0d 100644 --- a/tests/protos.rs +++ b/tests/protos.rs @@ -20,6 +20,7 @@ fn protos() { tonic_build::configure() .configure_deprecations() + .configure_docs() .out_dir("src/") // saves generated structures at this location .compile_protos( &["proto/qdrant.proto"], // proto entry point @@ -47,9 +48,19 @@ fn protos() { /// Extension to [`Builder`] to configure builder attributes. trait BuilderExt { fn configure_deprecations(self) -> Self; + fn configure_docs(self) -> Self; } impl BuilderExt for Builder { + fn configure_docs(self) -> Self { + // prost camel-cases `Bits1_5` to `Bits15`, which reads as "15 bits". + // Disambiguate via a doc comment on the variant since we can't modify the enum variant creation. + self.field_attribute( + "TurboQuantBitSize.Bits1_5", + "#[doc = \" 1.5 bit variant (not 15 bits)\"]", + ) + } + fn configure_deprecations(self) -> Self { // Clear deprecated field for VectorOutput.data diff --git a/tests/snippet_tests/mod.rs b/tests/snippet_tests/mod.rs index 9253d8b..7dec322 100644 --- a/tests/snippet_tests/mod.rs +++ b/tests/snippet_tests/mod.rs @@ -6,11 +6,14 @@ mod test_count_points; mod test_create_collection; mod test_create_collection_with_bq; mod test_create_collection_with_metadata; +mod test_create_collection_with_turbo_quantization; mod test_create_field_index; mod test_create_full_snapshot; mod test_create_shard_key; mod test_create_snapshot; mod test_create_text_index; +mod test_create_vector_name_dense; +mod test_create_vector_name_sparse; mod test_delete_collection; mod test_delete_field_index; mod test_delete_full_snapshot; @@ -18,6 +21,7 @@ mod test_delete_payload; mod test_delete_points; mod test_delete_shard_key; mod test_delete_snapshot; +mod test_delete_vector_name; mod test_delete_vectors; mod test_discover_batch_points; mod test_discover_points; diff --git a/tests/snippet_tests/test_create_collection_with_turbo_quantization.rs b/tests/snippet_tests/test_create_collection_with_turbo_quantization.rs new file mode 100644 index 0000000..2580dc2 --- /dev/null +++ b/tests/snippet_tests/test_create_collection_with_turbo_quantization.rs @@ -0,0 +1,27 @@ + +#[tokio::test] +async fn test_create_collection_with_turbo_quantization() { + async fn create_collection_with_turbo_quantization() -> Result<(), Box> { + use qdrant_client::qdrant::{ + CreateCollectionBuilder, Distance, TurboQuantBitSize, TurboQuantizationBuilder, + VectorParamsBuilder, + }; + use qdrant_client::Qdrant; + + let client = Qdrant::from_url("http://localhost:6334").build()?; + + client + .create_collection( + CreateCollectionBuilder::new("{collection_name}") + .vectors_config(VectorParamsBuilder::new(1536, Distance::Cosine)) + .quantization_config( + TurboQuantizationBuilder::new() + .bits(TurboQuantBitSize::Bits2) + .always_ram(true), + ), + ) + .await?; + Ok(()) + } + let _ = create_collection_with_turbo_quantization().await; +} diff --git a/tests/snippet_tests/test_create_vector_name_dense.rs b/tests/snippet_tests/test_create_vector_name_dense.rs new file mode 100644 index 0000000..15e4d5c --- /dev/null +++ b/tests/snippet_tests/test_create_vector_name_dense.rs @@ -0,0 +1,25 @@ + +#[tokio::test] +async fn test_create_vector_name_dense() { + async fn create_vector_name_dense() -> Result<(), Box> { + use qdrant_client::qdrant::{ + CreateVectorNameRequestBuilder, DenseVectorCreationConfigBuilder, Distance, + }; + use qdrant_client::Qdrant; + + let client = Qdrant::from_url("http://localhost:6334").build()?; + + client + .create_vector_name( + CreateVectorNameRequestBuilder::new( + "{collection_name}", + "image", + DenseVectorCreationConfigBuilder::new(512, Distance::Cosine), + ) + .wait(true), + ) + .await?; + Ok(()) + } + let _ = create_vector_name_dense().await; +} diff --git a/tests/snippet_tests/test_create_vector_name_sparse.rs b/tests/snippet_tests/test_create_vector_name_sparse.rs new file mode 100644 index 0000000..a49a63c --- /dev/null +++ b/tests/snippet_tests/test_create_vector_name_sparse.rs @@ -0,0 +1,25 @@ + +#[tokio::test] +async fn test_create_vector_name_sparse() { + async fn create_vector_name_sparse() -> Result<(), Box> { + use qdrant_client::qdrant::{ + CreateVectorNameRequestBuilder, Modifier, SparseVectorCreationConfigBuilder, + }; + use qdrant_client::Qdrant; + + let client = Qdrant::from_url("http://localhost:6334").build()?; + + client + .create_vector_name( + CreateVectorNameRequestBuilder::new( + "{collection_name}", + "text_sparse", + SparseVectorCreationConfigBuilder::new().modifier(Modifier::Idf), + ) + .wait(true), + ) + .await?; + Ok(()) + } + let _ = create_vector_name_sparse().await; +} diff --git a/tests/snippet_tests/test_delete_vector_name.rs b/tests/snippet_tests/test_delete_vector_name.rs new file mode 100644 index 0000000..8a0e4df --- /dev/null +++ b/tests/snippet_tests/test_delete_vector_name.rs @@ -0,0 +1,18 @@ + +#[tokio::test] +async fn test_delete_vector_name() { + async fn delete_vector_name() -> Result<(), Box> { + use qdrant_client::qdrant::DeleteVectorNameRequestBuilder; + use qdrant_client::Qdrant; + + let client = Qdrant::from_url("http://localhost:6334").build()?; + + client + .delete_vector_name( + DeleteVectorNameRequestBuilder::new("{collection_name}", "image").wait(true), + ) + .await?; + Ok(()) + } + let _ = delete_vector_name().await; +} diff --git a/tools/sync_proto.sh b/tools/sync_proto.sh index b579ddf..d51455c 100755 --- a/tools/sync_proto.sh +++ b/tools/sync_proto.sh @@ -28,6 +28,7 @@ rm $CLIENT_DIR/raft_service.proto rm $CLIENT_DIR/shard_snapshots_service.proto rm $CLIENT_DIR/health_check.proto rm $CLIENT_DIR/telemetry_internal.proto +rm $CLIENT_DIR/storage_read_service.proto cat $CLIENT_DIR/qdrant.proto \ | grep -v 'collections_internal_service.proto' \ @@ -37,6 +38,7 @@ cat $CLIENT_DIR/qdrant.proto \ | grep -v 'shard_snapshots_service.proto' \ | grep -v 'health_check.proto' \ | grep -v 'telemetry_internal.proto' \ + | grep -v 'storage_read_service.proto' \ > $CLIENT_DIR/qdrant_tmp.proto mv $CLIENT_DIR/qdrant_tmp.proto $CLIENT_DIR/qdrant.proto