From 4e04815dda6a392ee12402e0af5b1de612ffcbc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Nam=20Long?= <26674308+J2TeamNNL@users.noreply.github.com> Date: Sun, 1 Nov 2026 08:54:46 +0700 Subject: [PATCH 01/20] feat(plugin-qdrant): add a Qdrant driver over the REST API --- .github/plugin-registry.json | 3 + apple/Packages/TableProCore/Package.swift | 13 + .../SQLDDLFallbackPolicy.swift | 1 + .../TableProCoreTypes/DatabaseType.swift | 4 +- .../QdrantConnectionSettings.swift | 59 ++++ .../TableProQdrantCore/QdrantConsole.swift | 111 +++++++ .../TableProQdrantCore/QdrantError.swift | 54 ++++ .../QdrantFilterBuilder.swift | 246 +++++++++++++++ .../TableProQdrantCore/QdrantJSON.swift | 71 +++++ .../TableProQdrantCore/QdrantPoint.swift | 118 +++++++ .../TableProQdrantCore/QdrantQuery.swift | 294 ++++++++++++++++++ .../TableProQdrantCore/QdrantSchema.swift | 239 ++++++++++++++ .../Wire/QdrantClient.swift | 167 ++++++++++ .../Wire/QdrantScrollPager.swift | 111 +++++++ .../Wire/QdrantTransport.swift | 149 +++++++++ .../SQLLexicalProfile.swift | 1 + .../DatabaseTypeTests.swift | 10 +- .../QdrantClientTests.swift | 77 +++++ .../QdrantFilterBuilderTests.swift | 94 ++++++ .../QdrantSchemaAndWriteTests.swift | 198 ++++++++++++ .../QdrantTestSupport.swift | 73 +++++ apple/Plugins/QdrantDriverPlugin/Info.plist | 12 + .../QdrantDriverPlugin/QdrantPlugin.swift | 111 +++++++ .../QdrantPluginDriver+Execution.swift | 172 ++++++++++ .../QdrantPluginDriver+Metadata.swift | 209 +++++++++++++ .../QdrantPluginDriver.swift | 120 +++++++ .../qdrant-icon.imageset/Contents.json | 16 + .../qdrant-icon.imageset/qdrant.svg | 1 + ...luginMetadataRegistry+QdrantDefaults.swift | 120 +++++++ ...ginMetadataRegistry+RegistryDefaults.swift | 2 +- .../Execution/ExternalStatementGate.swift | 1 + .../SQL/CatalogChangeClassifier.swift | 2 +- .../Core/Utilities/SQL/QueryClassifier.swift | 40 +++ .../Models/Connection/DatabaseType.swift | 1 + .../TablePro/Resources/Localizable.xcstrings | 56 ++++ ...rnalStatementGatePrivateRequestTests.swift | 2 + ...PluginMetadataRegistryTypeCountTests.swift | 2 +- .../Plugins/PortForwardCapabilityTests.swift | 2 +- .../Query/AllTablesListingTests.swift | 2 +- .../SQL/QueryClassifierHardeningTests.swift | 27 ++ .../Models/Query/QueryTabBaseQueryTests.swift | 2 +- .../Main/QueryParameterDetectionTests.swift | 2 +- .../Views/SQLReviewSheetTests.swift | 2 +- apple/project.yml | 10 + apple/scripts/probes/check-qdrant.sh | 42 +++ apple/scripts/probes/qdrant-probe.swift | 113 +++++++ 46 files changed, 3152 insertions(+), 10 deletions(-) create mode 100644 apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConnectionSettings.swift create mode 100644 apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConsole.swift create mode 100644 apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantError.swift create mode 100644 apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift create mode 100644 apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantJSON.swift create mode 100644 apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantPoint.swift create mode 100644 apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantQuery.swift create mode 100644 apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift create mode 100644 apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantClient.swift create mode 100644 apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift create mode 100644 apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantTransport.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaAndWriteTests.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantTestSupport.swift create mode 100644 apple/Plugins/QdrantDriverPlugin/Info.plist create mode 100644 apple/Plugins/QdrantDriverPlugin/QdrantPlugin.swift create mode 100644 apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift create mode 100644 apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift create mode 100644 apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift create mode 100644 apple/TablePro/Assets.xcassets/qdrant-icon.imageset/Contents.json create mode 100644 apple/TablePro/Assets.xcassets/qdrant-icon.imageset/qdrant.svg create mode 100644 apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift create mode 100755 apple/scripts/probes/check-qdrant.sh create mode 100644 apple/scripts/probes/qdrant-probe.swift diff --git a/.github/plugin-registry.json b/.github/plugin-registry.json index f6bf29a547..67b26128fc 100644 --- a/.github/plugin-registry.json +++ b/.github/plugin-registry.json @@ -61,6 +61,9 @@ "ParquetExport": { "name": "Parquet Export" }, + "QdrantDriverPlugin": { + "name": "Qdrant Driver" + }, "SnowflakeDriverPlugin": { "name": "Snowflake Driver" }, diff --git a/apple/Packages/TableProCore/Package.swift b/apple/Packages/TableProCore/Package.swift index 94fe92c270..19797c7aa6 100644 --- a/apple/Packages/TableProCore/Package.swift +++ b/apple/Packages/TableProCore/Package.swift @@ -37,6 +37,7 @@ let package = Package( .library(name: "TableProGoogleCloud", targets: ["TableProGoogleCloud"]), .library(name: "TableProSpannerCore", targets: ["TableProSpannerCore"]), .library(name: "TableProWeaviateCore", targets: ["TableProWeaviateCore"]), + .library(name: "TableProQdrantCore", targets: ["TableProQdrantCore"]), .library(name: "TableProFirebaseCore", targets: ["TableProFirebaseCore"]), .library(name: "TableProBigtableCore", targets: ["TableProBigtableCore"]), .library(name: "TableProNumberFormatting", targets: ["TableProNumberFormatting"]), @@ -192,6 +193,12 @@ let package = Package( dependencies: [], path: "Sources/TableProWeaviateCore" ), + .target( + name: "TableProQdrantCore", + dependencies: [], + path: "Sources/TableProQdrantCore", + swiftSettings: pluginCoreSettings + ), .target( name: "TableProBigtableCore", dependencies: ["TableProGoogleCloud"], @@ -295,6 +302,12 @@ let package = Package( dependencies: ["TableProTeradataCore"], path: "Tests/TableProTeradataCoreTests" ), + .testTarget( + name: "TableProQdrantCoreTests", + dependencies: ["TableProQdrantCore"], + path: "Tests/TableProQdrantCoreTests", + swiftSettings: pluginCoreSettings + ), .testTarget( name: "TableProTrinoCoreTests", dependencies: ["TableProTrinoCore", "TableProSQLGrammar"], diff --git a/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift b/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift index d6962fdd63..a8ea9dba11 100644 --- a/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift +++ b/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift @@ -36,6 +36,7 @@ public enum SQLDDLFallbackPolicy { "Elasticsearch", "Typesense", "Weaviate", + "Qdrant", "Kafka", "etcd", "Redis", diff --git a/apple/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift b/apple/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift index bb002a60ef..eb69ef68e6 100644 --- a/apple/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift +++ b/apple/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift @@ -42,6 +42,7 @@ public struct DatabaseType: Hashable, Codable, Sendable, RawRepresentable { public static let kafka = DatabaseType(rawValue: "Kafka") public static let cloudflareR2SQL = DatabaseType(rawValue: "Cloudflare R2 SQL") public static let weaviate = DatabaseType(rawValue: "Weaviate") + public static let qdrant = DatabaseType(rawValue: "Qdrant") public static let firestore = DatabaseType(rawValue: "Firestore") public static let firebaseRealtimeDatabase = DatabaseType(rawValue: "Firebase Realtime Database") public static let bigtable = DatabaseType(rawValue: "Bigtable") @@ -50,7 +51,7 @@ public struct DatabaseType: Hashable, Codable, Sendable, RawRepresentable { .mysql, .mariadb, .tidb, .databend, .oceanbase, .postgresql, .sqlite, .redis, .mongodb, .clickhouse, .mssql, .oracle, .dameng, .duckdb, .cassandra, .redshift, .etcd, .cloudflareD1, .dynamodb, .bigquery, .spanner, .snowflake, .libsql, .beancount, - .surrealdb, .teradata, .trino, .kafka, .cloudflareR2SQL, .weaviate, .firestore, .firebaseRealtimeDatabase, .bigtable + .surrealdb, .teradata, .trino, .kafka, .cloudflareR2SQL, .weaviate, .qdrant, .firestore, .firebaseRealtimeDatabase, .bigtable ] /// Icon name for this database type — asset catalog name (e.g. "mysql-icon") or SF Symbol fallback @@ -87,6 +88,7 @@ public struct DatabaseType: Hashable, Codable, Sendable, RawRepresentable { case .kafka: return "kafka-icon" case .cloudflareR2SQL: return "cloudflare-r2-sql-icon" case .weaviate: return "weaviate-icon" + case .qdrant: return "qdrant-icon" case .firestore, .firebaseRealtimeDatabase: return "firebase-icon" case .bigtable: return "bigtable-icon" default: return "externaldrive" diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConnectionSettings.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConnectionSettings.swift new file mode 100644 index 0000000000..97a914a012 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConnectionSettings.swift @@ -0,0 +1,59 @@ +import Foundation + +public enum QdrantFieldID { + public static let apiKey = "qdApiKey" + public static let skipTLSVerify = "qdSkipTLSVerify" +} + +public struct QdrantConnectionSettings: Sendable, Equatable { + public static let defaultPort = 6_333 + + public let host: String + public let port: Int + public let usesTLS: Bool + public let apiKey: String + public let skipTLSVerify: Bool + + public init(host: String, port: Int, usesTLS: Bool, apiKey: String, skipTLSVerify: Bool) { + self.host = host.trimmingCharacters(in: .whitespacesAndNewlines) + self.port = port + self.usesTLS = usesTLS + self.apiKey = apiKey.trimmingCharacters(in: .whitespacesAndNewlines) + self.skipTLSVerify = skipTLSVerify + } + + public static func parse( + host: String, + port: Int, + usesTLS: Bool, + fields: [String: String] + ) throws -> QdrantConnectionSettings { + let resolvedHost = host.trimmingCharacters(in: .whitespacesAndNewlines) + let settings = QdrantConnectionSettings( + host: resolvedHost.isEmpty ? "localhost" : resolvedHost, + port: port > 0 ? port : defaultPort, + usesTLS: usesTLS, + apiKey: fields[QdrantFieldID.apiKey] ?? "", + skipTLSVerify: fields[QdrantFieldID.skipTLSVerify] == "true" + ) + _ = try settings.baseURL() + return settings + } + + /// Qdrant reads the key from `api-key`, and also accepts it as a bearer token; the dedicated + /// header is the one both a self-hosted node and Qdrant Cloud document. + public var headers: [String: String] { + apiKey.isEmpty ? [:] : ["api-key": apiKey] + } + + public func baseURL() throws -> URL { + var components = URLComponents() + components.scheme = usesTLS ? "https" : "http" + components.host = host + components.port = port + guard let url = components.url else { + throw QdrantError.configuration(String(localized: "The host is not valid in a URL.")) + } + return url + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConsole.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConsole.swift new file mode 100644 index 0000000000..0ceee6f41e --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConsole.swift @@ -0,0 +1,111 @@ +import Foundation + +public struct QdrantConsoleRequest: Sendable, Equatable { + public let method: String + public let path: String + public let body: String? + + public init(method: String, path: String, body: String?) { + self.method = method + self.path = path + self.body = body + } + + /// The POST endpoints that only read. Retrieving points by id is a POST to `/points` itself. + static let readSuffixes: [[String]] = [ + ["points"], ["points", "scroll"], ["points", "search"], ["points", "query"], ["points", "count"], + ["points", "recommend"], ["points", "discover"], ["points", "facet"], + ["points", "search", "batch"], ["points", "query", "batch"], ["points", "recommend", "batch"], + ["points", "discover", "batch"], ["points", "search", "groups"], ["points", "query", "groups"], + ["points", "recommend", "groups"], ["points", "search", "matrix", "pairs"], + ["points", "search", "matrix", "offsets"] + ] + + public var isRead: Bool { + if method == "GET" || method == "HEAD" { return true } + guard method == "POST" else { return false } + let route = path.split(separator: "?", maxSplits: 1).first.map(String.init) ?? path + let segments = route.split(separator: "/").map { $0.lowercased() } + guard segments.count >= 3, segments[0] == "collections" else { return false } + return Self.readSuffixes.contains { Array(segments.dropFirst(2)) == $0 } + } +} + +/// One REST request typed in the editor: `METHOD /path`, then an optional JSON body on the same +/// line or on the lines below it. +public enum QdrantConsoleParser { + static let methods: Set = ["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD"] + + public static func parse(_ input: String) -> QdrantConsoleRequest? { + let trimmed = input.trimmingCharacters(in: .whitespacesAndNewlines) + let lines = trimmed.split(separator: "\n", maxSplits: 1, omittingEmptySubsequences: false) + guard let header = lines.first else { return nil } + let following = lines.count > 1 + ? String(lines[1]).trimmingCharacters(in: .whitespacesAndNewlines) + : "" + + let parts = header.split(maxSplits: 2, omittingEmptySubsequences: true, whereSeparator: \.isWhitespace) + guard parts.count >= 2 else { return nil } + let method = String(parts[0]).uppercased() + guard methods.contains(method) else { return nil } + let path = String(parts[1]) + guard path.hasPrefix("/"), !path.hasPrefix("//") else { return nil } + let inline = parts.count > 2 ? String(parts[2]).trimmingCharacters(in: .whitespacesAndNewlines) : "" + let body = inline.isEmpty ? following : inline + return QdrantConsoleRequest(method: method, path: path, body: body.isEmpty ? nil : body) + } +} + +public enum QdrantPathEncoding { + private static let allowed: CharacterSet = { + var set = CharacterSet.alphanumerics + set.insert(charactersIn: "-_.") + return set + }() + + public static func segment(_ value: String) -> String { + value.addingPercentEncoding(withAllowedCharacters: allowed) ?? value + } + + /// The path keeps its own query string (`?wait=true`): assigning it whole to `URLComponents.path` + /// would encode the `?` and send the request to a path that does not exist. + public static func resolve(_ path: String, against base: URL) -> URL? { + guard path.hasPrefix("/"), !path.hasPrefix("//"), !path.contains("://"), + var components = URLComponents(url: base, resolvingAgainstBaseURL: false), + let requested = URLComponents(string: path) + else { return nil } + components.percentEncodedPath = requested.percentEncodedPath + components.percentEncodedQuery = requested.percentEncodedQuery + return components.url + } +} + +public enum QdrantOperations { + public static let exportTag = "QDRANT_EXPORT:" + + /// The console request that removes one collection. Plain request text, so the confirmation shows + /// it verbatim and the query classifier tiers it destructive by its verb. + public static func deleteCollection(named name: String, objectType: String) -> String? { + guard objectType.uppercased() == "TABLE", !name.isEmpty else { return nil } + return "DELETE /collections/\(QdrantPathEncoding.segment(name))" + } + + /// An empty filter matches every point, which keeps the collection and its config. + public static func deleteAllPoints(collection: String) -> String? { + guard !collection.isEmpty else { return nil } + return "POST /collections/\(QdrantPathEncoding.segment(collection))/points/delete?wait=true\n{\"filter\":{}}" + } + + public static func encodeExport(collection: String) -> String { + exportTag + Data(collection.utf8).base64EncodedString() + } + + public static func decodeExport(_ query: String) -> String? { + guard query.hasPrefix(exportTag), + let data = Data(base64Encoded: String(query.dropFirst(exportTag.count))), + let collection = String(data: data, encoding: .utf8), + !collection.isEmpty + else { return nil } + return collection + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantError.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantError.swift new file mode 100644 index 0000000000..e97312d379 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantError.swift @@ -0,0 +1,54 @@ +import Foundation + +public enum QdrantError: Error, LocalizedError, Equatable, Sendable { + case configuration(String) + case notConnected + case transport(String) + case authentication(String) + case api(status: Int, message: String) + case malformedResponse(String) + case cancelled + + public var errorDescription: String? { + switch self { + case .configuration(let detail), .transport(let detail), .malformedResponse(let detail), + .authentication(let detail): + return detail + case .notConnected: + return String(localized: "Not connected to Qdrant.") + case .api(_, let message): + return message + case .cancelled: + return String(localized: "The request was cancelled.") + } + } + + public static func from(status: Int, body: Data) -> QdrantError { + let message = apiMessage(from: body) + ?? String(format: String(localized: "Qdrant returned HTTP %d."), status) + if status == 401 || status == 403 { + return .authentication(message) + } + return .api(status: status, message: message) + } + + /// Qdrant puts the reason in `status.error`; a proxy in front of it may answer with plain text. + public static func apiMessage(from body: Data) -> String? { + guard let json = try? JSONSerialization.jsonObject(with: body) else { + let text = String(data: body, encoding: .utf8)?.trimmingCharacters(in: .whitespacesAndNewlines) + return (text?.isEmpty ?? true) ? nil : text + } + guard let object = json as? [String: Any] else { return nil } + if let status = object["status"] as? [String: Any], + let error = status["error"] as? String, !error.isEmpty { + return error + } + if let error = object["error"] as? String, !error.isEmpty { + return error + } + if let message = object["message"] as? String, !message.isEmpty { + return message + } + return nil + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift new file mode 100644 index 0000000000..7fa0dc57d8 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift @@ -0,0 +1,246 @@ +import Foundation + +public struct QdrantFilterSpec: Sendable, Equatable { + public let column: String + public let op: String + public let value: String + public let secondValue: String? + + public init(column: String, op: String, value: String, secondValue: String? = nil) { + self.column = column + self.op = op + self.value = value + self.secondValue = secondValue + } +} + +public enum QdrantFilterError: Error, LocalizedError, Equatable { + case unsupportedOperator(String) + case missingUpperBound(column: String) + case notANumber(column: String, value: String) + case notABoolean(column: String, value: String) + case notAPointID(value: String) + case emptyList(column: String) + case comparisonNeedsNumberOrDate(column: String, op: String) + case textMatchNeedsTextIndex(column: String, op: String) + case vectorNotFilterable(column: String) + case rawFilterNotAnObject + + public var errorDescription: String? { + switch self { + case .unsupportedOperator(let op): + return String(format: String(localized: "Qdrant cannot filter with %@."), op) + case .missingUpperBound(let column): + return String(format: String(localized: "BETWEEN on %@ needs an upper bound."), column) + case .notANumber(let column, let value): + return String(format: String(localized: "%@ holds numbers, and %@ is not a number."), column, value) + case .notABoolean(let column, let value): + return String( + format: String(localized: "%@ holds booleans, so it matches only true or false, not %@."), + column, value + ) + case .notAPointID(let value): + return String( + format: String(localized: "%@ is not a point id. Qdrant ids are unsigned integers or UUIDs."), + value + ) + case .emptyList(let column): + return String(format: String(localized: "IN on %@ needs at least one value."), column) + case .comparisonNeedsNumberOrDate(let column, let op): + return String( + format: String(localized: "Qdrant compares only numbers and dates with %@, and %@ holds neither."), + op, column + ) + case .textMatchNeedsTextIndex(let column, let op): + return String( + format: String(localized: "%@ on %@ needs a full-text payload index on that field. Without one Qdrant matches nothing."), + op, column + ) + case .vectorNotFilterable(let column): + return String(format: String(localized: "Qdrant cannot filter on %@."), column) + case .rawFilterNotAnObject: + return String(localized: "A raw filter for Qdrant is a JSON condition, such as {\"key\": \"city\", \"match\": {\"value\": \"Berlin\"}}.") + } + } +} + +/// Turns the filter bar's rows into Qdrant's filter JSON. AND rows go under `must` and OR rows under +/// `should`; a negated row is a nested filter with `must_not`, which Qdrant accepts as a condition. +public enum QdrantFilterBuilder { + public static let rawFilterColumn = "__RAW__" + + public static func filter( + _ filters: [QdrantFilterSpec], + logicMode: String, + collection: QdrantCollection + ) throws -> [String: Any]? { + guard !filters.isEmpty else { return nil } + let conditions = try filters.map { try condition(for: $0, collection: collection) } + let clause = logicMode.uppercased() == "OR" ? "should" : "must" + return [clause: conditions] + } + + public static func condition(for filter: QdrantFilterSpec, collection: QdrantCollection) throws -> [String: Any] { + if filter.column == rawFilterColumn { + return try raw(filter.value) + } + guard !QdrantSchema.isVectorColumn(filter.column) else { + throw QdrantFilterError.vectorNotFilterable(column: filter.column) + } + let op = filter.op.uppercased() + if filter.column == QdrantSchema.idColumn { + return try idCondition(op: op, filter: filter) + } + let type = collection.fieldType(of: filter.column) ?? .json + let column = QdrantSchema.payloadKey(forColumn: filter.column) + + switch op { + case "=": + return try equality(column: column, value: filter.value, type: type) + case "!=", "<>": + return negated(try equality(column: column, value: filter.value, type: type)) + case ">", ">=", "<", "<=": + return try range(column: column, type: type, op: op, bounds: [rangeKey(op): filter.value]) + case "BETWEEN": + guard let upper = filter.secondValue, !upper.trimmingCharacters(in: .whitespaces).isEmpty else { + throw QdrantFilterError.missingUpperBound(column: column) + } + let lower = lowerBound(of: filter.value, upperBound: upper) + return try range(column: column, type: type, op: op, bounds: ["gte": lower, "lte": upper]) + case "IN": + return ["key": column, "match": ["any": try list(filter.value, column: column, type: type)]] + case "NOT IN": + return ["key": column, "match": ["except": try list(filter.value, column: column, type: type)]] + case "CONTAINS": + return try textMatch(column: column, value: filter.value, type: type, op: op) + case "NOT CONTAINS": + return negated(try textMatch(column: column, value: filter.value, type: type, op: op)) + case "IS NULL": + return ["is_null": ["key": column]] + case "IS NOT NULL": + return negated(["is_null": ["key": column]]) + case "IS EMPTY": + return ["is_empty": ["key": column]] + case "IS NOT EMPTY": + return negated(["is_empty": ["key": column]]) + default: + throw QdrantFilterError.unsupportedOperator(op) + } + } + + private static func raw(_ text: String) throws -> [String: Any] { + guard let data = text.data(using: .utf8), + let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any] + else { throw QdrantFilterError.rawFilterNotAnObject } + return object + } + + private static func idCondition(op: String, filter: QdrantFilterSpec) throws -> [String: Any] { + func ids(_ text: String) throws -> [Any] { + let parts = text.split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty } + guard !parts.isEmpty else { throw QdrantFilterError.emptyList(column: QdrantSchema.idColumn) } + return try parts.map { part in + guard let id = QdrantPointID.json(from: part) else { throw QdrantFilterError.notAPointID(value: part) } + return id + } + } + switch op { + case "=", "IN": + return ["has_id": try ids(filter.value)] + case "!=", "<>", "NOT IN": + return negated(["has_id": try ids(filter.value)]) + default: + throw QdrantFilterError.unsupportedOperator(op) + } + } + + /// `match.value` takes a string, an integer or a boolean. A float has no exact match in Qdrant, + /// so equality on one is the closed range around it. + private static func equality(column: String, value: String, type: QdrantFieldType) throws -> [String: Any] { + switch type { + case .float, .datetime: + return try range(column: column, type: type, op: "=", bounds: ["gte": value, "lte": value]) + default: + return ["key": column, "match": ["value": try scalar(value, column: column, type: type)]] + } + } + + private static func range( + column: String, + type: QdrantFieldType, + op: String, + bounds: [String: String] + ) throws -> [String: Any] { + guard type.isOrdered || type == .json else { + throw QdrantFilterError.comparisonNeedsNumberOrDate(column: column, op: op) + } + var range: [String: Any] = [:] + for (key, text) in bounds { + let trimmed = text.trimmingCharacters(in: .whitespaces) + if type == .datetime { + range[key] = trimmed + } else if let number = Double(trimmed) { + range[key] = number + } else if type == .json { + range[key] = trimmed + } else { + throw QdrantFilterError.notANumber(column: column, value: text) + } + } + return ["key": column, "range": range] + } + + private static func textMatch(column: String, value: String, type: QdrantFieldType, op: String) throws -> [String: Any] { + guard type == .text else { + throw QdrantFilterError.textMatchNeedsTextIndex(column: column, op: op) + } + return ["key": column, "match": ["text": value]] + } + + private static func list(_ value: String, column: String, type: QdrantFieldType) throws -> [Any] { + let parts = value.split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty } + guard !parts.isEmpty else { throw QdrantFilterError.emptyList(column: column) } + return try parts.map { try scalar($0, column: column, type: type) } + } + + private static func scalar(_ value: String, column: String, type: QdrantFieldType) throws -> Any { + let trimmed = value.trimmingCharacters(in: .whitespaces) + switch type { + case .integer: + guard let number = Int64(trimmed) else { throw QdrantFilterError.notANumber(column: column, value: value) } + return NSNumber(value: number) + case .bool: + switch trimmed.lowercased() { + case "true", "1": return true + case "false", "0": return false + default: throw QdrantFilterError.notABoolean(column: column, value: value) + } + case .json: + if let number = Int64(trimmed) { return NSNumber(value: number) } + if trimmed == "true" { return true } + if trimmed == "false" { return false } + return value + default: + return value + } + } + + private static func rangeKey(_ op: String) -> String { + switch op { + case ">": return "gt" + case ">=": return "gte" + case "<": return "lt" + default: return "lte" + } + } + + private static func lowerBound(of joinedValue: String, upperBound: String) -> String { + let suffix = "," + upperBound + guard joinedValue.hasSuffix(suffix) else { return joinedValue } + return String(joinedValue.dropLast(suffix.count)) + } + + private static func negated(_ condition: [String: Any]) -> [String: Any] { + ["must_not": [condition]] + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantJSON.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantJSON.swift new file mode 100644 index 0000000000..bee609b383 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantJSON.swift @@ -0,0 +1,71 @@ +import Foundation + +public enum QdrantJSON { + public static func dictionary(_ value: Any?) -> [String: Any]? { + value as? [String: Any] + } + + public static func data(_ object: Any, pretty: Bool = false) throws -> Data { + guard JSONSerialization.isValidJSONObject(object) else { + throw QdrantError.malformedResponse(String(localized: "Request body is not valid JSON.")) + } + var options: JSONSerialization.WritingOptions = [.sortedKeys, .withoutEscapingSlashes] + if pretty { + options.insert(.prettyPrinted) + } + return try JSONSerialization.data(withJSONObject: object, options: options) + } + + public static func text(_ object: Any, pretty: Bool = false) throws -> String { + String(data: try data(object, pretty: pretty), encoding: .utf8) ?? "{}" + } + + public static func isBoolean(_ number: NSNumber) -> Bool { + CFGetTypeID(number) == CFBooleanGetTypeID() + } + + public static func displayText(_ value: Any?) -> String? { + switch value { + case nil, is NSNull: + return nil + case let text as String: + return text + case let number as NSNumber: + if isBoolean(number) { + return number.boolValue ? "true" : "false" + } + return number.stringValue + case let object as [String: Any]: + return try? text(object) + case let array as [Any]: + return try? text(array) + default: + return String(describing: value as Any) + } + } + + /// A payload is schemaless, so a cell is written back as the type its column holds. Parsing every + /// cell as JSON would turn a string field holding `42` into a number and break a `match` on it. + public static func parsedValue(_ text: String, fieldType: QdrantFieldType) -> Any { + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + switch fieldType { + case .keyword, .text, .string, .uuid, .datetime: + return text + case .integer: + return Int64(trimmed) ?? Double(trimmed).map { $0 as Any } ?? text + case .float: + return Double(trimmed) ?? text + case .bool: + switch trimmed.lowercased() { + case "true": return true + case "false": return false + default: return text + } + case .geo, .object, .array, .json: + guard let data = trimmed.data(using: .utf8), + let parsed = try? JSONSerialization.jsonObject(with: data, options: [.fragmentsAllowed]) + else { return text } + return parsed + } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantPoint.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantPoint.swift new file mode 100644 index 0000000000..c3846c1466 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantPoint.swift @@ -0,0 +1,118 @@ +import Foundation + +/// A point id is an unsigned integer or a UUID, and Qdrant refuses one in the other's form, so the +/// grid's text is turned back into the kind it came from. +public enum QdrantPointID { + public static func json(from text: String) -> Any? { + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return nil } + if trimmed.allSatisfy(\.isASCII), trimmed.allSatisfy(\.isNumber), let number = UInt64(trimmed) { + return NSNumber(value: number) + } + guard UUID(uuidString: trimmed) != nil else { return nil } + return trimmed.lowercased() + } + + public static func text(_ value: Any?) -> String? { + QdrantJSON.displayText(value) + } +} + +public struct QdrantPoint: @unchecked Sendable { + public let id: Any? + public let payload: [String: Any] + /// `nil` when the request left vectors out; an array for an unnamed vector; a map by name otherwise. + public let vector: Any? + public let score: Double? + + public init(id: Any?, payload: [String: Any], vector: Any?, score: Double?) { + self.id = id + self.payload = payload + self.vector = vector + self.score = score + } + + public var idText: String? { QdrantPointID.text(id) } + + public var payloadKeys: [String] { payload.keys.sorted() } + + public static func parse(_ json: Any) -> QdrantPoint? { + guard let object = json as? [String: Any], object.keys.contains("id") else { return nil } + let vector = object["vector"] ?? object["vectors"] + return QdrantPoint( + id: object["id"], + payload: object["payload"] as? [String: Any] ?? [:], + vector: vector is NSNull ? nil : vector, + score: (object["score"] as? NSNumber)?.doubleValue + ) + } + + /// The point list of any read endpoint: `scroll` and `query` wrap it in `points`, retrieve and the + /// older `search` answer the array itself. `nil` when the result is not a list of points. + public static func list(fromResponse json: Any?) -> [QdrantPoint]? { + guard let result = (json as? [String: Any])?["result"] else { return nil } + let items: [Any] + if let wrapped = (result as? [String: Any])?["points"] as? [Any] { + items = wrapped + } else if let array = result as? [Any] { + items = array + } else { + return nil + } + let points = items.compactMap(parse) + return points.count == items.count ? points : nil + } + + public func vectorValue(for column: String) -> Any? { + guard let vector else { return nil } + if column == QdrantSchema.vectorColumn { + if vector is [Any] { return vector } + return (vector as? [String: Any])?[""] + } + let name = String(column.dropFirst(QdrantSchema.vectorColumn.count + 1)) + return (vector as? [String: Any])?[name] + } + + public func cell(for column: String) -> String? { + if column == QdrantSchema.idColumn { + return idText + } + if QdrantSchema.isVectorColumn(column) { + return QdrantJSON.displayText(vectorValue(for: column)) + } + if column == QdrantSchema.scoreColumn { + return score.map { String($0) } + } + return QdrantJSON.displayText(payload[QdrantSchema.payloadKey(forColumn: column)]) + } + + public func row(columns: [String]) -> [String?] { + columns.map(cell(for:)) + } + + /// The columns a console result shows: what came back, in the order the collection lists it. + public static func columns(of points: [QdrantPoint], collection: QdrantCollection?) -> [String] { + var columns = [QdrantSchema.idColumn] + if points.contains(where: { $0.score != nil }) { + columns.append(QdrantSchema.scoreColumn) + } + let returned = Set(points.flatMap(\.payloadKeys).map(QdrantSchema.column(forPayloadKey:))) + let declared = collection?.payloadFields.map { QdrantSchema.column(forPayloadKey: $0.name) } ?? [] + columns += declared.filter { returned.contains($0) && !columns.contains($0) } + columns += returned.subtracting(declared).subtracting(columns).sorted() + + var vectorColumns: [String] = [] + for point in points { + if point.vector is [Any] { + vectorColumns.append(QdrantSchema.vectorColumn) + } else if let named = point.vector as? [String: Any] { + vectorColumns += named.keys.map { "\(QdrantSchema.vectorColumn).\($0)" } + } + } + var seen = Set(columns) + for column in vectorColumns.sorted() where seen.insert(column).inserted { + columns.append(column) + } + return columns + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantQuery.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantQuery.swift new file mode 100644 index 0000000000..ad7f1d9c09 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantQuery.swift @@ -0,0 +1,294 @@ +import Foundation + +public struct QdrantSortSpec: Sendable, Equatable { + public let column: String + public let ascending: Bool + + public init(column: String, ascending: Bool) { + self.column = column + self.ascending = ascending + } +} + +public struct QdrantBrowseRequest: Sendable, Equatable { + public let collection: String + public let offset: Int + public let limit: Int + public let sort: QdrantSortSpec? + public let filters: [QdrantFilterSpec] + public let logicMode: String + + public init( + collection: String, + offset: Int, + limit: Int, + sort: QdrantSortSpec?, + filters: [QdrantFilterSpec], + logicMode: String + ) { + self.collection = collection + self.offset = max(offset, 0) + self.limit = max(limit, 1) + self.sort = sort + self.filters = filters + self.logicMode = logicMode + } +} + +/// A grid page travels from `buildFilteredQuery` to `execute` as tagged text, because the page is +/// built before the driver knows the collection's field types the filter JSON depends on. +public enum QdrantBrowseQuery { + public static let tag = "QDRANT_BROWSE:" + + public static func encode(_ request: QdrantBrowseRequest) -> String { + var payload: [String: Any] = [ + "collection": request.collection, + "offset": request.offset, + "limit": request.limit, + "logicMode": request.logicMode, + "filters": request.filters.map { filter -> [String: Any] in + var encoded: [String: Any] = ["column": filter.column, "op": filter.op, "value": filter.value] + if let secondValue = filter.secondValue { + encoded["secondValue"] = secondValue + } + return encoded + } + ] + if let sort = request.sort { + payload["sort"] = ["column": sort.column, "ascending": sort.ascending] + } + let body = (try? QdrantJSON.data(payload)) ?? Data() + return tag + body.base64EncodedString() + } + + public static func parse(_ query: String) -> QdrantBrowseRequest? { + guard query.hasPrefix(tag), + let data = Data(base64Encoded: String(query.dropFirst(tag.count))), + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let collection = json["collection"] as? String + else { return nil } + let sort = (json["sort"] as? [String: Any]).flatMap { item -> QdrantSortSpec? in + guard let column = item["column"] as? String else { return nil } + return QdrantSortSpec(column: column, ascending: item["ascending"] as? Bool ?? true) + } + let filters = (json["filters"] as? [[String: Any]] ?? []).compactMap { item -> QdrantFilterSpec? in + guard let column = item["column"] as? String, let op = item["op"] as? String else { return nil } + return QdrantFilterSpec( + column: column, + op: op, + value: item["value"] as? String ?? "", + secondValue: item["secondValue"] as? String + ) + } + return QdrantBrowseRequest( + collection: collection, + offset: json["offset"] as? Int ?? 0, + limit: json["limit"] as? Int ?? 100, + sort: sort, + filters: filters, + logicMode: json["logicMode"] as? String ?? "AND" + ) + } + + public static func isTagged(_ query: String) -> Bool { + query.hasPrefix(tag) + } + + /// The body of one page. `order_by` is only on the query endpoint, which takes a numeric offset; + /// an unsorted page goes through `scroll`, which pages by point id instead. + public static func queryBody( + for request: QdrantBrowseRequest, + filter: [String: Any]?, + sort: QdrantSortSpec + ) -> [String: Any] { + var body: [String: Any] = [ + "query": ["order_by": [ + "key": QdrantSchema.payloadKey(forColumn: sort.column), + "direction": sort.ascending ? "asc" : "desc" + ]], + "limit": request.limit, + "offset": request.offset, + "with_payload": true, + "with_vector": true + ] + if let filter { + body["filter"] = filter + } + return body + } + + public static func scrollBody( + filter: [String: Any]?, + limit: Int, + cursor: Any?, + withData: Bool, + withVector: Bool? = nil + ) -> [String: Any] { + var body: [String: Any] = [ + "limit": limit, + "with_payload": withData, + "with_vector": withVector ?? withData + ] + if let filter { + body["filter"] = filter + } + if let cursor { + body["offset"] = cursor + } + return body + } +} + +public struct QdrantWriteRequest: Sendable, Equatable { + public let method: String + public let path: String + public let body: String? + + public init(method: String, path: String, body: String?) { + self.method = method + self.path = path + self.body = body + } +} + +/// A grid save is shown in the review sheet and runs as the request line the console would send, +/// so what the user approves, what the read-only gate classifies and what runs are one string. +public enum QdrantWriteCodec { + public static func encode(_ request: QdrantWriteRequest) -> String { + guard let body = request.body else { return "\(request.method) \(request.path)" } + return "\(request.method) \(request.path)\n\(body)" + } +} + +public struct QdrantCellChange: Sendable, Equatable { + public let column: String + public let newText: String? + + public init(column: String, newText: String?) { + self.column = column + self.newText = newText + } +} + +public struct QdrantTrackedChange: Sendable, Equatable { + public enum Kind: String, Sendable, Equatable { + case insert + case update + case delete + } + + public let kind: Kind + public let id: String? + public let values: [String: String?] + public let cellChanges: [QdrantCellChange] + + public init(kind: Kind, id: String?, values: [String: String?] = [:], cellChanges: [QdrantCellChange] = []) { + self.kind = kind + self.id = id + self.values = values + self.cellChanges = cellChanges + } +} + +public enum QdrantSkipReason: String, Sendable, Equatable { + case missingID + case invalidID + case noEditableColumns + case payloadNotEncodable +} + +public struct QdrantSkippedChange: Sendable, Equatable { + public let kind: QdrantTrackedChange.Kind + public let reason: QdrantSkipReason +} + +public struct QdrantWriteBatch: Sendable, Equatable { + public let requests: [QdrantWriteRequest] + public let skipped: [QdrantSkippedChange] +} + +public enum QdrantStatementGenerator { + /// `uuid` supplies the id of an inserted row the user left blank. + public static func generate( + collection: QdrantCollection, + changes: [QdrantTrackedChange], + uuid: () -> String = { UUID().uuidString.lowercased() } + ) -> QdrantWriteBatch { + let base = "/collections/\(QdrantPathEncoding.segment(collection.name))/points" + var requests: [QdrantWriteRequest] = [] + var skipped: [QdrantSkippedChange] = [] + + for change in changes { + switch outcome(for: change, collection: collection, base: base, uuid: uuid) { + case .success(let request): + requests.append(request) + case .failure(let reason): + skipped.append(QdrantSkippedChange(kind: change.kind, reason: reason)) + } + } + return QdrantWriteBatch(requests: requests, skipped: skipped) + } + + private enum Outcome { + case success(QdrantWriteRequest) + case failure(QdrantSkipReason) + } + + private static func outcome( + for change: QdrantTrackedChange, + collection: QdrantCollection, + base: String, + uuid: () -> String + ) -> Outcome { + let immutable = Set(QdrantSchema.immutableColumns(for: collection)) + switch change.kind { + case .insert: + let idText = change.id.flatMap { $0.isEmpty ? nil : $0 } ?? uuid() + guard let id = QdrantPointID.json(from: idText) else { return .failure(.invalidID) } + let payload = payloadObject( + change.values.filter { !immutable.contains($0.key) }, + collection: collection + ) + // `insert_only` keeps an id that already exists from being overwritten, vector and all. + let body: [String: Any] = [ + "points": [["id": id, "vector": [String: Any](), "payload": payload]], + "update_mode": "insert_only" + ] + return request("PUT", "\(base)?wait=true", body) + case .update: + guard let idText = change.id, !idText.isEmpty else { return .failure(.missingID) } + guard let id = QdrantPointID.json(from: idText) else { return .failure(.invalidID) } + var patch: [String: String?] = [:] + for cell in change.cellChanges where !immutable.contains(cell.column) { + patch[cell.column] = cell.newText + } + guard !patch.isEmpty else { return .failure(.noEditableColumns) } + let body: [String: Any] = ["payload": payloadObject(patch, collection: collection), "points": [id]] + return request("POST", "\(base)/payload?wait=true", body) + case .delete: + guard let idText = change.id, !idText.isEmpty else { return .failure(.missingID) } + guard let id = QdrantPointID.json(from: idText) else { return .failure(.invalidID) } + return request("POST", "\(base)/delete?wait=true", ["points": [id]]) + } + } + + private static func request(_ method: String, _ path: String, _ body: [String: Any]) -> Outcome { + guard let text = try? QdrantJSON.text(body) else { return .failure(.payloadNotEncodable) } + return .success(QdrantWriteRequest(method: method, path: path, body: text)) + } + + private static func payloadObject(_ values: [String: String?], collection: QdrantCollection) -> [String: Any] { + var payload: [String: Any] = [:] + for (column, text) in values { + guard let text else { + payload[QdrantSchema.payloadKey(forColumn: column)] = NSNull() + continue + } + payload[QdrantSchema.payloadKey(forColumn: column)] = QdrantJSON.parsedValue( + text, + fieldType: collection.fieldType(of: column) ?? .string + ) + } + return payload + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift new file mode 100644 index 0000000000..e9d92af337 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift @@ -0,0 +1,239 @@ +import Foundation + +/// The type a payload column holds. The first seven are Qdrant's payload index schemas; the rest +/// are read off sampled values for a field nobody indexed. +public enum QdrantFieldType: String, Sendable, Equatable, CaseIterable { + case keyword + case integer + case float + case bool + case geo + case text + case datetime + case uuid + case string + case object + case array + case json + + public var isOrdered: Bool { + self == .integer || self == .float || self == .datetime + } + + public static func indexed(_ schema: Any?) -> QdrantFieldType? { + let name: String? + if let object = schema as? [String: Any] { + name = object["data_type"] as? String + ?? (object["params"] as? [String: Any])?["type"] as? String + } else { + name = schema as? String + } + return name.flatMap { QdrantFieldType(rawValue: $0.lowercased()) } + } + + public static func observed(_ value: Any) -> QdrantFieldType? { + switch value { + case is NSNull: + return nil + case is String: + return .string + case let number as NSNumber: + if QdrantJSON.isBoolean(number) { return .bool } + return CFNumberIsFloatType(number) ? .float : .integer + case is [String: Any]: + return .object + case is [Any]: + return .array + default: + return .json + } + } + + /// An integer column that also holds a fraction reads as a float; anything else that disagrees is + /// mixed JSON, written back as whatever the cell parses to. + func merged(with other: QdrantFieldType) -> QdrantFieldType { + if self == other { return self } + if Set([self, other]) == [.integer, .float] { return .float } + return .json + } +} + +public struct QdrantVectorSpec: Sendable, Equatable { + /// `nil` for the collection's single unnamed vector. + public let name: String? + public let isSparse: Bool + + public init(name: String?, isSparse: Bool = false) { + self.name = name + self.isSparse = isSparse + } + + public var column: String { + name.map { "\(QdrantSchema.vectorColumn).\($0)" } ?? QdrantSchema.vectorColumn + } +} + +public struct QdrantPayloadField: Sendable, Equatable { + public let name: String + public let type: QdrantFieldType + public let isIndexed: Bool + /// False while every value seen was null, so the first real value sets the type outright. + public let hasTypedValue: Bool + + public init(name: String, type: QdrantFieldType, isIndexed: Bool, hasTypedValue: Bool = true) { + self.name = name + self.type = type + self.isIndexed = isIndexed + self.hasTypedValue = hasTypedValue + } +} + +public struct QdrantCollection: Sendable, Equatable { + public let name: String + public let vectors: [QdrantVectorSpec] + public let payloadFields: [QdrantPayloadField] + public let pointsCount: Int? + public let info: String + + public init( + name: String, + vectors: [QdrantVectorSpec], + payloadFields: [QdrantPayloadField], + pointsCount: Int? = nil, + info: String = "{}" + ) { + self.name = name + self.vectors = vectors + self.payloadFields = payloadFields + self.pointsCount = pointsCount + self.info = info + } + + public func fieldType(of column: String) -> QdrantFieldType? { + let key = QdrantSchema.payloadKey(forColumn: column) + return payloadFields.first { $0.name == key }?.type + } + + public var vectorColumns: [String] { vectors.map(\.column) } + + /// The collection with every payload key `points` hold that it lacks, typed by their values. An + /// indexed field keeps the index's type. + public func including(_ points: [QdrantPoint]) -> QdrantCollection { + var fields = payloadFields + var position = Dictionary(uniqueKeysWithValues: fields.enumerated().map { ($1.name, $0) }) + for point in points { + for key in point.payloadKeys { + let observed = point.payload[key].flatMap(QdrantFieldType.observed) + guard let index = position[key] else { + position[key] = fields.count + fields.append(QdrantPayloadField( + name: key, type: observed ?? .json, isIndexed: false, hasTypedValue: observed != nil + )) + continue + } + let field = fields[index] + guard !field.isIndexed, let observed else { continue } + let type = field.hasTypedValue ? field.type.merged(with: observed) : observed + fields[index] = QdrantPayloadField(name: key, type: type, isIndexed: false) + } + } + return QdrantCollection(name: name, vectors: vectors, payloadFields: fields, pointsCount: pointsCount, info: info) + } + + /// `GET /collections/{name}` answers the config and the indexed payload fields; the fields nobody + /// indexed are only visible in the points themselves, so a sample of them fills the rest in. + public static func parse(name: String, info json: Any?, sample: [QdrantPoint]) -> QdrantCollection { + let result = (json as? [String: Any])?["result"] as? [String: Any] ?? [:] + let params = ((result["config"] as? [String: Any])?["params"] as? [String: Any]) ?? [:] + + var vectors: [QdrantVectorSpec] = [] + if let named = params["vectors"] as? [String: Any] { + if named["size"] != nil { + vectors.append(QdrantVectorSpec(name: nil)) + } else { + vectors += named.keys.sorted().map { QdrantVectorSpec(name: $0) } + } + } + if let sparse = params["sparse_vectors"] as? [String: Any] { + vectors += sparse.keys.sorted().map { QdrantVectorSpec(name: $0, isSparse: true) } + } + + let indexed = result["payload_schema"] as? [String: Any] ?? [:] + let fields = indexed.keys.sorted().compactMap { key in + QdrantFieldType.indexed(indexed[key]).map { QdrantPayloadField(name: key, type: $0, isIndexed: true) } + } + + return QdrantCollection( + name: name, + vectors: vectors, + payloadFields: fields, + pointsCount: result["points_count"] as? Int, + info: (try? QdrantJSON.text(result, pretty: true)) ?? "{}" + ).including(sample) + } +} + +public struct QdrantColumn: Sendable, Equatable { + public let name: String + public let type: String + public let isPrimaryKey: Bool +} + +public enum QdrantSchema { + public static let idColumn = "id" + public static let vectorColumn = "vector" + public static let scoreColumn = "score" + + public static func collectionNames(from json: Any?) -> [String] { + let result = (json as? [String: Any])?["result"] as? [String: Any] + let collections = result?["collections"] as? [[String: Any]] ?? [] + return collections.compactMap { $0["name"] as? String }.sorted() + } + + public static func isVectorColumn(_ column: String) -> Bool { + column == vectorColumn || column.hasPrefix("\(vectorColumn).") + } + + static let payloadPrefix = "payload." + + /// A payload key may be `id`, `score` or `vector`, which name the point's own columns, so such a + /// key shows as `payload.` and maps back to its key wherever a column is read or written. + public static func isReserved(_ key: String) -> Bool { + key == idColumn || key == scoreColumn || isVectorColumn(key) + } + + public static func column(forPayloadKey key: String) -> String { + isReserved(key) ? payloadPrefix + key : key + } + + public static func payloadKey(forColumn column: String) -> String { + guard column.hasPrefix(payloadPrefix) else { return column } + let key = String(column.dropFirst(payloadPrefix.count)) + return isReserved(key) ? key : column + } + + public static func immutableColumns(for collection: QdrantCollection) -> [String] { + [idColumn] + collection.vectorColumns + } + + public static func columns(for collection: QdrantCollection) -> [QdrantColumn] { + var columns = [QdrantColumn(name: idColumn, type: "id", isPrimaryKey: true)] + columns += collection.payloadFields.map { + QdrantColumn(name: column(forPayloadKey: $0.name), type: $0.type.rawValue, isPrimaryKey: false) + } + columns += collection.vectors.map { + QdrantColumn(name: $0.column, type: $0.isSparse ? "sparse_vector" : "vector", isPrimaryKey: false) + } + return columns + } + + public static func typeName(of column: String, in collection: QdrantCollection) -> String { + if column == idColumn { return "id" } + if column == scoreColumn { return "float" } + if let spec = collection.vectors.first(where: { $0.column == column }) { + return spec.isSparse ? "sparse_vector" : "vector" + } + if isVectorColumn(column) { return "vector" } + return collection.fieldType(of: column)?.rawValue ?? QdrantFieldType.json.rawValue + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantClient.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantClient.swift new file mode 100644 index 0000000000..75a2c7ed4f --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantClient.swift @@ -0,0 +1,167 @@ +import Foundation + +public struct QdrantScrollPage: @unchecked Sendable { + public let points: [QdrantPoint] + /// The id the next page starts from, `nil` once the collection is exhausted. + public let nextCursor: Any? +} + +public enum QdrantCursorLookup: @unchecked Sendable { + /// Scroll from this id; `nil` is the start of the collection. + case at(Any?) + /// The collection ends before the requested offset. + case pastEnd +} + +public final class QdrantClient: @unchecked Sendable { + public static let sampleSize = 100 + /// How many ids one request skips while walking to a page no cursor is held for. + public static let skipBatchSize = 1_000 + + public let settings: QdrantConnectionSettings + private let transport: QdrantTransport + private let timeout: @Sendable () -> TimeInterval + private let lock = NSLock() + private var version: String? + + public init( + settings: QdrantConnectionSettings, + transport: QdrantTransport, + timeout: @escaping @Sendable () -> TimeInterval + ) { + self.settings = settings + self.transport = transport + self.timeout = timeout + } + + public var serverVersion: String? { + lock.withLock { version } + } + + public func cancelAll() { + transport.cancelAll() + } + + public func connect(requestTimeout: @Sendable () -> TimeInterval) async throws { + let root = try await send(method: "GET", path: "/", timeoutInterval: requestTimeout()) + try throwIfFailed(root) + if let json = QdrantJSON.dictionary(root.json), let found = json["version"] as? String { + lock.withLock { version = found } + } + try throwIfFailed(try await send(method: "GET", path: "/collections", timeoutInterval: requestTimeout())) + } + + /// `/healthz` answers without the key, so a revoked key would leave the session looking live. + public func ping() async throws { + try throwIfFailed(try await send(method: "GET", path: "/collections")) + } + + public func collectionNames() async throws -> [String] { + let response = try await send(method: "GET", path: "/collections") + try throwIfFailed(response) + return QdrantSchema.collectionNames(from: response.json) + } + + public func collection(named name: String) async throws -> QdrantCollection { + let info = try await send(method: "GET", path: collectionPath(name)) + try throwIfFailed(info) + let sample = try await scroll( + collection: name, + body: ["limit": Self.sampleSize, "with_payload": true, "with_vector": false] + ) + return QdrantCollection.parse(name: name, info: info.json, sample: sample.points) + } + + public func scroll(collection: String, body: [String: Any]) async throws -> QdrantScrollPage { + let response = try await post("\(collectionPath(collection))/points/scroll", body) + let result = (response.json as? [String: Any])?["result"] as? [String: Any] + let points = (result?["points"] as? [Any] ?? []).compactMap(QdrantPoint.parse) + let next = result?["next_page_offset"] + return QdrantScrollPage(points: points, nextCursor: next is NSNull ? nil : next) + } + + public func query(collection: String, body: [String: Any]) async throws -> [QdrantPoint] { + let response = try await post("\(collectionPath(collection))/points/query", body) + return QdrantPoint.list(fromResponse: response.json) ?? [] + } + + public func count(collection: String, filter: [String: Any]?) async throws -> Int { + var body: [String: Any] = ["exact": true] + if let filter { + body["filter"] = filter + } + let response = try await post("\(collectionPath(collection))/points/count", body) + let result = (response.json as? [String: Any])?["result"] as? [String: Any] + guard let count = result?["count"] as? Int else { + throw QdrantError.malformedResponse(String(localized: "Qdrant answered a count without a number.")) + } + return count + } + + /// Walks to the cursor of `offset` from the nearest one already known, reading ids only. + public func cursor( + collection: String, + filter: [String: Any]?, + offset: Int, + from known: (offset: Int, cursor: Any?) + ) async throws -> QdrantCursorLookup { + var position = known.offset + var cursor = known.cursor + while position < offset { + try Task.checkCancellation() + let step = min(offset - position, Self.skipBatchSize) + let page = try await scroll( + collection: collection, + body: QdrantBrowseQuery.scrollBody(filter: filter, limit: step, cursor: cursor, withData: false) + ) + position += page.points.count + guard let next = page.nextCursor else { return .pastEnd } + cursor = next + } + return .at(cursor) + } + + public func execute(_ request: QdrantConsoleRequest) async throws -> QdrantHTTPResponse { + let response = try await send(method: request.method, path: request.path, body: request.body.map { Data($0.utf8) }) + try throwIfFailed(response) + return response + } + + public func send(method: String, path: String, body: Data? = nil) async throws -> QdrantHTTPResponse { + try await send(method: method, path: path, body: body, timeoutInterval: timeout()) + } + + private func post(_ path: String, _ body: [String: Any]) async throws -> QdrantHTTPResponse { + let response = try await send(method: "POST", path: path, body: try QdrantJSON.data(body)) + try throwIfFailed(response) + return response + } + + private func collectionPath(_ name: String) -> String { + "/collections/\(QdrantPathEncoding.segment(name))" + } + + private func send(method: String, path: String, body: Data? = nil, timeoutInterval: TimeInterval) async throws -> QdrantHTTPResponse { + guard let url = QdrantPathEncoding.resolve(path, against: try settings.baseURL()) else { + throw QdrantError.configuration(String(format: String(localized: "Invalid path: %@"), path)) + } + var headers = settings.headers + headers["Accept"] = "application/json" + if body != nil { + headers["Content-Type"] = "application/json" + } + return try await transport.send(QdrantHTTPRequest( + method: method, + url: url, + headers: headers, + body: body, + timeoutInterval: timeoutInterval + )) + } + + public func throwIfFailed(_ response: QdrantHTTPResponse) throws { + guard (200..<300).contains(response.statusCode) else { + throw QdrantError.from(status: response.statusCode, body: response.body) + } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift new file mode 100644 index 0000000000..e7e67dbbd0 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift @@ -0,0 +1,111 @@ +import Foundation + +/// The grid asks for a page by row offset and `scroll` answers with the id the next page starts +/// from. This remembers that id for every page already read, so paging forward costs one request +/// and a jump costs one id-only walk from the nearest page before it. +public final class QdrantCursorCache: @unchecked Sendable { + private static let maxQueries = 32 + + private let lock = NSLock() + private var cursors: [String: [Int: Any]] = [:] + private var order: [String] = [] + + public init() {} + + public func nearest(at offset: Int, key: String) -> (offset: Int, cursor: Any?) { + lock.withLock { + guard let known = cursors[key], + let best = known.keys.filter({ $0 <= offset }).max() + else { return (0, nil) } + return (best, known[best]) + } + } + + public func store(_ cursor: Any, at offset: Int, key: String) { + lock.withLock { + if cursors[key] == nil { + order.append(key) + if order.count > Self.maxQueries { + cursors[order.removeFirst()] = nil + } + } + cursors[key, default: [:]][offset] = cursor + } + } + + public func removeAll() { + lock.withLock { + cursors.removeAll() + order.removeAll() + } + } +} + +public extension QdrantClient { + func scrollPage( + collection: String, + filter: [String: Any]?, + offset: Int, + limit: Int, + cache: QdrantCursorCache + ) async throws -> [QdrantPoint] { + let filterKey = filter.flatMap { try? QdrantJSON.text($0) } ?? "" + let key = "\(collection)\u{0}\(filterKey)" + let start: Any? + switch try await cursor( + collection: collection, + filter: filter, + offset: offset, + from: cache.nearest(at: offset, key: key) + ) { + case .pastEnd: + return [] + case .at(let cursor): + start = cursor + if offset > 0, let cursor { + cache.store(cursor, at: offset, key: key) + } + } + let page = try await scroll( + collection: collection, + body: QdrantBrowseQuery.scrollBody(filter: filter, limit: limit, cursor: start, withData: true) + ) + if let next = page.nextCursor { + cache.store(next, at: offset + page.points.count, key: key) + } + return page.points + } + + /// Every point of a collection a page at a time, for export. Without vectors it reads payloads only, + /// which is how an export learns every payload key before it writes its header. + func pointPages( + collection: String, + pageSize: Int, + withVector: Bool = true + ) -> AsyncThrowingStream<[QdrantPoint], Error> { + AsyncThrowingStream { continuation in + let task = Task { + do { + var cursor: Any? + repeat { + try Task.checkCancellation() + let page = try await self.scroll( + collection: collection, + body: QdrantBrowseQuery.scrollBody( + filter: nil, limit: max(pageSize, 1), cursor: cursor, withData: true, withVector: withVector + ) + ) + if !page.points.isEmpty { + continuation.yield(page.points) + } + cursor = page.nextCursor + } while cursor != nil + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + continuation.onTermination = { @Sendable _ in task.cancel() } + } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantTransport.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantTransport.swift new file mode 100644 index 0000000000..b596180e49 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantTransport.swift @@ -0,0 +1,149 @@ +import Foundation + +public struct QdrantHTTPRequest: Sendable, Equatable { + public let method: String + public let url: URL + public let headers: [String: String] + public let body: Data? + public let timeoutInterval: TimeInterval + + public init(method: String, url: URL, headers: [String: String], body: Data?, timeoutInterval: TimeInterval) { + self.method = method + self.url = url + self.headers = headers + self.body = body + self.timeoutInterval = timeoutInterval + } +} + +public struct QdrantHTTPResponse: Sendable, Equatable { + public let statusCode: Int + public let body: Data + + public init(statusCode: Int, body: Data) { + self.statusCode = statusCode + self.body = body + } + + public var json: Any? { + try? JSONSerialization.jsonObject(with: body, options: [.fragmentsAllowed]) + } + + public var text: String { + String(data: body, encoding: .utf8) ?? "" + } +} + +public protocol QdrantTransport: Sendable { + func send(_ request: QdrantHTTPRequest) async throws -> QdrantHTTPResponse + func cancelAll() +} + +public final class URLSessionQdrantTransport: QdrantTransport, @unchecked Sendable { + private let session: URLSession + private let lock = NSLock() + private var inFlight: [ObjectIdentifier: URLSessionTask] = [:] + + public init( + configuration: URLSessionConfiguration = .ephemeral, + resourceTimeout: TimeInterval, + skipTLSVerify: Bool = false + ) { + configuration.timeoutIntervalForResource = resourceTimeout + session = URLSession( + configuration: configuration, + delegate: skipTLSVerify ? TrustingTLSDelegate() : nil, + delegateQueue: nil + ) + } + + deinit { + session.invalidateAndCancel() + } + + public func cancelAll() { + let tasks = lock.withLock { Array(inFlight.values) } + tasks.forEach { $0.cancel() } + } + + public func send(_ request: QdrantHTTPRequest) async throws -> QdrantHTTPResponse { + var urlRequest = URLRequest(url: request.url) + urlRequest.httpMethod = request.method + urlRequest.httpBody = request.body + urlRequest.timeoutInterval = request.timeoutInterval + for (name, value) in request.headers { + urlRequest.setValue(value, forHTTPHeaderField: name) + } + + let tracker = TaskTracker(transport: self) + defer { tracker.finish() } + do { + let (data, response) = try await session.data(for: urlRequest, delegate: tracker) + guard let http = response as? HTTPURLResponse else { + throw QdrantError.transport(String(localized: "Qdrant answered with something other than HTTP.")) + } + return QdrantHTTPResponse(statusCode: http.statusCode, body: data) + } catch let error as URLError where error.code == .cancelled { + throw QdrantError.cancelled + } catch let error as QdrantError { + throw error + } catch let error as URLError { + throw QdrantError.transport(error.localizedDescription) + } + } + + fileprivate func register(_ task: URLSessionTask) { + lock.withLock { inFlight[ObjectIdentifier(task)] = task } + } + + fileprivate func unregister(_ task: URLSessionTask) { + lock.withLock { inFlight[ObjectIdentifier(task)] = nil } + } +} + +private final class TrustingTLSDelegate: NSObject, URLSessionDelegate { + func urlSession( + _ session: URLSession, + didReceive challenge: URLAuthenticationChallenge, + completionHandler: @escaping (URLSession.AuthChallengeDisposition, URLCredential?) -> Void + ) { + guard challenge.protectionSpace.authenticationMethod == NSURLAuthenticationMethodServerTrust, + let trust = challenge.protectionSpace.serverTrust + else { + completionHandler(.performDefaultHandling, nil) + return + } + completionHandler(.useCredential, URLCredential(trust: trust)) + } +} + +private final class TaskTracker: NSObject, URLSessionTaskDelegate, @unchecked Sendable { + private weak var transport: URLSessionQdrantTransport? + private let lock = NSLock() + private var task: URLSessionTask? + + init(transport: URLSessionQdrantTransport) { + self.transport = transport + } + + func urlSession(_ session: URLSession, didCreateTask task: URLSessionTask) { + lock.withLock { self.task = task } + transport?.register(task) + } + + // A redirect would carry the API key to wherever it points. + func urlSession( + _ session: URLSession, + task: URLSessionTask, + willPerformHTTPRedirection response: HTTPURLResponse, + newRequest request: URLRequest, + completionHandler: @escaping (URLRequest?) -> Void + ) { + completionHandler(nil) + } + + func finish() { + guard let task = lock.withLock({ task }) else { return } + transport?.unregister(task) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift index 4241c8d5ce..5863f57a77 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift @@ -319,6 +319,7 @@ public struct SQLLexicalProfile: Sendable, Hashable { "Elasticsearch": commandLineFamily, "Typesense": commandLineFamily, "Weaviate": commandLineFamily, + "Qdrant": commandLineFamily, "Kafka": commandLineFamily, "Firestore": commandLineFamily, "Firebase Realtime Database": commandLineFamily, diff --git a/apple/Packages/TableProCore/Tests/TableProModelsTests/DatabaseTypeTests.swift b/apple/Packages/TableProCore/Tests/TableProModelsTests/DatabaseTypeTests.swift index ca8b6a75bd..6a12c2ce24 100644 --- a/apple/Packages/TableProCore/Tests/TableProModelsTests/DatabaseTypeTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProModelsTests/DatabaseTypeTests.swift @@ -61,7 +61,7 @@ struct DatabaseTypeTests { @Test("allKnownTypes contains all expected types") func allKnownTypesComplete() { - #expect(DatabaseType.allKnownTypes.count == 33) + #expect(DatabaseType.allKnownTypes.count == 34) #expect(DatabaseType.allKnownTypes.contains(.mysql)) #expect(DatabaseType.allKnownTypes.contains(.tidb)) #expect(DatabaseType.allKnownTypes.contains(.databend)) @@ -78,6 +78,7 @@ struct DatabaseTypeTests { #expect(DatabaseType.allKnownTypes.contains(.kafka)) #expect(DatabaseType.allKnownTypes.contains(.cloudflareR2SQL)) #expect(DatabaseType.allKnownTypes.contains(.weaviate)) + #expect(DatabaseType.allKnownTypes.contains(.qdrant)) #expect(DatabaseType.allKnownTypes.contains(.firestore)) #expect(DatabaseType.allKnownTypes.contains(.firebaseRealtimeDatabase)) #expect(DatabaseType.allKnownTypes.contains(.bigtable)) @@ -105,6 +106,13 @@ struct DatabaseTypeTests { #expect(DatabaseType.weaviate.pluginTypeId == "Weaviate") } + @Test("Qdrant resolves its icon and plugin type id") + func qdrantIdentity() { + #expect(DatabaseType.qdrant.rawValue == "Qdrant") + #expect(DatabaseType.qdrant.iconName == "qdrant-icon") + #expect(DatabaseType.qdrant.pluginTypeId == "Qdrant") + } + @Test("Both Firebase types resolve the Firebase icon and keep their own plugin type id") func firebaseIdentity() { #expect(DatabaseType.firestore.rawValue == "Firestore") diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift new file mode 100644 index 0000000000..ee12db647e --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift @@ -0,0 +1,77 @@ +import Foundation +@testable import TableProQdrantCore +import Testing + +struct QdrantClientTests { + @Test("Paging forward reuses the cursor the previous page returned") + func pagingForward() async throws { + let transport = scrollingTransport(count: 25) + let client = makeClient(transport: transport) + let cache = QdrantCursorCache() + + let first = try await client.scrollPage(collection: "docs", filter: nil, offset: 0, limit: 10, cache: cache) + let second = try await client.scrollPage(collection: "docs", filter: nil, offset: 10, limit: 10, cache: cache) + + #expect(first.compactMap(\.idText) == (1...10).map(String.init)) + #expect(second.compactMap(\.idText) == (11...20).map(String.init)) + #expect(transport.recorded.count == 2) + #expect(bodyObject(transport.recorded[1])["offset"] as? Int == 11) + } + + @Test("A jump with no cursor walks ids only, then reads the page") + func jump() async throws { + let transport = scrollingTransport(count: 25) + let client = makeClient(transport: transport) + + let page = try await client.scrollPage(collection: "docs", filter: nil, offset: 20, limit: 10, cache: QdrantCursorCache()) + + #expect(page.compactMap(\.idText) == (21...25).map(String.init)) + let walk = bodyObject(transport.recorded[0]) + #expect(walk["limit"] as? Int == 20) + #expect(walk["with_payload"] as? Bool == false) + #expect(walk["with_vector"] as? Bool == false) + } + + @Test("An offset past the end answers an empty page") + func pastEnd() async throws { + let client = makeClient(transport: scrollingTransport(count: 5)) + let page = try await client.scrollPage(collection: "docs", filter: nil, offset: 10, limit: 10, cache: QdrantCursorCache()) + #expect(page.isEmpty) + } + + @Test("Export walks every page until the cursor runs out") + func export() async throws { + let client = makeClient(transport: scrollingTransport(count: 7)) + var ids: [String] = [] + for try await page in client.pointPages(collection: "docs", pageSize: 3) { + ids += page.compactMap(\.idText) + } + #expect(ids == (1...7).map(String.init)) + } + + @Test("The API key travels in the api-key header and errors carry Qdrant's reason") + func headersAndErrors() async throws { + let transport = RecordingTransport { _ in + jsonResponse(["status": ["error": "Not found: Collection `nope` doesn't exist!"]], status: 404) + } + let client = makeClient(apiKey: "secret", transport: transport) + await #expect(throws: QdrantError.api(status: 404, message: "Not found: Collection `nope` doesn't exist!")) { + _ = try await client.collection(named: "nope") + } + #expect(transport.recorded.first?.headers["api-key"] == "secret") + } + + @Test("A rejected key is an authentication error") + func authentication() { + let error = QdrantError.from(status: 401, body: Data("Invalid api-key".utf8)) + #expect(error == .authentication("Invalid api-key")) + } + + @Test("A request path keeps its query string") + func pathResolution() throws { + let base = try #require(URL(string: "http://localhost:6333")) + let url = QdrantPathEncoding.resolve("/collections/docs/points?wait=true", against: base) + #expect(url?.absoluteString == "http://localhost:6333/collections/docs/points?wait=true") + #expect(QdrantPathEncoding.resolve("//evil.example/x", against: base) == nil) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift new file mode 100644 index 0000000000..14e50cbec6 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift @@ -0,0 +1,94 @@ +import Foundation +@testable import TableProQdrantCore +import Testing + +struct QdrantFilterBuilderTests { + private func condition(_ column: String, _ op: String, _ value: String = "", _ second: String? = nil) throws -> String { + canonical(try QdrantFilterBuilder.condition( + for: QdrantFilterSpec(column: column, op: op, value: value, secondValue: second), + collection: sampleCollection + )) + } + + @Test("Equality matches a keyword as text and an integer as a number") + func equality() throws { + #expect(try condition("city", "=", "Berlin") == #"{"key":"city","match":{"value":"Berlin"}}"#) + #expect(try condition("count", "=", "7") == #"{"key":"count","match":{"value":7}}"#) + #expect(try condition("active", "=", "true") == #"{"key":"active","match":{"value":true}}"#) + } + + @Test("A float has no exact match, so equality is the closed range around it") + func floatEquality() throws { + #expect(try condition("pop", "=", "3.5") == #"{"key":"pop","range":{"gte":3.5,"lte":3.5}}"#) + } + + @Test("Not equal nests the match under must_not") + func notEqual() throws { + #expect(try condition("city", "!=", "Berlin") == #"{"must_not":[{"key":"city","match":{"value":"Berlin"}}]}"#) + } + + @Test("Comparisons and BETWEEN become a range") + func ranges() throws { + #expect(try condition("pop", ">", "2") == #"{"key":"pop","range":{"gt":2}}"#) + #expect(try condition("count", "BETWEEN", "1,5", "5") == #"{"key":"count","range":{"gte":1,"lte":5}}"#) + } + + @Test("A comparison on a string field is refused, not sent") + func comparisonOnText() { + #expect(throws: QdrantFilterError.comparisonNeedsNumberOrDate(column: "note", op: ">")) { + try condition("note", ">", "a") + } + } + + @Test("IN and NOT IN use match any and except") + func lists() throws { + #expect(try condition("city", "IN", "Berlin, Paris") == #"{"key":"city","match":{"any":["Berlin","Paris"]}}"#) + #expect(try condition("count", "NOT IN", "1,2") == #"{"key":"count","match":{"except":[1,2]}}"#) + } + + @Test("CONTAINS needs a full-text index, since Qdrant matches nothing without one") + func contains() throws { + #expect(try condition("body", "CONTAINS", "vector") == #"{"key":"body","match":{"text":"vector"}}"#) + #expect(throws: QdrantFilterError.textMatchNeedsTextIndex(column: "city", op: "CONTAINS")) { + try condition("city", "CONTAINS", "Ber") + } + } + + @Test("Null and empty checks") + func nullness() throws { + #expect(try condition("note", "IS NULL") == #"{"is_null":{"key":"note"}}"#) + #expect(try condition("tags", "IS NOT EMPTY") == #"{"must_not":[{"is_empty":{"key":"tags"}}]}"#) + } + + @Test("The id column filters with has_id, keeping integer and UUID ids apart") + func ids() throws { + let uuid = "5c56c793-69f3-4fbf-87e6-c4bf54c28c26" + #expect(try condition("id", "IN", "1, \(uuid)") == #"{"has_id":[1,"\#(uuid)"]}"#) + #expect(throws: QdrantFilterError.notAPointID(value: "abc")) { try condition("id", "=", "abc") } + } + + @Test("Vectors cannot be filtered and unsupported operators say so") + func refusals() { + #expect(throws: QdrantFilterError.vectorNotFilterable(column: "vector")) { try condition("vector", "=", "1") } + #expect(throws: QdrantFilterError.unsupportedOperator("STARTS WITH")) { try condition("city", "STARTS WITH", "B") } + } + + @Test("A raw row is a Qdrant condition in JSON") + func raw() throws { + #expect(try condition("__RAW__", "", #"{"key":"city","match":{"value":"X"}}"#) == #"{"key":"city","match":{"value":"X"}}"#) + #expect(throws: QdrantFilterError.rawFilterNotAnObject) { try condition("__RAW__", "", "city = 1") } + } + + @Test("AND rows go under must and OR rows under should") + func logic() throws { + let specs = [ + QdrantFilterSpec(column: "city", op: "=", value: "Berlin"), + QdrantFilterSpec(column: "count", op: "<", value: "3") + ] + let and = try #require(try QdrantFilterBuilder.filter(specs, logicMode: "AND", collection: sampleCollection)) + let or = try #require(try QdrantFilterBuilder.filter(specs, logicMode: "OR", collection: sampleCollection)) + #expect((and["must"] as? [Any])?.count == 2) + #expect((or["should"] as? [Any])?.count == 2) + #expect(try QdrantFilterBuilder.filter([], logicMode: "AND", collection: sampleCollection) == nil) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaAndWriteTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaAndWriteTests.swift new file mode 100644 index 0000000000..190a771a8d --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaAndWriteTests.swift @@ -0,0 +1,198 @@ +import Foundation +@testable import TableProQdrantCore +import Testing + +struct QdrantSchemaAndWriteTests { + private func info(vectors: Any, payloadSchema: [String: Any] = [:]) -> [String: Any] { + ["result": [ + "points_count": 4, + "config": ["params": ["vectors": vectors]], + "payload_schema": payloadSchema + ]] + } + + private func point(_ json: String) throws -> QdrantPoint { + let object = try JSONSerialization.jsonObject(with: Data(json.utf8)) + return try #require(QdrantPoint.parse(object)) + } + + @Test("Columns are the id, indexed fields, sampled fields, then vectors") + func columns() throws { + let sample = [ + try point(#"{"id":1,"payload":{"city":"Berlin","count":3,"meta":{"x":1}}}"#), + try point(#"{"id":2,"payload":{"count":3.5,"tags":["a"]}}"#) + ] + let collection = QdrantCollection.parse( + name: "docs", + info: info(vectors: ["size": 4, "distance": "Cosine"], payloadSchema: ["city": ["data_type": "keyword"]]), + sample: sample + ) + #expect(QdrantSchema.columns(for: collection).map(\.name) == ["id", "city", "count", "meta", "tags", "vector"]) + #expect(collection.fieldType(of: "city") == .keyword) + #expect(collection.fieldType(of: "count") == .float) + #expect(collection.fieldType(of: "meta") == .object) + #expect(collection.pointsCount == 4) + #expect(QdrantSchema.immutableColumns(for: collection) == ["id", "vector"]) + } + + @Test("A payload key named like a point column shows as payload. and maps back") + func reservedPayloadKeys() throws { + let row = try point(#"{"id":7,"payload":{"id":"external-1","vector":"text","city":"Berlin"},"vector":[1,2]}"#) + let collection = QdrantCollection.parse(name: "docs", info: info(vectors: ["size": 2]), sample: [row]) + let columns = QdrantSchema.columns(for: collection).map(\.name) + #expect(columns == ["id", "city", "payload.id", "payload.vector", "vector"]) + #expect(row.row(columns: columns) == ["7", "Berlin", "external-1", "text", "[1,2]"]) + + let condition = try QdrantFilterBuilder.condition( + for: QdrantFilterSpec(column: "payload.id", op: "=", value: "external-1"), + collection: collection + ) + #expect(canonical(condition) == #"{"key":"id","match":{"value":"external-1"}}"#) + + let batch = QdrantStatementGenerator.generate(collection: collection, changes: [ + QdrantTrackedChange(kind: .update, id: "7", cellChanges: [QdrantCellChange(column: "payload.id", newText: "external-2")]) + ]) + #expect(batch.requests.first?.body == #"{"payload":{"id":"external-2"},"points":[7]}"#) + } + + @Test("Keys outside the sample join the collection, and a key first seen as null takes its later type") + func includingLaterPoints() throws { + let sample = [try point(#"{"id":1,"payload":{"city":"Berlin","rank":null}}"#)] + let collection = QdrantCollection.parse(name: "docs", info: info(vectors: ["size": 2]), sample: sample) + let widened = collection.including([try point(#"{"id":2,"payload":{"late":true,"rank":3}}"#)]) + #expect(widened.payloadFields.map(\.name) == ["city", "rank", "late"]) + #expect(widened.fieldType(of: "rank") == .integer) + #expect(widened.fieldType(of: "late") == .bool) + } + + @Test("Named and sparse vectors each get a read-only column") + func namedVectors() throws { + var json = info(vectors: ["image": ["size": 2], "text": ["size": 3]]) + var result = try #require(json["result"] as? [String: Any]) + var config = try #require(result["config"] as? [String: Any]) + var params = try #require(config["params"] as? [String: Any]) + params["sparse_vectors"] = ["keywords": [String: Any]()] + config["params"] = params + result["config"] = config + json["result"] = result + + let collection = QdrantCollection.parse(name: "docs", info: json, sample: []) + #expect(collection.vectorColumns == ["vector.image", "vector.text", "vector.keywords"]) + + let row = try point(#"{"id":"5c56c793-69f3-4fbf-87e6-c4bf54c28c26","vector":{"image":[0.5,1],"keywords":{"indices":[1],"values":[0.5]}}}"#) + #expect(row.cell(for: "id") == "5c56c793-69f3-4fbf-87e6-c4bf54c28c26") + #expect(row.cell(for: "vector.image") == "[0.5,1]") + #expect(row.cell(for: "vector.keywords") == #"{"indices":[1],"values":[0.5]}"#) + #expect(row.cell(for: "vector.text") == nil) + } + + @Test("A query result shows the score after the id") + func queryColumns() throws { + let json = try JSONSerialization.jsonObject(with: Data( + #"{"result":{"points":[{"id":1,"score":0.9,"payload":{"city":"Berlin"},"vector":[1,2]}]},"status":"ok"}"#.utf8 + )) + let points = try #require(QdrantPoint.list(fromResponse: json)) + let columns = QdrantPoint.columns(of: points, collection: nil) + #expect(columns == ["id", "score", "city", "vector"]) + #expect(points[0].row(columns: columns) == ["1", "0.9", "Berlin", "[1,2]"]) + #expect(QdrantPoint.list(fromResponse: ["result": ["count": 3]]) == nil) + } + + @Test("A payload edit is a set-payload request typed by the column") + func update() throws { + let batch = QdrantStatementGenerator.generate(collection: sampleCollection, changes: [ + QdrantTrackedChange(kind: .update, id: "7", cellChanges: [ + QdrantCellChange(column: "note", newText: "42"), + QdrantCellChange(column: "count", newText: "42"), + QdrantCellChange(column: "vector", newText: "[1]") + ]) + ]) + let request = try #require(batch.requests.first) + #expect(request.method == "POST") + #expect(request.path == "/collections/docs/points/payload?wait=true") + #expect(request.body == #"{"payload":{"count":42,"note":"42"},"points":[7]}"#) + } + + @Test("A delete names the point, and a row without an id is skipped, not sent") + func delete() throws { + let uuid = "5c56c793-69f3-4fbf-87e6-c4bf54c28c26" + let batch = QdrantStatementGenerator.generate(collection: sampleCollection, changes: [ + QdrantTrackedChange(kind: .delete, id: uuid), + QdrantTrackedChange(kind: .delete, id: nil), + QdrantTrackedChange(kind: .update, id: "1", cellChanges: [QdrantCellChange(column: "id", newText: "2")]) + ]) + #expect(batch.requests.map(QdrantWriteCodec.encode) == [ + "POST /collections/docs/points/delete?wait=true\n{\"points\":[\"\(uuid)\"]}" + ]) + #expect(batch.skipped.map(\.reason) == [.missingID, .noEditableColumns]) + } + + @Test("An insert never overwrites an existing id and gets a UUID when left blank") + func insert() throws { + let batch = QdrantStatementGenerator.generate( + collection: sampleCollection, + changes: [QdrantTrackedChange(kind: .insert, id: nil, values: ["city": "Hanoi", "active": "true"])], + uuid: { "00000000-0000-0000-0000-000000000001" } + ) + let request = try #require(batch.requests.first) + #expect(request.method == "PUT") + #expect(request.path == "/collections/docs/points?wait=true") + #expect(request.body == #"{"points":[{"id":"00000000-0000-0000-0000-000000000001","payload":{"active":true,"city":"Hanoi"},"vector":{}}],"update_mode":"insert_only"}"#) + } + + @Test("Browse requests round-trip through their tag") + func browseRoundTrip() throws { + let request = QdrantBrowseRequest( + collection: "docs", + offset: 200, + limit: 100, + sort: QdrantSortSpec(column: "pop", ascending: false), + filters: [QdrantFilterSpec(column: "count", op: "BETWEEN", value: "1,5", secondValue: "5")], + logicMode: "OR" + ) + #expect(QdrantBrowseQuery.parse(QdrantBrowseQuery.encode(request)) == request) + } + + @Test("A sorted page uses order_by with a numeric offset") + func sortedBody() { + let request = QdrantBrowseRequest(collection: "docs", offset: 50, limit: 25, sort: nil, filters: [], logicMode: "AND") + let body = QdrantBrowseQuery.queryBody(for: request, filter: nil, sort: QdrantSortSpec(column: "pop", ascending: false)) + #expect(canonical(body) == #"{"limit":25,"offset":50,"query":{"order_by":{"direction":"desc","key":"pop"}},"with_payload":true,"with_vector":true}"#) + } + + @Test("The console reads a request line with its body below or beside it") + func console() { + let below = QdrantConsoleParser.parse("POST /collections/docs/points/query\n{\"limit\": 3}") + #expect(below == QdrantConsoleRequest(method: "POST", path: "/collections/docs/points/query", body: "{\"limit\": 3}")) + let inline = QdrantConsoleParser.parse("post /collections/docs/points/count {\"exact\": true}") + #expect(inline?.method == "POST") + #expect(inline?.body == "{\"exact\": true}") + #expect(QdrantConsoleParser.parse("SELECT 1") == nil) + #expect(QdrantConsoleParser.parse("GET collections") == nil) + } + + @Test("Only the read endpoints count as reads") + func readEndpoints() { + #expect(QdrantConsoleRequest(method: "POST", path: "/collections/docs/points/query", body: nil).isRead) + #expect(QdrantConsoleRequest(method: "POST", path: "/collections/docs/points", body: nil).isRead) + #expect(!QdrantConsoleRequest(method: "POST", path: "/collections/docs/points/delete?wait=true", body: nil).isRead) + #expect(!QdrantConsoleRequest(method: "PUT", path: "/collections/docs/points", body: nil).isRead) + } + + @Test("Point ids parse back to the kind Qdrant expects") + func pointIDs() { + #expect(QdrantPointID.json(from: "42") as? NSNumber == 42) + #expect(QdrantPointID.json(from: "5C56C793-69F3-4FBF-87E6-C4BF54C28C26") as? String == "5c56c793-69f3-4fbf-87e6-c4bf54c28c26") + #expect(QdrantPointID.json(from: "-1") == nil) + #expect(QdrantPointID.json(from: "abc") == nil) + } + + @Test("A cell is written back as its column's type") + func parsedValues() { + #expect(QdrantJSON.parsedValue("42", fieldType: .keyword) as? String == "42") + #expect(QdrantJSON.parsedValue("42", fieldType: .integer) as? Int64 == 42) + #expect(QdrantJSON.parsedValue("false", fieldType: .bool) as? Bool == false) + #expect((QdrantJSON.parsedValue(#"["a"]"#, fieldType: .array) as? [String]) == ["a"]) + #expect(QdrantJSON.parsedValue("not json", fieldType: .object) as? String == "not json") + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantTestSupport.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantTestSupport.swift new file mode 100644 index 0000000000..7c907a2f13 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantTestSupport.swift @@ -0,0 +1,73 @@ +import Foundation +@testable import TableProQdrantCore + +final class RecordingTransport: QdrantTransport, @unchecked Sendable { + private let lock = NSLock() + private var handler: (QdrantHTTPRequest) throws -> QdrantHTTPResponse + private(set) var requests: [QdrantHTTPRequest] = [] + + init(_ handler: @escaping (QdrantHTTPRequest) throws -> QdrantHTTPResponse) { + self.handler = handler + } + + func send(_ request: QdrantHTTPRequest) async throws -> QdrantHTTPResponse { + let handler = lock.withLock { + requests.append(request) + return self.handler + } + return try handler(request) + } + + func cancelAll() {} + + var recorded: [QdrantHTTPRequest] { lock.withLock { requests } } +} + +func jsonResponse(_ object: Any, status: Int = 200) -> QdrantHTTPResponse { + QdrantHTTPResponse(statusCode: status, body: (try? JSONSerialization.data(withJSONObject: object)) ?? Data()) +} + +func bodyObject(_ request: QdrantHTTPRequest) -> [String: Any] { + guard let body = request.body else { return [:] } + return (try? JSONSerialization.jsonObject(with: body) as? [String: Any]) ?? [:] +} + +func makeClient(apiKey: String = "", transport: RecordingTransport) -> QdrantClient { + QdrantClient( + settings: QdrantConnectionSettings(host: "localhost", port: 6_333, usesTLS: false, apiKey: apiKey, skipTLSVerify: false), + transport: transport, + timeout: { 5 } + ) +} + +/// A collection of integer ids 1...count answering `scroll` the way Qdrant does: from `offset` +/// (an id) up to `limit` points, with the next id as the cursor. +func scrollingTransport(count: Int) -> RecordingTransport { + RecordingTransport { request in + let body = bodyObject(request) + let limit = body["limit"] as? Int ?? 10 + let start = (body["offset"] as? Int) ?? 1 + let ids = Array(start.. String { + (try? QdrantJSON.text(object)) ?? "" +} diff --git a/apple/Plugins/QdrantDriverPlugin/Info.plist b/apple/Plugins/QdrantDriverPlugin/Info.plist new file mode 100644 index 0000000000..a9c5f4eb04 --- /dev/null +++ b/apple/Plugins/QdrantDriverPlugin/Info.plist @@ -0,0 +1,12 @@ + + + + + TableProPluginKitVersion + TABLEPRO_PLUGINKIT_VERSION + TableProProvidesDatabaseTypeIds + + Qdrant + + + diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPlugin.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPlugin.swift new file mode 100644 index 0000000000..594b9d6c9d --- /dev/null +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPlugin.swift @@ -0,0 +1,111 @@ +import Foundation +import TableProPluginKit +import TableProQdrantCore + +final class QdrantPlugin: NSObject, TableProPlugin, DriverPlugin { + static let pluginName = "Qdrant Driver" + static let pluginVersion = "1.0.0" + static let pluginDescription = "Qdrant support over the REST API" + static let capabilities: [PluginCapability] = [.databaseDriver] + + static let databaseTypeId = "Qdrant" + static let databaseDisplayName = "Qdrant" + static let iconName = "qdrant-icon" + static let defaultPort = QdrantConnectionSettings.defaultPort + static let isDownloadable = true + + static let navigationModel: NavigationModel = .standard + static let pathFieldRole: PathFieldRole = .database + static let requiresAuthentication = false + static let brandColorHex = "#DC244C" + static let queryLanguageName = "REST" + static let editorLanguage: EditorLanguage = .javascript + static let supportsForeignKeys = false + static let supportsSchemaEditing = false + static let supportsDatabaseSwitching = false + static let supportsImport = false + static let supportsExport = true + static let supportsSSH = true + static let supportsSSL = true + static let supportsReadOnlyMode = true + static let supportsForeignKeyDisable = false + static let supportsAddColumn = false + static let supportsModifyColumn = false + static let supportsDropColumn = false + static let supportsAddIndex = false + static let supportsDropIndex = false + static let supportsModifyPrimaryKey = false + static let databaseGroupingStrategy: GroupingStrategy = .flat + static let defaultGroupName = "default" + static let tableEntityName = "Collections" + static let containerEntityName = "Cluster" + static let immutableColumns: [String] = [QdrantSchema.idColumn, QdrantSchema.vectorColumn] + static let defaultPrimaryKeyColumn: String? = QdrantSchema.idColumn + static let structureColumnFields: [StructureColumnField] = [.name, .type, .nullable] + static let sqlDialect: SQLDialectDescriptor? = nil + + static let columnTypesByCategory: [String: [String]] = qdrantColumnTypes + + static let additionalConnectionFields: [ConnectionField] = qdrantConnectionFields() + + static var statementCompletions: [CompletionEntry] { qdrantCompletions } + + func createDriver(config: DriverConnectionConfig) -> any PluginDatabaseDriver { + QdrantPluginDriver(config: config) + } +} + +func qdrantConnectionFields() -> [ConnectionField] { + [ + ConnectionField( + id: QdrantFieldID.apiKey, + label: String(localized: "API Key"), + placeholder: "Qdrant API key", + fieldType: .secure, + section: .authentication, + hidesPassword: true + ).withHidesUsername(true), + ConnectionField( + id: QdrantFieldID.skipTLSVerify, + label: String(localized: "Skip TLS Verification"), + defaultValue: "false", + fieldType: .toggle, + section: .advanced + ) + ] +} + +let qdrantCompletions: [CompletionEntry] = [ + CompletionEntry(label: "GET /collections", insertText: "GET /collections"), + CompletionEntry(label: "GET /collections/{name}", insertText: "GET /collections/docs"), + CompletionEntry( + label: "POST /points/scroll", + insertText: """ + POST /collections/docs/points/scroll + {"limit": 10, "with_payload": true} + """ + ), + CompletionEntry( + label: "POST /points/query", + insertText: """ + POST /collections/docs/points/query + {"query": [0.1, 0.2, 0.3, 0.4], "limit": 10, "with_payload": true} + """ + ), + CompletionEntry( + label: "POST /points/count", + insertText: """ + POST /collections/docs/points/count + {"exact": true} + """ + ) +] + +let qdrantColumnTypes: [String: [String]] = [ + "Text": ["keyword", "text", "string", "uuid"], + "Numeric": ["integer", "float"], + "Boolean": ["bool"], + "Date": ["datetime"], + "Structured": ["geo", "object", "array", "json"], + "Vector": ["vector", "sparse_vector"] +] diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift new file mode 100644 index 0000000000..4102b56433 --- /dev/null +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift @@ -0,0 +1,172 @@ +import Foundation +import TableProPluginKit +import TableProQdrantCore + +extension QdrantPluginDriver { + static let exportPageSize = 500 + + func execute(query: String) async throws -> PluginQueryResult { + let started = Date() + let client = try requireClient() + let trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines) + + if trimmed.lowercased() == "select 1" { + try await client.ping() + return PluginQueryResult( + columns: ["ok"], + columnTypeNames: ["integer"], + rows: [[.text("1")]], + rowsAffected: 0, + executionTime: Date().timeIntervalSince(started) + ) + } + if let request = QdrantBrowseQuery.parse(trimmed) { + return try await executeBrowse(request, client: client, started: started) + } + if let request = QdrantConsoleParser.parse(trimmed) { + return try await executeConsole(request, client: client, started: started) + } + throw QdrantError.malformedResponse( + String(localized: "Enter a request like GET /collections or POST /collections/docs/points/query.") + ) + } + + private func executeBrowse( + _ request: QdrantBrowseRequest, + client: QdrantClient, + started: Date + ) async throws -> PluginQueryResult { + let collection = try await collection(request.collection) + let filter = try QdrantFilterBuilder.filter(request.filters, logicMode: request.logicMode, collection: collection) + let points: [QdrantPoint] + if let sort = request.sort { + points = try await client.query( + collection: request.collection, + body: QdrantBrowseQuery.queryBody(for: request, filter: filter, sort: sort) + ) + } else { + points = try await client.scrollPage( + collection: request.collection, + filter: filter, + offset: request.offset, + limit: request.limit, + cache: cursors + ) + } + let columns = QdrantSchema.columns(for: collection).map(\.name) + return render(points, columns: columns, collection: collection, started: started) + } + + private func executeConsole( + _ request: QdrantConsoleRequest, + client: QdrantClient, + started: Date + ) async throws -> PluginQueryResult { + let response = try await client.execute(request) + if !request.isRead { + forgetAfterWrite() + } + if let points = QdrantPoint.list(fromResponse: response.json), !points.isEmpty { + let name = Self.collectionName(in: request.path) + let collection = name.flatMap(rememberedCollection) + let columns = QdrantPoint.columns(of: points, collection: collection) + return render( + points, + columns: columns, + collection: collection ?? QdrantCollection(name: name ?? "", vectors: [], payloadFields: []), + started: started + ) + } + return renderJSON(response, started: started) + } + + static func collectionName(in path: String) -> String? { + let segments = path.split(separator: "?").first?.split(separator: "/") ?? [] + guard segments.count >= 2, segments[0] == "collections" else { return nil } + return String(segments[1]).removingPercentEncoding + } + + private func render( + _ points: [QdrantPoint], + columns: [String], + collection: QdrantCollection, + started: Date + ) -> PluginQueryResult { + PluginQueryResult( + columns: columns, + columnTypeNames: columns.map { QdrantSchema.typeName(of: $0, in: collection) }, + rows: points.map { point in point.row(columns: columns).map { $0.map(PluginCellValue.text) ?? .null } }, + rowsAffected: 0, + executionTime: Date().timeIntervalSince(started) + ) + } + + private func renderJSON(_ response: QdrantHTTPResponse, started: Date) -> PluginQueryResult { + let text: String + if let json = response.json, JSONSerialization.isValidJSONObject(json), + let pretty = try? QdrantJSON.text(json, pretty: true) { + text = pretty + } else { + text = response.text + } + return PluginQueryResult( + columns: ["response"], + columnTypeNames: ["json"], + rows: [[.text(text)]], + rowsAffected: 0, + executionTime: Date().timeIntervalSince(started) + ) + } + + // MARK: - Export + + /// Walks the collection a page at a time instead of buffering it whole, which the protocol's + /// default `streamRows` would do by running `execute` once. + func streamRows(query: String) -> AsyncThrowingStream { + AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + let task = Task { + do { + if let name = QdrantOperations.decodeExport(query) { + try await self.streamCollection(name, continuation: continuation) + } else { + let result = try await self.execute(query: query) + continuation.yield(.header(PluginStreamHeader( + columns: result.columns, + columnTypeNames: result.columnTypeNames, + estimatedRowCount: nil + ))) + if !result.rows.isEmpty { continuation.yield(.rows(result.rows)) } + } + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + continuation.onTermination = { @Sendable _ in task.cancel() } + } + } + + private func streamCollection( + _ name: String, + continuation: AsyncThrowingStream.Continuation + ) async throws { + let client = try requireClient() + var collection = try await collection(name, refresh: true) + // The grid's columns come from a sample; an export has to carry every key, so it reads + // payloads alone first, which skips the vectors that make up most of a page. + for try await points in client.pointPages(collection: name, pageSize: Self.exportPageSize, withVector: false) { + collection = collection.including(points) + } + let columns = QdrantSchema.columns(for: collection).map(\.name) + continuation.yield(.header(PluginStreamHeader( + columns: columns, + columnTypeNames: columns.map { QdrantSchema.typeName(of: $0, in: collection) }, + estimatedRowCount: collection.pointsCount + ))) + for try await points in client.pointPages(collection: name, pageSize: Self.exportPageSize) { + continuation.yield(.rows(points.map { point in + point.row(columns: columns).map { $0.map(PluginCellValue.text) ?? .null } + })) + } + } +} diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift new file mode 100644 index 0000000000..d18a73f8aa --- /dev/null +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift @@ -0,0 +1,209 @@ +import Foundation +import os +import TableProPluginKit +import TableProQdrantCore + +extension QdrantPluginDriver { + func fetchDatabases() async throws -> [String] { + [QdrantPlugin.defaultGroupName] + } + + func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { + PluginDatabaseMetadata(name: database) + } + + func fetchTables(schema: String?) async throws -> [PluginTableInfo] { + try await requireClient().collectionNames().map { PluginTableInfo(name: $0, type: "TABLE") } + } + + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { + let collection = try await collection(table, refresh: true) + return QdrantSchema.columns(for: collection).map { column in + PluginColumnInfo( + name: column.name, + dataType: column.type, + isNullable: !column.isPrimaryKey, + isPrimaryKey: column.isPrimaryKey + ) + } + } + + func fetchImmutableColumns(table: String, schema: String?) async throws -> Set { + Set(QdrantSchema.immutableColumns(for: try await collection(table))) + } + + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { + let collection = try await collection(table) + var indexes = [ + PluginIndexInfo( + name: QdrantSchema.idColumn, + columns: [QdrantSchema.idColumn], + isUnique: true, + isPrimary: true, + type: "PRIMARY KEY" + ) + ] + indexes += collection.payloadFields.filter(\.isIndexed).map { field in + PluginIndexInfo( + name: field.name, + columns: [field.name], + isUnique: false, + isPrimary: false, + type: field.type.rawValue + ) + } + return indexes + } + + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { + [] + } + + func fetchTableDDL(table: String, schema: String?) async throws -> String { + try await collection(table, refresh: true).info + } + + func fetchViewDefinition(view: String, schema: String?) async throws -> String { + throw QdrantError.configuration(String(localized: "Qdrant does not support views.")) + } + + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + PluginTableMetadata(tableName: table, engine: "Qdrant") + } + + func fetchApproximateRowCount(table: String, schema: String?) async throws -> Int? { + try await collection(table, refresh: true).pointsCount + } + + func fetchFilteredRowCount(table: String, queryFilters: [PluginQueryFilter], logicMode: String) async throws -> Int? { + let collection = try await collection(table) + let filter = try QdrantFilterBuilder.filter(specs(queryFilters), logicMode: logicMode, collection: collection) + return try await requireClient().count(collection: table, filter: filter) + } + + func buildBrowseQuery( + table: String, + sortColumns: [(columnIndex: Int, ascending: Bool)], + columns: [String], + limit: Int, + offset: Int + ) -> String? { + buildFilteredQuery( + table: table, + schema: nil, + queryFilters: [], + logicMode: "AND", + sortColumns: sortColumns, + columns: columns, + limit: limit, + offset: offset, + columnKinds: [:] + ) + } + + /// Qdrant orders by one payload key only, so the first sortable column wins. The id is the + /// order a page already has, and a vector has no order. + func buildFilteredQuery( + table: String, + schema: String?, + queryFilters filters: [PluginQueryFilter], + logicMode: String, + sortColumns: [(columnIndex: Int, ascending: Bool)], + columns: [String], + limit: Int, + offset: Int, + columnKinds: [String: PluginColumnKind] + ) -> String? { + let sort = sortColumns.lazy.compactMap { sort -> QdrantSortSpec? in + guard sort.columnIndex >= 0, sort.columnIndex < columns.count else { return nil } + let column = columns[sort.columnIndex] + guard column != QdrantSchema.idColumn, !QdrantSchema.isVectorColumn(column) else { return nil } + return QdrantSortSpec(column: column, ascending: sort.ascending) + }.first + return QdrantBrowseQuery.encode(QdrantBrowseRequest( + collection: table, + offset: offset, + limit: limit, + sort: sort, + filters: specs(filters), + logicMode: logicMode + )) + } + + func specs(_ filters: [PluginQueryFilter]) -> [QdrantFilterSpec] { + filters.map { + QdrantFilterSpec(column: $0.column, op: $0.op, value: $0.value, secondValue: $0.secondValue) + } + } + + func generateStatements( + table: String, + columns: [String], + primaryKeyColumns: [String], + changes: [PluginRowChange], + insertedRowData: [Int: [PluginCellValue]], + deletedRowIndices: Set, + insertedRowIndices: Set + ) -> [(statement: String, parameters: [PluginCellValue])]? { + let collection = rememberedCollection(table) + ?? QdrantCollection(name: table, vectors: [], payloadFields: []) + let tracked = changes.compactMap { change in + trackedChange( + change, + columns: columns, + insertedRowData: insertedRowData, + deletedRowIndices: deletedRowIndices, + insertedRowIndices: insertedRowIndices + ) + } + let batch = QdrantStatementGenerator.generate(collection: collection, changes: tracked) + for skipped in batch.skipped { + Self.logger.warning( + "Skipped a \(skipped.kind.rawValue, privacy: .public) on \(table, privacy: .private): \(skipped.reason.rawValue, privacy: .public)" + ) + } + return batch.requests.map { (QdrantWriteCodec.encode($0), []) } + } + + private func trackedChange( + _ change: PluginRowChange, + columns: [String], + insertedRowData: [Int: [PluginCellValue]], + deletedRowIndices: Set, + insertedRowIndices: Set + ) -> QdrantTrackedChange? { + switch change.type { + case .insert: + guard insertedRowIndices.contains(change.rowIndex) else { return nil } + var values: [String: String?] = [:] + if let row = insertedRowData[change.rowIndex] { + for (index, column) in columns.enumerated() where index < row.count { + guard let text = row[index].asText else { continue } + values[column] = text + } + } else { + for cell in change.cellChanges { + guard let text = cell.newValue.asText else { continue } + values[cell.columnName] = text + } + } + return QdrantTrackedChange(kind: .insert, id: values[QdrantSchema.idColumn] ?? nil, values: values) + case .update: + let cells = change.cellChanges.map { + QdrantCellChange(column: $0.columnName, newText: $0.newValue.asText) + } + return QdrantTrackedChange(kind: .update, id: pointID(from: change, columns: columns), cellChanges: cells) + case .delete: + guard deletedRowIndices.contains(change.rowIndex) else { return nil } + return QdrantTrackedChange(kind: .delete, id: pointID(from: change, columns: columns)) + } + } + + private func pointID(from change: PluginRowChange, columns: [String]) -> String? { + guard let original = change.originalRow, + let index = columns.firstIndex(of: QdrantSchema.idColumn), + index < original.count + else { return nil } + return original[index].asText + } +} diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift new file mode 100644 index 0000000000..22edbe01e6 --- /dev/null +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift @@ -0,0 +1,120 @@ +import Foundation +import os +import TableProPluginKit +import TableProQdrantCore + +internal final class QdrantPluginDriver: PluginDatabaseDriver, @unchecked Sendable { + static let logger = Logger(subsystem: "com.TablePro", category: "QdrantPluginDriver") + + private let config: DriverConnectionConfig + private let lock = NSLock() + private var client: QdrantClient? + private var cachedCollections: [String: QdrantCollection] = [:] + let cursors = QdrantCursorCache() + let queryTimeout = HttpQueryTimeoutBox() + + init(config: DriverConnectionConfig) { + self.config = config + } + + var serverVersion: String? { lock.withLock { client?.serverVersion } } + + var supportsTransactions: Bool { false } + + var capabilities: PluginCapabilities { [.cancelQuery] } + + var parameterStyle: ParameterStyle { .questionMark } + + func beginTransaction() async throws {} + func commitTransaction() async throws {} + func rollbackTransaction() async throws {} + + func connect() async throws { + let settings = try QdrantConnectionSettings.parse( + host: config.host, + port: config.port, + usesTLS: config.ssl.isEnabled, + fields: config.additionalFields + ) + let skipTLS = settings.skipTLSVerify + || (config.ssl.isEnabled && !config.ssl.verifiesCertificate) + let deadline = PluginConnectDeadline(milliseconds: PluginConnectTimeout.milliseconds( + in: config.additionalFields, + default: Int(HttpQueryTimeout().requestTimeoutInterval * 1_000) + )) + let timeout = queryTimeout + let client = QdrantClient( + settings: settings, + transport: URLSessionQdrantTransport( + resourceTimeout: HttpQueryTimeout.sessionResourceTimeout, + skipTLSVerify: skipTLS + ), + timeout: { timeout.requestTimeoutInterval } + ) + try await client.connect(requestTimeout: { deadline.remainingSeconds() }) + lock.withLock { + self.client = client + cachedCollections.removeAll() + } + cursors.removeAll() + } + + func disconnect() { + lock.withLock { + client?.cancelAll() + client = nil + cachedCollections.removeAll() + } + cursors.removeAll() + } + + func ping() async throws { + try await requireClient().ping() + } + + func cancelQuery() throws { + lock.withLock { client?.cancelAll() } + } + + func applyQueryTimeout(_ seconds: Int) async throws { + queryTimeout.set(serverTimeoutSeconds: seconds) + } + + func requireClient() throws -> QdrantClient { + guard let client = lock.withLock({ client }) else { + throw QdrantError.notConnected + } + return client + } + + func rememberedCollection(_ name: String) -> QdrantCollection? { + lock.withLock { cachedCollections[name] } + } + + func collection(_ name: String, refresh: Bool = false) async throws -> QdrantCollection { + if !refresh, let cached = rememberedCollection(name) { + return cached + } + let collection = try await requireClient().collection(named: name) + lock.withLock { cachedCollections[name] = collection } + return collection + } + + /// A write moves rows between pages and may add payload keys, so both caches start over. + func forgetAfterWrite() { + lock.withLock { cachedCollections.removeAll() } + cursors.removeAll() + } + + func defaultExportQuery(table: String) -> String? { + QdrantOperations.encodeExport(collection: table) + } + + func dropObjectStatement(name: String, objectType: String, schema: String?, cascade: Bool) -> String? { + QdrantOperations.deleteCollection(named: name, objectType: objectType) + } + + func truncateTableStatements(table: String, schema: String?, cascade: Bool) -> [String]? { + QdrantOperations.deleteAllPoints(collection: table).map { [$0] } + } +} diff --git a/apple/TablePro/Assets.xcassets/qdrant-icon.imageset/Contents.json b/apple/TablePro/Assets.xcassets/qdrant-icon.imageset/Contents.json new file mode 100644 index 0000000000..f9d32664df --- /dev/null +++ b/apple/TablePro/Assets.xcassets/qdrant-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images": [ + { + "filename": "qdrant.svg", + "idiom": "universal" + } + ], + "info": { + "author": "xcode", + "version": 1 + }, + "properties": { + "preserves-vector-representation": true, + "template-rendering-intent": "template" + } +} diff --git a/apple/TablePro/Assets.xcassets/qdrant-icon.imageset/qdrant.svg b/apple/TablePro/Assets.xcassets/qdrant-icon.imageset/qdrant.svg new file mode 100644 index 0000000000..41257b1911 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/qdrant-icon.imageset/qdrant.svg @@ -0,0 +1 @@ +Qdrant diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift new file mode 100644 index 0000000000..d48af46cf2 --- /dev/null +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift @@ -0,0 +1,120 @@ +import Foundation +import TableProPluginKit + +extension PluginMetadataRegistry { + func qdrantPluginDefaults() -> [(typeId: String, snapshot: PluginMetadataSnapshot)] { + [ + ("Qdrant", PluginMetadataSnapshot( + displayName: "Qdrant", iconName: "qdrant-icon", defaultPort: 6_333, + requiresAuthentication: false, supportsForeignKeys: false, supportsSchemaEditing: false, + isDownloadable: true, primaryUrlScheme: "", parameterStyle: .questionMark, + navigationModel: .standard, explainVariants: [], pathFieldRole: .database, + supportsHealthMonitor: true, urlSchemes: [], postConnectActions: [], + brandColorHex: "#DC244C", + queryLanguageName: "REST", editorLanguage: .javascript, + connectionMode: .network, supportsDatabaseSwitching: false, + capabilities: PluginMetadataSnapshot.CapabilityFlags( + supportsSchemaSwitching: false, + supportsImport: false, + supportsExport: true, + supportsSSH: true, + supportsSSL: true, + supportsCascadeDrop: false, + supportsForeignKeyDisable: false, + supportsReadOnlyMode: true, + supportsQueryProgress: false, + requiresReconnectForDatabaseSwitch: false, + supportsDropDatabase: false, + supportsAddColumn: false, + supportsModifyColumn: false, + supportsDropColumn: false, + supportsAddIndex: false, + supportsDropIndex: false, + supportsModifyPrimaryKey: false, + supportsOpportunisticTLS: false, + supportsPrincipalConnectionLimit: false + ), + schema: PluginMetadataSnapshot.SchemaInfo( + defaultSchemaName: "", + defaultGroupName: "default", + tableEntityName: "Collections", + containerEntityName: "Cluster", + defaultPrimaryKeyColumn: "id", + immutableColumns: ["id", "vector"], + systemDatabaseNames: [], + systemSchemaNames: [], + fileExtensions: [], + databaseGroupingStrategy: .flat, + structureColumnFields: [.name, .type, .nullable] + ), + editor: PluginMetadataSnapshot.EditorConfig( + sqlDialect: nil, + statementCompletions: qdrantCompletions, + columnTypesByCategory: qdrantColumnTypes + ), + connection: PluginMetadataSnapshot.ConnectionConfig( + additionalConnectionFields: qdrantConnectionFields(), + category: .document, + tagline: String(localized: "Open-source vector search engine"), + hidesBuiltInPassword: true, + hidesBuiltInDatabase: true + ) + )) + ] + } +} + +private let qdrantCompletions: [CompletionEntry] = [ + CompletionEntry(label: "GET /collections", insertText: "GET /collections"), + CompletionEntry(label: "GET /collections/{name}", insertText: "GET /collections/docs"), + CompletionEntry( + label: "POST /points/scroll", + insertText: """ + POST /collections/docs/points/scroll + {"limit": 10, "with_payload": true} + """ + ), + CompletionEntry( + label: "POST /points/query", + insertText: """ + POST /collections/docs/points/query + {"query": [0.1, 0.2, 0.3, 0.4], "limit": 10, "with_payload": true} + """ + ), + CompletionEntry( + label: "POST /points/count", + insertText: """ + POST /collections/docs/points/count + {"exact": true} + """ + ) +] + +private let qdrantColumnTypes: [String: [String]] = [ + "Text": ["keyword", "text", "string", "uuid"], + "Numeric": ["integer", "float"], + "Boolean": ["bool"], + "Date": ["datetime"], + "Structured": ["geo", "object", "array", "json"], + "Vector": ["vector", "sparse_vector"] +] + +private func qdrantConnectionFields() -> [ConnectionField] { + [ + ConnectionField( + id: "qdApiKey", + label: String(localized: "API Key"), + placeholder: "Qdrant API key", + fieldType: .secure, + section: .authentication, + hidesPassword: true + ).withHidesUsername(true), + ConnectionField( + id: "qdSkipTLSVerify", + label: String(localized: "Skip TLS Verification"), + defaultValue: "false", + fieldType: .toggle, + section: .advanced + ) + ] +} diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift index 1df518f949..3f1f086029 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift @@ -1046,7 +1046,7 @@ extension PluginMetadataRegistry { + cloudPluginDefaults() + elasticsearchPluginDefaults() + surrealDBPluginDefaults() + kafkaPluginDefaults() + typesensePluginDefaults() + r2SQLPluginDefaults() + weaviatePluginDefaults() + hanaPluginDefaults() + firebasePluginDefaults() + bigtablePluginDefaults() - + trinoPluginDefaults() + + trinoPluginDefaults() + qdrantPluginDefaults() } // swiftlint:enable function_body_length } diff --git a/apple/TablePro/Core/Services/Execution/ExternalStatementGate.swift b/apple/TablePro/Core/Services/Execution/ExternalStatementGate.swift index dbc89e62f4..81e1d0ffe4 100644 --- a/apple/TablePro/Core/Services/Execution/ExternalStatementGate.swift +++ b/apple/TablePro/Core/Services/Execution/ExternalStatementGate.swift @@ -176,6 +176,7 @@ internal enum ExternalStatementGate { "ELASTICSEARCH_EXPORT:", "ELASTICSEARCH_SEARCH:", "ELASTICSEARCH_WRITE:", "ETCD_COUNT:", "ETCD_RANGE:", "ETCD_REFUSAL:", "FIREBASE_BROWSE:", + "QDRANT_BROWSE:", "QDRANT_EXPORT:", "SPANNER_BROWSE:", "TYPESENSE_EXPORT:", "TYPESENSE_SEARCH:", "TYPESENSE_WRITE:", "WEAVIATE_EXPORT:", "WEAVIATE_SEARCH:", "WEAVIATE_WRITE:", diff --git a/apple/TablePro/Core/Utilities/SQL/CatalogChangeClassifier.swift b/apple/TablePro/Core/Utilities/SQL/CatalogChangeClassifier.swift index 662751fd98..de983a7598 100644 --- a/apple/TablePro/Core/Utilities/SQL/CatalogChangeClassifier.swift +++ b/apple/TablePro/Core/Utilities/SQL/CatalogChangeClassifier.swift @@ -52,6 +52,6 @@ enum CatalogChangeClassifier { } private static let documentStoreTypes: Set = [ - .mongodb, .elasticsearch, .typesense, .weaviate, .etcd, .firestore, .firebaseRealtimeDatabase, .bigtable + .mongodb, .elasticsearch, .typesense, .weaviate, .qdrant, .etcd, .firestore, .firebaseRealtimeDatabase, .bigtable ] } diff --git a/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift b/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift index 16b070d1cb..d004213774 100644 --- a/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift +++ b/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift @@ -1013,6 +1013,8 @@ private extension QueryClassifier { return worstRequest(typesenseClassification) case .weaviate: return worstRequest(weaviateClassification) + case .qdrant: + return worstRequest(qdrantClassification) case .firestore, .firebaseRealtimeDatabase: return worstRequest { firebaseClassification($0, databaseType: databaseType) } case .bigtable: @@ -1202,6 +1204,44 @@ private extension QueryClassifier { return QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: false) } + /// POSTs under `/collections//` that only read. Retrieving points by id is a POST to `points`. + static let qdrantReadRoutes: Set<[String]> = [ + ["POINTS"], ["POINTS", "SCROLL"], ["POINTS", "SEARCH"], ["POINTS", "QUERY"], ["POINTS", "COUNT"], + ["POINTS", "RECOMMEND"], ["POINTS", "DISCOVER"], ["POINTS", "FACET"], + ["POINTS", "SEARCH", "BATCH"], ["POINTS", "QUERY", "BATCH"], ["POINTS", "RECOMMEND", "BATCH"], + ["POINTS", "DISCOVER", "BATCH"], ["POINTS", "SEARCH", "GROUPS"], ["POINTS", "QUERY", "GROUPS"], + ["POINTS", "RECOMMEND", "GROUPS"], ["POINTS", "SEARCH", "MATRIX", "PAIRS"], + ["POINTS", "SEARCH", "MATRIX", "OFFSETS"], + ] + + /// A snapshot is written to the server's disk, and recovering one fetches a URL from the server. + /// A batch can carry deletes, and a POST ending in `delete` or `clear` removes points or payload. + static func qdrantClassification(_ trimmed: String) -> QueryClassification { + if trimmed.hasPrefix("QDRANT_BROWSE:") || trimmed.hasPrefix("QDRANT_EXPORT:") { + return .safe + } + let (verb, path) = typesenseRequestLine(trimmed) + guard !verb.isEmpty else { return .safe } + guard let route = typesenseRoute(of: path) else { + return QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: true) + } + let reachesFilesystem = route.contains("SNAPSHOTS") + if verb == "GET" || verb == "HEAD" { + return QueryClassification(tier: .safe, reachesFilesystemOrExecutesCode: reachesFilesystem) + } + let isCollectionRoute = route.count >= 3 && route[0] == "COLLECTIONS" + let subroute = isCollectionRoute ? Array(route.dropFirst(2)) : [] + if verb == "POST", isCollectionRoute, qdrantReadRoutes.contains(subroute) { + return QueryClassification(tier: .safe, reachesFilesystemOrExecutesCode: reachesFilesystem) + } + let removes = verb == "DELETE" + || route.last == "DELETE" || route.last == "CLEAR" || subroute == ["POINTS", "BATCH"] + return QueryClassification( + tier: removes ? .destructive : .write, + reachesFilesystemOrExecutesCode: reachesFilesystem + ) + } + /// A grid page is a tagged spec and anything else a request line read the way the driver reads /// it. A write that leaves a node holding nothing deletes it, so it ranks with DELETE. static func firebaseClassification(_ trimmed: String, databaseType: DatabaseType) -> QueryClassification { diff --git a/apple/TablePro/Models/Connection/DatabaseType.swift b/apple/TablePro/Models/Connection/DatabaseType.swift index bce99781a3..201ed09ebf 100644 --- a/apple/TablePro/Models/Connection/DatabaseType.swift +++ b/apple/TablePro/Models/Connection/DatabaseType.swift @@ -53,6 +53,7 @@ extension DatabaseType { static let trino = DatabaseType(rawValue: "Trino") static let presto = DatabaseType(rawValue: "Presto") static let weaviate = DatabaseType(rawValue: "Weaviate") + static let qdrant = DatabaseType(rawValue: "Qdrant") static let bigtable = DatabaseType(rawValue: "Bigtable") static let firestore = DatabaseType(rawValue: "Firestore") static let firebaseRealtimeDatabase = DatabaseType(rawValue: "Firebase Realtime Database") diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index 4b999ea5d9..580a625c79 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -3571,6 +3571,14 @@ } } }, + "%@ holds booleans, so it matches only true or false, not %@." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "%@ holds numbers, and %@ is not a number." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "%@ holds text in some rows and documents or arrays in others, so this value could be either. Change it with a query." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -3969,6 +3977,10 @@ } } }, + "%@ is not a point id. Qdrant ids are unsigned integers or UUIDs." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "%@ is not a Redis database index." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -4624,6 +4636,10 @@ } } }, + "%@ on %@ needs a full-text payload index on that field. Without one Qdrant matches nothing." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "%@ on branch %@" : { "localizations" : { "en" : { @@ -16271,6 +16287,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "A raw filter for Qdrant is a JSON condition, such as {\"key\": \"city\", \"match\": {\"value\": \"Berlin\"}}." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "A raw Firestore filter is a JSON Filter object, such as %@." : { "extractionState" : "manual", "generatesSymbol" : false @@ -71712,6 +71732,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "Enter a request like GET /collections or POST /collections/docs/points/query." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Enter a request like: GET /collections" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -121465,6 +121489,10 @@ } } }, + "Not connected to Qdrant." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Not connected to Redis" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -138925,6 +138953,34 @@ } } }, + "Qdrant answered a count without a number." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Qdrant answered with something other than HTTP." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Qdrant cannot filter on %@." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Qdrant cannot filter with %@." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Qdrant compares only numbers and dates with %@, and %@ holds neither." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Qdrant does not support views." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Qdrant returned HTTP %d." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Queries" : { "comment" : "A section header for the list of queries in the favorites tab.", "isCommentAutoGenerated" : true, diff --git a/apple/TableProTests/Core/Execution/ExternalStatementGatePrivateRequestTests.swift b/apple/TableProTests/Core/Execution/ExternalStatementGatePrivateRequestTests.swift index 1e7e0924a5..5db44d6b98 100644 --- a/apple/TableProTests/Core/Execution/ExternalStatementGatePrivateRequestTests.swift +++ b/apple/TableProTests/Core/Execution/ExternalStatementGatePrivateRequestTests.swift @@ -75,6 +75,8 @@ struct ExternalStatementGatePrivateRequestTests { (.weaviate, "WEAVIATE_SEARCH:"), (.weaviate, "WEAVIATE_WRITE:"), (.weaviate, "WEAVIATE_EXPORT:"), + (.qdrant, "QDRANT_BROWSE:"), + (.qdrant, "QDRANT_EXPORT:"), (.etcd, "ETCD_RANGE:"), (.etcd, "ETCD_COUNT:"), (.etcd, "ETCD_REFUSAL:"), diff --git a/apple/TableProTests/Core/Plugins/PluginMetadataRegistryTypeCountTests.swift b/apple/TableProTests/Core/Plugins/PluginMetadataRegistryTypeCountTests.swift index e5d6ddaacd..e6a78c3db6 100644 --- a/apple/TableProTests/Core/Plugins/PluginMetadataRegistryTypeCountTests.swift +++ b/apple/TableProTests/Core/Plugins/PluginMetadataRegistryTypeCountTests.swift @@ -10,7 +10,7 @@ struct PluginMetadataRegistryTypeCountTests { "CockroachDB", "Dameng", "Databend", "DuckDB", "DynamoDB", "Elasticsearch", "etcd", "Kafka", "KingbaseES", "libSQL", "MariaDB", "MongoDB", "MySQL", "Oracle", "PGlite", "PostgreSQL", "Redis", "Redshift", "SAP HANA", "ScyllaDB", "Snowflake", "Spanner", "SQL Server", "SQLite", "SurrealDB", "Teradata", "TiDB", - "Trino", "Presto", "Turso", "Typesense", "Weaviate", "OceanBase", "Firestore", "Firebase Realtime Database", "Bigtable" + "Trino", "Presto", "Turso", "Typesense", "Weaviate", "Qdrant", "OceanBase", "Firestore", "Firebase Realtime Database", "Bigtable" ] private static func builtInTypeIds() -> Set { diff --git a/apple/TableProTests/Core/Plugins/PortForwardCapabilityTests.swift b/apple/TableProTests/Core/Plugins/PortForwardCapabilityTests.swift index dabb529605..89ce819b76 100644 --- a/apple/TableProTests/Core/Plugins/PortForwardCapabilityTests.swift +++ b/apple/TableProTests/Core/Plugins/PortForwardCapabilityTests.swift @@ -33,7 +33,7 @@ struct PortForwardCapabilityTests { @Test( "a self-hosted HTTP engine offers every transport that forwards a port", - arguments: [DatabaseType.weaviate, .typesense, .elasticsearch] + arguments: [DatabaseType.weaviate, .qdrant, .typesense, .elasticsearch] ) func httpEnginesOfferPortForwards(type: DatabaseType) { #expect(manager.supportsSSH(for: type)) diff --git a/apple/TableProTests/Core/Services/Query/AllTablesListingTests.swift b/apple/TableProTests/Core/Services/Query/AllTablesListingTests.swift index d5dabb8050..3d055d399c 100644 --- a/apple/TableProTests/Core/Services/Query/AllTablesListingTests.swift +++ b/apple/TableProTests/Core/Services/Query/AllTablesListingTests.swift @@ -28,7 +28,7 @@ struct AllTablesListingTests { } @Test("An engine that only highlights like MongoDB or Redis gets no shell command", arguments: [ - DatabaseType.elasticsearch, .typesense, .weaviate, .etcd, .cassandra + DatabaseType.elasticsearch, .typesense, .weaviate, .qdrant, .etcd, .cassandra ]) func noListingForOtherEngines(databaseType: DatabaseType) { #expect(AllTablesListing.statement(databaseType: databaseType, pluginListing: nil) == nil) diff --git a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift index 1261179ed8..0c8880901d 100644 --- a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift @@ -899,6 +899,33 @@ struct QueryClassifierNonSqlTests { #expect(!QueryClassifier.isWriteQuery(search, databaseType: .weaviate)) } + @Test("Qdrant reads are safe, POST searches read, and point deletes are destructive") + func qdrantTiers() { + #expect(!QueryClassifier.isWriteQuery("QDRANT_BROWSE:e30=", databaseType: .qdrant)) + #expect(!QueryClassifier.isWriteQuery("GET /collections", databaseType: .qdrant)) + #expect(!QueryClassifier.isWriteQuery("POST /collections/docs/points/query\n{\"limit\": 3}", databaseType: .qdrant)) + #expect(!QueryClassifier.isWriteQuery("POST /collections/docs/points/scroll {}", databaseType: .qdrant)) + #expect(!QueryClassifier.isWriteQuery("POST /collections/docs/points {\"ids\": [1]}", databaseType: .qdrant)) + #expect(QueryClassifier.classifyTier("PUT /collections/docs/points?wait=true\n{}", databaseType: .qdrant) == .write) + #expect(QueryClassifier.classifyTier("POST /collections/docs/points/payload?wait=true {}", databaseType: .qdrant) == .write) + #expect(QueryClassifier.classifyTier("POST /collections/docs/points/delete?wait=true {}", databaseType: .qdrant) == .destructive) + #expect(QueryClassifier.classifyTier("POST /collections/docs/points/payload/clear {}", databaseType: .qdrant) == .destructive) + #expect(QueryClassifier.classifyTier("POST /collections/docs/points/batch {}", databaseType: .qdrant) == .destructive) + #expect(QueryClassifier.classifyTier("DELETE /collections/docs", databaseType: .qdrant) == .destructive) + #expect(QueryClassifier.isWriteQuery("POST /collections/docs/points/query/../delete {}", databaseType: .qdrant)) + } + + @Test("A Qdrant snapshot reaches the server's disk") + func qdrantSnapshots() { + #expect(QueryClassifier.reachesFilesystemOrExecutesCode( + "POST /collections/docs/snapshots", databaseType: .qdrant + )) + #expect(QueryClassifier.reachesFilesystemOrExecutesCode( + "PUT /collections/docs/snapshots/recover {\"location\": \"file:///etc\"}", databaseType: .qdrant + )) + #expect(!QueryClassifier.reachesFilesystemOrExecutesCode("GET /collections", databaseType: .qdrant)) + } + @Test("Bigtable request lines are tiered by their verb, and a PUT that nulls a cell is destructive") func bigtableTiers() { #expect(!QueryClassifier.isWriteQuery("BIGTABLE_BROWSE:e30=", databaseType: .bigtable)) diff --git a/apple/TableProTests/Models/Query/QueryTabBaseQueryTests.swift b/apple/TableProTests/Models/Query/QueryTabBaseQueryTests.swift index 3e05253cff..2816488850 100644 --- a/apple/TableProTests/Models/Query/QueryTabBaseQueryTests.swift +++ b/apple/TableProTests/Models/Query/QueryTabBaseQueryTests.swift @@ -51,7 +51,7 @@ struct QueryTabBaseQueryTests { /// Their plugins build the browse query. Without one the tab must not be handed another engine's /// command: either the plugin builds it or the build throws. @Test("An engine that highlights like MongoDB or Redis is not browsed with their commands", arguments: [ - DatabaseType.elasticsearch, .typesense, .weaviate, .etcd + DatabaseType.elasticsearch, .typesense, .weaviate, .qdrant, .etcd ]) func otherEnginesNeverGetShellBrowseQueries(databaseType: DatabaseType) { let query = try? QueryTab.buildBaseTableQuery(tableName: "products", databaseType: databaseType) diff --git a/apple/TableProTests/Views/Main/QueryParameterDetectionTests.swift b/apple/TableProTests/Views/Main/QueryParameterDetectionTests.swift index 1cd4d7e40c..1e16db3a95 100644 --- a/apple/TableProTests/Views/Main/QueryParameterDetectionTests.swift +++ b/apple/TableProTests/Views/Main/QueryParameterDetectionTests.swift @@ -13,7 +13,7 @@ extension GlobalStateSuites { "Only SQL reads :name as a bind parameter", arguments: [ (DatabaseType.mysql, true), (.postgresql, true), (.sqlite, true), - (.elasticsearch, false), (.typesense, false), (.weaviate, false), (.mongodb, false), + (.elasticsearch, false), (.typesense, false), (.weaviate, false), (.qdrant, false), (.mongodb, false), (.redis, false), (.etcd, false), (.surrealdb, false), (.kafka, false) ] ) diff --git a/apple/TableProTests/Views/SQLReviewSheetTests.swift b/apple/TableProTests/Views/SQLReviewSheetTests.swift index cf7c7ce451..5a3a86bf19 100644 --- a/apple/TableProTests/Views/SQLReviewSheetTests.swift +++ b/apple/TableProTests/Views/SQLReviewSheetTests.swift @@ -159,7 +159,7 @@ struct SQLReviewSheetTests { } @Test("A request body keeps a stored $oid as the JSON it is", arguments: [ - DatabaseType.elasticsearch, .typesense, .weaviate + DatabaseType.elasticsearch, .typesense, .weaviate, .qdrant ]) func requestBodyKeepsExtendedJSON(databaseType: DatabaseType) { let statement = """ diff --git a/apple/project.yml b/apple/project.yml index cbf751b9d2..809fb3d359 100644 --- a/apple/project.yml +++ b/apple/project.yml @@ -1253,6 +1253,15 @@ targets: - package: TableProCore product: TableProLogRedaction + QdrantDriverPlugin: + templates: [DriverPlugin] + templateAttributes: + folder: QdrantDriverPlugin + principalClass: QdrantPlugin + dependencies: + - package: TableProCore + product: TableProQdrantCore + WeaviateDriverPlugin: templates: [DriverPlugin] templateAttributes: @@ -1306,6 +1315,7 @@ aggregateTargets: - MongoDBDriver - OracleDriver - ParquetExport + - QdrantDriverPlugin - SnowflakeDriverPlugin - SpannerDriverPlugin - SurrealDBDriverPlugin diff --git a/apple/scripts/probes/check-qdrant.sh b/apple/scripts/probes/check-qdrant.sh new file mode 100755 index 0000000000..9b459f86c7 --- /dev/null +++ b/apple/scripts/probes/check-qdrant.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# +# Run TableProQdrantCore against a live Qdrant node: cursor paging, every filter kind, order_by, +# set-payload, insert_only inserts with no vector, delete, truncate, export and drop. The probe +# creates and drops a collection named tablepro_probe. +# +# Usage: +# scripts/probes/check-qdrant.sh [host] [port] +# +# QDRANT_API_KEY sends a key. Start a node with: +# docker run -d --name tablepro-qdrant-probe -p 6333:6333 qdrant/qdrant +# +# Exits 3 when no node answers. + +set -euo pipefail + +HOST="${1:-127.0.0.1}" +PORT="${2:-6333}" +HERE="$(cd "$(dirname "$0")" && pwd)" +CORE="$(cd "$HERE/../../Packages/TableProCore" && pwd)" + +if ! curl -fsS -o /dev/null ${QDRANT_API_KEY:+-H "api-key: $QDRANT_API_KEY"} "http://$HOST:$PORT/collections"; then + echo "No Qdrant node answers at $HOST:$PORT." >&2 + exit 3 +fi + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT +mkdir -p "$WORK/Sources/probe" +cp "$HERE/qdrant-probe.swift" "$WORK/Sources/probe/main.swift" +cat > "$WORK/Package.swift" < Int { + let f = try QdrantFilterBuilder.filter(specs, logicMode: mode, collection: collection) + return try await client.count(collection: "tablepro_probe", filter: f) +} +check(try await filtered([QdrantFilterSpec(column: "city", op: "=", value: "even")]) == 125, "match keyword") +check(try await filtered([QdrantFilterSpec(column: "n", op: "BETWEEN", value: "10,19", secondValue: "19")]) == 10, "between") +check(try await filtered([QdrantFilterSpec(column: "n", op: ">", value: "240"), QdrantFilterSpec(column: "city", op: "!=", value: "even")]) == 5, "and with not") +check(try await filtered([QdrantFilterSpec(column: "n", op: "<", value: "3"), QdrantFilterSpec(column: "n", op: ">", value: "248")], "OR") == 4, "or") +check(try await filtered([QdrantFilterSpec(column: "body", op: "CONTAINS", value: "hello")]) == 250, "full text") +check(try await filtered([QdrantFilterSpec(column: "id", op: "IN", value: "1,2,3")]) == 3, "has_id") +check(try await filtered([QdrantFilterSpec(column: "n", op: "IN", value: "1,2,300")]) == 2, "match any") +check(try await filtered([QdrantFilterSpec(column: "missing", op: "IS NULL", value: "")]) == 0, "is_null") +check(try await filtered([QdrantFilterSpec(column: "missing", op: "IS EMPTY", value: "")]) == 250, "is_empty") + +let sortReq = QdrantBrowseRequest(collection: "tablepro_probe", offset: 5, limit: 3, sort: QdrantSortSpec(column: "n", ascending: false), filters: [], logicMode: "AND") +let sorted = try await client.query(collection: "tablepro_probe", body: QdrantBrowseQuery.queryBody(for: sortReq, filter: nil, sort: sortReq.sort!)) +check(sorted.map { $0.idText ?? "" } == ["245", "244", "243"], "order_by with offset") + +let batch = QdrantStatementGenerator.generate(collection: collection, changes: [ + QdrantTrackedChange(kind: .update, id: "1", cellChanges: [QdrantCellChange(column: "city", newText: "edited"), QdrantCellChange(column: "n", newText: "1000")]), + QdrantTrackedChange(kind: .delete, id: "2"), + QdrantTrackedChange(kind: .insert, id: "999", values: ["city": "new"]), + QdrantTrackedChange(kind: .insert, id: "3", values: ["city": "clobber"]) +]) +for request in batch.requests { + let text = QdrantWriteCodec.encode(request) + _ = try await client.execute(QdrantConsoleParser.parse(text)!) +} +let after = try await client.query(collection: "tablepro_probe", body: ["query": NSNull(), "filter": ["must": [["has_id": [1, 2, 3, 999]]]], "with_payload": true, "limit": 10] as [String: Any]) +let byID = Dictionary(uniqueKeysWithValues: after.map { ($0.idText ?? "", $0) }) +check(byID["1"]?.cell(for: "city") == "edited" && byID["1"]?.cell(for: "n") == "1000", "set payload typed") +check(byID["2"] == nil, "delete") +check(byID["999"]?.cell(for: "city") == "new", "insert") +check(byID["3"]?.cell(for: "city") == "odd", "insert_only keeps existing") + +let console = try await client.execute(QdrantConsoleParser.parse("POST /collections/tablepro_probe/points/query\n{\"query\": [1, 0], \"limit\": 2, \"with_vector\": true, \"with_payload\": true}")!) +let hits = QdrantPoint.list(fromResponse: console.json) ?? [] +print("query columns", QdrantPoint.columns(of: hits, collection: collection), hits.first?.row(columns: QdrantPoint.columns(of: hits, collection: collection)) ?? []) +check(hits.count == 2 && hits[0].score != nil, "vector query has scores") + +_ = try await client.execute(QdrantConsoleRequest(method: "PUT", path: "/collections/tablepro_probe/points?wait=true", body: #"{"points":[{"id":5000,"vector":[1,1],"payload":{"late":true,"id":"ext-1"}}]}"#)) +var widened = collection +for try await page in client.pointPages(collection: "tablepro_probe", pageSize: 60, withVector: false) { + check(page.allSatisfy { $0.vector == nil }, "payload-only pass has no vectors") + widened = widened.including(page) +} +let wcols = QdrantSchema.columns(for: widened).map(\.name) +print("export columns", wcols) +check(wcols.contains("late") && wcols.contains("payload.id"), "late and reserved keys found") +let lateFilter = try QdrantFilterBuilder.filter([QdrantFilterSpec(column: "payload.id", op: "=", value: "ext-1")], logicMode: "AND", collection: widened) +check(try await client.count(collection: "tablepro_probe", filter: lateFilter) == 1, "filter on payload.id") +var exported = 0 +for try await page in client.pointPages(collection: "tablepro_probe", pageSize: 60) { exported += page.count } +check(exported == 251, "export all") + +let truncate = QdrantOperations.deleteAllPoints(collection: "tablepro_probe")! +_ = try await client.execute(QdrantConsoleParser.parse(truncate)!) +check(try await client.count(collection: "tablepro_probe", filter: nil) == 0, "truncate") +_ = try await client.execute(QdrantConsoleParser.parse(QdrantOperations.deleteCollection(named: "tablepro_probe", objectType: "TABLE")!)!) +check(!(try await client.collectionNames().contains("tablepro_probe")), "drop") +do { + _ = try await client.collection(named: "tablepro_probe_missing") + check(false, "missing collection reports an error") +} catch { + check(error.localizedDescription.contains("doesn't exist"), "missing collection reports an error") +} +exit(failures == 0 ? 0 : 1) From 3526d12096d7490c0d89a1d64b1f01699bcf5dda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Nam=20Long?= <26674308+J2TeamNNL@users.noreply.github.com> Date: Sun, 1 Nov 2026 08:55:16 +0700 Subject: [PATCH 02/20] docs(changelog): add the Qdrant driver entry --- apple/changelog.d/3972.added.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 apple/changelog.d/3972.added.md diff --git a/apple/changelog.d/3972.added.md b/apple/changelog.d/3972.added.md new file mode 100644 index 0000000000..7ece2164bb --- /dev/null +++ b/apple/changelog.d/3972.added.md @@ -0,0 +1 @@ +Qdrant support: browse collections, filter points, edit payloads, and run REST requests such as vector queries. From 6937ba5de11cf7109b4a6bfad691e212de87b170 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Nam=20Long?= <26674308+J2TeamNNL@users.noreply.github.com> Date: Sun, 1 Nov 2026 09:23:34 +0700 Subject: [PATCH 03/20] chore(l10n): add the Qdrant tagline to the catalog --- apple/TablePro/Resources/Localizable.xcstrings | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index 580a625c79..d518e71edc 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -127980,6 +127980,9 @@ } } } + }, + "Open-source vector search engine" : { + }, "Opening the connection" : { "localizations" : { From 20a58ff35291275298466ca83ecf276878181944 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 20:39:17 +0700 Subject: [PATCH 04/20] refactor(plugin-qdrant): type writes from the grid, fix filters, sort and TLS Measured against Qdrant 1.20.0 (apple/scripts/probes/check-qdrant.sh). --- .../TableProQdrantCore/QdrantBrowse.swift | 115 +++++++ .../QdrantConnectionSettings.swift | 10 +- .../TableProQdrantCore/QdrantConsole.swift | 113 +++++-- .../TableProQdrantCore/QdrantError.swift | 14 +- .../QdrantFilterBuilder.swift | 264 +++++++++------- .../TableProQdrantCore/QdrantJSON.swift | 34 +- .../TableProQdrantCore/QdrantPoint.swift | 39 +-- .../TableProQdrantCore/QdrantQuery.swift | 294 ------------------ .../TableProQdrantCore/QdrantRowWrites.swift | 164 ++++++++++ .../TableProQdrantCore/QdrantSchema.swift | 162 +++++++--- .../TableProQdrantCore/QdrantValue.swift | 118 +++++++ .../Wire/QdrantClient.swift | 85 ++--- .../Wire/QdrantScrollPager.swift | 83 +++-- .../Wire/QdrantSortedPager.swift | 60 ++++ .../Wire/QdrantTransport.swift | 111 +------ .../WeaviateGraphQL.swift | 8 +- .../QdrantClientTests.swift | 102 ++++-- .../QdrantConsoleTests.swift | 92 ++++++ .../QdrantFilterBuilderTests.swift | 85 +++-- .../QdrantRowWriteTests.swift | 117 +++++++ .../QdrantSchemaAndWriteTests.swift | 198 ------------ .../QdrantSchemaTests.swift | 128 ++++++++ .../QdrantTestSupport.swift | 32 +- .../QdrantDriverPlugin/QdrantPlugin.swift | 20 +- .../QdrantPluginDriver+Execution.swift | 47 ++- .../QdrantPluginDriver+Metadata.swift | 140 +++++---- .../QdrantPluginDriver.swift | 14 +- .../QdrantURLSessionTransport.swift | 222 +++++++++++++ .../Core/DataWrite/DataWriteExecutor.swift | 17 +- ...luginMetadataRegistry+QdrantDefaults.swift | 21 +- .../Core/Utilities/SQL/QueryClassifier.swift | 63 ++-- .../DataWriteExecutorLookupTests.swift | 26 +- .../SQL/QueryClassifierHardeningTests.swift | 7 + apple/changelog.d/3972.added.md | 2 +- apple/project.yml | 4 +- apple/scripts/probes/check-qdrant.sh | 6 +- apple/scripts/probes/qdrant-probe.swift | 201 +++++++----- 37 files changed, 1960 insertions(+), 1258 deletions(-) create mode 100644 apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantBrowse.swift delete mode 100644 apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantQuery.swift create mode 100644 apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift create mode 100644 apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantValue.swift create mode 100644 apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantSortedPager.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantConsoleTests.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift delete mode 100644 apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaAndWriteTests.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift create mode 100644 apple/Plugins/QdrantDriverPlugin/QdrantURLSessionTransport.swift diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantBrowse.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantBrowse.swift new file mode 100644 index 0000000000..c2c5ddefab --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantBrowse.swift @@ -0,0 +1,115 @@ +import Foundation + +public struct QdrantSortSpec: Sendable, Equatable { + public let column: String + public let ascending: Bool + + public init(column: String, ascending: Bool) { + self.column = column + self.ascending = ascending + } +} + +public struct QdrantBrowseRequest: Sendable, Equatable { + public let collection: String + public let offset: Int + public let limit: Int + public let sorts: [QdrantSortSpec] + public let filters: [QdrantFilterSpec] + public let logicMode: String + + public init( + collection: String, + offset: Int, + limit: Int, + sorts: [QdrantSortSpec], + filters: [QdrantFilterSpec], + logicMode: String + ) { + self.collection = collection + self.offset = max(offset, 0) + self.limit = max(limit, 1) + self.sorts = sorts + self.filters = filters + self.logicMode = logicMode + } +} + +// The page is built before the driver knows the field types the filter JSON depends on, so it travels +// from `buildFilteredQuery` to `execute` as tagged text. +public enum QdrantBrowseQuery { + public static let tag = "QDRANT_BROWSE:" + + public static func encode(_ request: QdrantBrowseRequest) -> String { + let payload: [String: Any] = [ + "collection": request.collection, + "offset": request.offset, + "limit": request.limit, + "logicMode": request.logicMode, + "sorts": request.sorts.map { ["column": $0.column, "ascending": $0.ascending] }, + "filters": request.filters.map { filter -> [String: Any] in + var encoded: [String: Any] = ["column": filter.column, "op": filter.op, "value": filter.value] + if let secondValue = filter.secondValue { + encoded["secondValue"] = secondValue + } + return encoded + } + ] + let body = (try? QdrantJSON.data(payload)) ?? Data() + return tag + body.base64EncodedString() + } + + public static func parse(_ query: String) -> QdrantBrowseRequest? { + guard query.hasPrefix(tag), + let data = Data(base64Encoded: String(query.dropFirst(tag.count))), + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let collection = json["collection"] as? String + else { return nil } + let sorts = (json["sorts"] as? [[String: Any]] ?? []).compactMap { item -> QdrantSortSpec? in + guard let column = item["column"] as? String else { return nil } + return QdrantSortSpec(column: column, ascending: item["ascending"] as? Bool ?? true) + } + let filters = (json["filters"] as? [[String: Any]] ?? []).compactMap { item -> QdrantFilterSpec? in + guard let column = item["column"] as? String, let op = item["op"] as? String else { return nil } + return QdrantFilterSpec( + column: column, + op: op, + value: item["value"] as? String ?? "", + secondValue: item["secondValue"] as? String + ) + } + return QdrantBrowseRequest( + collection: collection, + offset: json["offset"] as? Int ?? 0, + limit: json["limit"] as? Int ?? 100, + sorts: sorts, + filters: filters, + logicMode: json["logicMode"] as? String ?? "AND" + ) + } +} + +public enum QdrantSortPlan: Sendable, Equatable { + case byID + case byPayload(column: String, key: String, ascending: Bool) + + public static func resolve(_ sorts: [QdrantSortSpec]) throws -> QdrantSortPlan { + guard let sort = sorts.first else { return .byID } + guard sorts.count == 1 else { + throw QdrantError.unsupported(String(localized: "Qdrant sorts by one field at a time. Remove the sort from the other columns.")) + } + if sort.column == QdrantSchema.idColumn { + guard sort.ascending else { + throw QdrantError.unsupported(String(localized: "Qdrant lists points by id in ascending order only.")) + } + return .byID + } + guard !QdrantSchema.isVectorColumn(sort.column), sort.column != QdrantSchema.scoreColumn else { + throw QdrantError.unsupported(String(format: String(localized: "Qdrant cannot sort by %@."), sort.column)) + } + guard let key = QdrantKeyPath.quoted(QdrantSchema.payloadKey(forColumn: sort.column)) else { + throw QdrantFilterError.keyNotAddressable(column: sort.column) + } + return .byPayload(column: sort.column, key: key, ascending: sort.ascending) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConnectionSettings.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConnectionSettings.swift index 97a914a012..63e3d523e6 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConnectionSettings.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConnectionSettings.swift @@ -2,7 +2,6 @@ import Foundation public enum QdrantFieldID { public static let apiKey = "qdApiKey" - public static let skipTLSVerify = "qdSkipTLSVerify" } public struct QdrantConnectionSettings: Sendable, Equatable { @@ -12,14 +11,12 @@ public struct QdrantConnectionSettings: Sendable, Equatable { public let port: Int public let usesTLS: Bool public let apiKey: String - public let skipTLSVerify: Bool - public init(host: String, port: Int, usesTLS: Bool, apiKey: String, skipTLSVerify: Bool) { + public init(host: String, port: Int, usesTLS: Bool, apiKey: String) { self.host = host.trimmingCharacters(in: .whitespacesAndNewlines) self.port = port self.usesTLS = usesTLS self.apiKey = apiKey.trimmingCharacters(in: .whitespacesAndNewlines) - self.skipTLSVerify = skipTLSVerify } public static func parse( @@ -33,15 +30,12 @@ public struct QdrantConnectionSettings: Sendable, Equatable { host: resolvedHost.isEmpty ? "localhost" : resolvedHost, port: port > 0 ? port : defaultPort, usesTLS: usesTLS, - apiKey: fields[QdrantFieldID.apiKey] ?? "", - skipTLSVerify: fields[QdrantFieldID.skipTLSVerify] == "true" + apiKey: fields[QdrantFieldID.apiKey] ?? "" ) _ = try settings.baseURL() return settings } - /// Qdrant reads the key from `api-key`, and also accepts it as a bearer token; the dedicated - /// header is the one both a self-hosted node and Qdrant Cloud document. public var headers: [String: String] { apiKey.isEmpty ? [:] : ["api-key": apiKey] } diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConsole.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConsole.swift index 0ceee6f41e..4bf7255aee 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConsole.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConsole.swift @@ -11,38 +11,24 @@ public struct QdrantConsoleRequest: Sendable, Equatable { self.body = body } - /// The POST endpoints that only read. Retrieving points by id is a POST to `/points` itself. - static let readSuffixes: [[String]] = [ - ["points"], ["points", "scroll"], ["points", "search"], ["points", "query"], ["points", "count"], - ["points", "recommend"], ["points", "discover"], ["points", "facet"], - ["points", "search", "batch"], ["points", "query", "batch"], ["points", "recommend", "batch"], - ["points", "discover", "batch"], ["points", "search", "groups"], ["points", "query", "groups"], - ["points", "recommend", "groups"], ["points", "search", "matrix", "pairs"], - ["points", "search", "matrix", "offsets"] - ] - - public var isRead: Bool { - if method == "GET" || method == "HEAD" { return true } - guard method == "POST" else { return false } - let route = path.split(separator: "?", maxSplits: 1).first.map(String.init) ?? path - let segments = route.split(separator: "/").map { $0.lowercased() } - guard segments.count >= 3, segments[0] == "collections" else { return false } - return Self.readSuffixes.contains { Array(segments.dropFirst(2)) == $0 } + // What the review sheet shows, the classifier tiers and the driver runs are this one string. + public var text: String { + guard let body else { return "\(method) \(path)" } + return "\(method) \(path)\n\(body)" } + + public var kind: QdrantRequestKind { QdrantRequestKind.of(method: method, path: path) } } -/// One REST request typed in the editor: `METHOD /path`, then an optional JSON body on the same -/// line or on the lines below it. +// `METHOD /path`, then an optional JSON body that may start on the request line and go on below it. public enum QdrantConsoleParser { static let methods: Set = ["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD"] public static func parse(_ input: String) -> QdrantConsoleRequest? { let trimmed = input.trimmingCharacters(in: .whitespacesAndNewlines) - let lines = trimmed.split(separator: "\n", maxSplits: 1, omittingEmptySubsequences: false) - guard let header = lines.first else { return nil } - let following = lines.count > 1 - ? String(lines[1]).trimmingCharacters(in: .whitespacesAndNewlines) - : "" + let newline = trimmed.firstIndex(where: \.isNewline) + let header = newline.map { trimmed[..<$0] } ?? trimmed[...] + let following = newline.map { trimmed[trimmed.index(after: $0)...] } ?? "" let parts = header.split(maxSplits: 2, omittingEmptySubsequences: true, whereSeparator: \.isWhitespace) guard parts.count >= 2 else { return nil } @@ -50,12 +36,60 @@ public enum QdrantConsoleParser { guard methods.contains(method) else { return nil } let path = String(parts[1]) guard path.hasPrefix("/"), !path.hasPrefix("//") else { return nil } - let inline = parts.count > 2 ? String(parts[2]).trimmingCharacters(in: .whitespacesAndNewlines) : "" - let body = inline.isEmpty ? following : inline + let inline = parts.count > 2 ? parts[2] : "" + let body = [inline, following] + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + .joined(separator: "\n") return QdrantConsoleRequest(method: method, path: path, body: body.isEmpty ? nil : body) } } +public struct QdrantRequestKind: Sendable, Equatable { + public enum Tier: Sendable, Equatable { + case read + case write + case remove + } + + public let tier: Tier + // A snapshot is written to the server's disk, and recovering one fetches a URL from the server. + public let touchesServerFiles: Bool + + public init(tier: Tier, touchesServerFiles: Bool) { + self.tier = tier + self.touchesServerFiles = touchesServerFiles + } + + // The POSTs under `/collections//` that only read. Retrieving points by id is a POST to `points`. + static let readRoutes: Set<[String]> = [ + ["points"], ["points", "scroll"], ["points", "search"], ["points", "query"], ["points", "count"], + ["points", "recommend"], ["points", "discover"], ["facet"], + ["points", "search", "batch"], ["points", "query", "batch"], ["points", "recommend", "batch"], + ["points", "discover", "batch"], ["points", "search", "groups"], ["points", "query", "groups"], + ["points", "recommend", "groups"], ["points", "search", "matrix", "pairs"], + ["points", "search", "matrix", "offsets"] + ] + + public static func of(method: String, path: String) -> QdrantRequestKind { + let verb = method.uppercased() + guard let route = QdrantPathEncoding.segments(of: path) else { + return QdrantRequestKind(tier: .write, touchesServerFiles: true) + } + let touchesServerFiles = route.contains("snapshots") + if verb == "GET" || verb == "HEAD" { + return QdrantRequestKind(tier: .read, touchesServerFiles: touchesServerFiles) + } + let subroute = route.count >= 3 && route[0] == "collections" ? Array(route.dropFirst(2)) : [] + if verb == "POST", readRoutes.contains(subroute) { + return QdrantRequestKind(tier: .read, touchesServerFiles: touchesServerFiles) + } + // A batch can carry deletes, and a POST ending in `delete` or `clear` removes points or payload. + let removes = verb == "DELETE" || route.last == "delete" || route.last == "clear" || subroute == ["points", "batch"] + return QdrantRequestKind(tier: removes ? .remove : .write, touchesServerFiles: touchesServerFiles) + } +} + public enum QdrantPathEncoding { private static let allowed: CharacterSet = { var set = CharacterSet.alphanumerics @@ -67,10 +101,25 @@ public enum QdrantPathEncoding { value.addingPercentEncoding(withAllowedCharacters: allowed) ?? value } - /// The path keeps its own query string (`?wait=true`): assigning it whole to `URLComponents.path` - /// would encode the `?` and send the request to a path that does not exist. + // Decoded and lowercased; nil for a dot segment, which would move the request to a route other than + // the one it reads as. + public static func segments(of path: String) -> [String]? { + guard let requested = URLComponents(string: path) else { return nil } + let segments = requested.percentEncodedPath.split(separator: "/").map { String($0) } + var decoded: [String] = [] + for segment in segments { + guard let text = segment.removingPercentEncoding, text != ".", text != "..", !text.contains("/") else { + return nil + } + decoded.append(text.lowercased()) + } + return decoded + } + + // Assigning the path whole to `URLComponents.path` would encode the `?` of `?wait=true`. public static func resolve(_ path: String, against base: URL) -> URL? { guard path.hasPrefix("/"), !path.hasPrefix("//"), !path.contains("://"), + segments(of: path) != nil, var components = URLComponents(url: base, resolvingAgainstBaseURL: false), let requested = URLComponents(string: path) else { return nil } @@ -83,14 +132,12 @@ public enum QdrantPathEncoding { public enum QdrantOperations { public static let exportTag = "QDRANT_EXPORT:" - /// The console request that removes one collection. Plain request text, so the confirmation shows - /// it verbatim and the query classifier tiers it destructive by its verb. public static func deleteCollection(named name: String, objectType: String) -> String? { guard objectType.uppercased() == "TABLE", !name.isEmpty else { return nil } return "DELETE /collections/\(QdrantPathEncoding.segment(name))" } - /// An empty filter matches every point, which keeps the collection and its config. + // An empty filter matches every point and keeps the collection with its config and indexes. public static func deleteAllPoints(collection: String) -> String? { guard !collection.isEmpty else { return nil } return "POST /collections/\(QdrantPathEncoding.segment(collection))/points/delete?wait=true\n{\"filter\":{}}" @@ -108,4 +155,8 @@ public enum QdrantOperations { else { return nil } return collection } + + public static func isTagged(_ query: String) -> Bool { + query.hasPrefix(QdrantBrowseQuery.tag) || query.hasPrefix(exportTag) + } } diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantError.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantError.swift index e97312d379..a38693615b 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantError.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantError.swift @@ -7,12 +7,13 @@ public enum QdrantError: Error, LocalizedError, Equatable, Sendable { case authentication(String) case api(status: Int, message: String) case malformedResponse(String) + case unsupported(String) case cancelled public var errorDescription: String? { switch self { case .configuration(let detail), .transport(let detail), .malformedResponse(let detail), - .authentication(let detail): + .authentication(let detail), .unsupported(let detail): return detail case .notConnected: return String(localized: "Not connected to Qdrant.") @@ -32,7 +33,7 @@ public enum QdrantError: Error, LocalizedError, Equatable, Sendable { return .api(status: status, message: message) } - /// Qdrant puts the reason in `status.error`; a proxy in front of it may answer with plain text. + // A proxy in front of Qdrant may answer with plain text instead of `status.error`. public static func apiMessage(from body: Data) -> String? { guard let json = try? JSONSerialization.jsonObject(with: body) else { let text = String(data: body, encoding: .utf8)?.trimmingCharacters(in: .whitespacesAndNewlines) @@ -43,11 +44,10 @@ public enum QdrantError: Error, LocalizedError, Equatable, Sendable { let error = status["error"] as? String, !error.isEmpty { return error } - if let error = object["error"] as? String, !error.isEmpty { - return error - } - if let message = object["message"] as? String, !message.isEmpty { - return message + for key in ["error", "message"] { + if let text = object[key] as? String, !text.isEmpty { + return text + } } return nil } diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift index 7fa0dc57d8..660b2191dc 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift @@ -14,16 +14,16 @@ public struct QdrantFilterSpec: Sendable, Equatable { } } -public enum QdrantFilterError: Error, LocalizedError, Equatable { +public enum QdrantFilterError: Error, LocalizedError, Equatable, Sendable { case unsupportedOperator(String) case missingUpperBound(column: String) - case notANumber(column: String, value: String) - case notABoolean(column: String, value: String) case notAPointID(value: String) case emptyList(column: String) case comparisonNeedsNumberOrDate(column: String, op: String) - case textMatchNeedsTextIndex(column: String, op: String) + case textMatchNeedsText(column: String, type: String) + case wholeValueNotMatchable(column: String, type: String) case vectorNotFilterable(column: String) + case keyNotAddressable(column: String) case rawFilterNotAnObject public var errorDescription: String? { @@ -32,13 +32,6 @@ public enum QdrantFilterError: Error, LocalizedError, Equatable { return String(format: String(localized: "Qdrant cannot filter with %@."), op) case .missingUpperBound(let column): return String(format: String(localized: "BETWEEN on %@ needs an upper bound."), column) - case .notANumber(let column, let value): - return String(format: String(localized: "%@ holds numbers, and %@ is not a number."), column, value) - case .notABoolean(let column, let value): - return String( - format: String(localized: "%@ holds booleans, so it matches only true or false, not %@."), - column, value - ) case .notAPointID(let value): return String( format: String(localized: "%@ is not a point id. Qdrant ids are unsigned integers or UUIDs."), @@ -51,21 +44,28 @@ public enum QdrantFilterError: Error, LocalizedError, Equatable { format: String(localized: "Qdrant compares only numbers and dates with %@, and %@ holds neither."), op, column ) - case .textMatchNeedsTextIndex(let column, let op): + case .textMatchNeedsText(let column, let type): + return String(format: String(localized: "%@ holds %@ values, and Qdrant matches words only in text."), column, type) + case .wholeValueNotMatchable(let column, let type): return String( - format: String(localized: "%@ on %@ needs a full-text payload index on that field. Without one Qdrant matches nothing."), - op, column + format: String(localized: "Qdrant cannot match a whole %@ value in %@. Use a raw filter on one of its fields."), + type, column ) case .vectorNotFilterable(let column): return String(format: String(localized: "Qdrant cannot filter on %@."), column) + case .keyNotAddressable(let column): + return String( + format: String(localized: "Qdrant cannot filter on %@, because a key holding a double quote has no path it accepts."), + column + ) case .rawFilterNotAnObject: return String(localized: "A raw filter for Qdrant is a JSON condition, such as {\"key\": \"city\", \"match\": {\"value\": \"Berlin\"}}.") } } } -/// Turns the filter bar's rows into Qdrant's filter JSON. AND rows go under `must` and OR rows under -/// `should`; a negated row is a nested filter with `must_not`, which Qdrant accepts as a condition. +// AND rows go under `must` and OR rows under `should`. A negated row is a nested filter with `must_not`, +// which, like `$ne` in a document store, also matches a point the key is missing from. public enum QdrantFilterBuilder { public static let rawFilterColumn = "__RAW__" @@ -76,8 +76,7 @@ public enum QdrantFilterBuilder { ) throws -> [String: Any]? { guard !filters.isEmpty else { return nil } let conditions = try filters.map { try condition(for: $0, collection: collection) } - let clause = logicMode.uppercased() == "OR" ? "should" : "must" - return [clause: conditions] + return [logicMode.uppercased() == "OR" ? "should" : "must": conditions] } public static func condition(for filter: QdrantFilterSpec, collection: QdrantCollection) throws -> [String: Any] { @@ -89,142 +88,172 @@ public enum QdrantFilterBuilder { } let op = filter.op.uppercased() if filter.column == QdrantSchema.idColumn { - return try idCondition(op: op, filter: filter) + return try idCondition(op: op, value: filter.value) + } + guard let key = QdrantKeyPath.quoted(QdrantSchema.payloadKey(forColumn: filter.column)) else { + throw QdrantFilterError.keyNotAddressable(column: filter.column) } - let type = collection.fieldType(of: filter.column) ?? .json - let column = QdrantSchema.payloadKey(forColumn: filter.column) + let field = Field(column: filter.column, key: key, type: collection.fieldType(ofColumn: filter.column) ?? .json) switch op { case "=": - return try equality(column: column, value: filter.value, type: type) + return try field.equality(filter.value) case "!=", "<>": - return negated(try equality(column: column, value: filter.value, type: type)) + return negated([try field.equality(filter.value)]) case ">", ">=", "<", "<=": - return try range(column: column, type: type, op: op, bounds: [rangeKey(op): filter.value]) + return try field.range(op: op, bounds: [rangeKey(op): filter.value]) case "BETWEEN": guard let upper = filter.secondValue, !upper.trimmingCharacters(in: .whitespaces).isEmpty else { - throw QdrantFilterError.missingUpperBound(column: column) + throw QdrantFilterError.missingUpperBound(column: filter.column) } - let lower = lowerBound(of: filter.value, upperBound: upper) - return try range(column: column, type: type, op: op, bounds: ["gte": lower, "lte": upper]) + return try field.range(op: op, bounds: ["gte": lowerBound(of: filter.value, upperBound: upper), "lte": upper]) case "IN": - return ["key": column, "match": ["any": try list(filter.value, column: column, type: type)]] + return try field.anyOf(list(filter.value, column: filter.column)) case "NOT IN": - return ["key": column, "match": ["except": try list(filter.value, column: column, type: type)]] + return negated(try field.anyOfConditions(list(filter.value, column: filter.column))) case "CONTAINS": - return try textMatch(column: column, value: filter.value, type: type, op: op) + return try field.textMatch(filter.value) case "NOT CONTAINS": - return negated(try textMatch(column: column, value: filter.value, type: type, op: op)) + return negated([try field.textMatch(filter.value)]) case "IS NULL": - return ["is_null": ["key": column]] + return field.isEmpty case "IS NOT NULL": - return negated(["is_null": ["key": column]]) + return negated([field.isEmpty]) case "IS EMPTY": - return ["is_empty": ["key": column]] + return field.isBlank case "IS NOT EMPTY": - return negated(["is_empty": ["key": column]]) + return negated([field.isBlank]) default: throw QdrantFilterError.unsupportedOperator(op) } } - private static func raw(_ text: String) throws -> [String: Any] { - guard let data = text.data(using: .utf8), - let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any] - else { throw QdrantFilterError.rawFilterNotAnObject } - return object - } + private struct Field { + let column: String + let key: String + let type: QdrantFieldType - private static func idCondition(op: String, filter: QdrantFilterSpec) throws -> [String: Any] { - func ids(_ text: String) throws -> [Any] { - let parts = text.split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty } - guard !parts.isEmpty else { throw QdrantFilterError.emptyList(column: QdrantSchema.idColumn) } - return try parts.map { part in - guard let id = QdrantPointID.json(from: part) else { throw QdrantFilterError.notAPointID(value: part) } - return id + func equality(_ value: String) throws -> [String: Any] { + switch type { + case .keyword, .text, .string, .uuid: + return match(value) + case .integer: + guard let number = QdrantValue.integer(value) else { + throw QdrantValueError.notAnInteger(column: column, value: value) + } + return match(NSNumber(value: number)) + case .bool: + guard let flag = QdrantValue.bool(value) else { + throw QdrantValueError.notABoolean(column: column, value: value) + } + return match(flag) + case .float, .datetime: + return try range(op: "=", bounds: ["gte": value, "lte": value]) + case .array, .json: + let trimmed = value.trimmingCharacters(in: .whitespaces) + if let number = QdrantValue.integer(trimmed) { return match(NSNumber(value: number)) } + if trimmed == "true" || trimmed == "false" { return match(trimmed == "true") } + if QdrantValue.number(trimmed) != nil { return try range(op: "=", bounds: ["gte": value, "lte": value]) } + return match(value) + case .geo, .object: + throw QdrantFilterError.wholeValueNotMatchable(column: column, type: type.rawValue) } } - switch op { - case "=", "IN": - return ["has_id": try ids(filter.value)] - case "!=", "<>", "NOT IN": - return negated(["has_id": try ids(filter.value)]) - default: - throw QdrantFilterError.unsupportedOperator(op) + + // Qdrant's `match.any` takes strings alone or integers alone; anything else is one `should`. + func anyOf(_ values: [String]) throws -> [String: Any] { + let conditions = try anyOfConditions(values) + let matched = conditions.compactMap { ($0["match"] as? [String: Any])?["value"] } + if matched.count == conditions.count { + if matched.allSatisfy({ $0 is String }) || matched.allSatisfy(isInteger) { + return ["key": key, "match": ["any": matched]] + } + } + return ["should": conditions] } - } - /// `match.value` takes a string, an integer or a boolean. A float has no exact match in Qdrant, - /// so equality on one is the closed range around it. - private static func equality(column: String, value: String, type: QdrantFieldType) throws -> [String: Any] { - switch type { - case .float, .datetime: - return try range(column: column, type: type, op: "=", bounds: ["gte": value, "lte": value]) - default: - return ["key": column, "match": ["value": try scalar(value, column: column, type: type)]] + func anyOfConditions(_ values: [String]) throws -> [[String: Any]] { + try values.map(equality) } - } - private static func range( - column: String, - type: QdrantFieldType, - op: String, - bounds: [String: String] - ) throws -> [String: Any] { - guard type.isOrdered || type == .json else { - throw QdrantFilterError.comparisonNeedsNumberOrDate(column: column, op: op) - } - var range: [String: Any] = [:] - for (key, text) in bounds { - let trimmed = text.trimmingCharacters(in: .whitespaces) - if type == .datetime { - range[key] = trimmed - } else if let number = Double(trimmed) { - range[key] = number - } else if type == .json { - range[key] = trimmed - } else { - throw QdrantFilterError.notANumber(column: column, value: text) + func range(op: String, bounds: [String: String]) throws -> [String: Any] { + guard type.isOrdered || type == .json || type == .array else { + throw QdrantFilterError.comparisonNeedsNumberOrDate(column: column, op: op) + } + var range: [String: Any] = [:] + for (bound, text) in bounds { + let trimmed = text.trimmingCharacters(in: .whitespaces) + if type == .datetime { + range[bound] = trimmed + } else if let number = QdrantValue.number(trimmed) { + range[bound] = NSNumber(value: number) + } else if type.isOrdered { + throw QdrantValueError.notANumber(column: column, value: text) + } else { + range[bound] = trimmed + } } + return ["key": key, "range": range] + } + + // Without a full-text index Qdrant still matches the words, lowercased. + func textMatch(_ value: String) throws -> [String: Any] { + guard type.holdsText else { + throw QdrantFilterError.textMatchNeedsText(column: column, type: type.rawValue) + } + return ["key": key, "match": ["text": value]] } - return ["key": column, "range": range] - } - private static func textMatch(column: String, value: String, type: QdrantFieldType, op: String) throws -> [String: Any] { - guard type == .text else { - throw QdrantFilterError.textMatchNeedsTextIndex(column: column, op: op) + // `is_empty` covers a missing key, null and []; `is_null` would miss the missing key the grid shows as NULL. + var isEmpty: [String: Any] { + ["is_empty": ["key": key]] + } + + var isBlank: [String: Any] { + guard type.holdsText else { return isEmpty } + return ["should": [isEmpty, match("")]] + } + + private func match(_ value: Any) -> [String: Any] { + ["key": key, "match": ["value": value]] + } + + private func isInteger(_ value: Any) -> Bool { + guard let number = value as? NSNumber else { return false } + return !QdrantJSON.isBoolean(number) && !CFNumberIsFloatType(number) } - return ["key": column, "match": ["text": value]] } - private static func list(_ value: String, column: String, type: QdrantFieldType) throws -> [Any] { - let parts = value.split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty } - guard !parts.isEmpty else { throw QdrantFilterError.emptyList(column: column) } - return try parts.map { try scalar($0, column: column, type: type) } + private static func raw(_ text: String) throws -> [String: Any] { + guard let object = QdrantJSON.parse(text) as? [String: Any] else { + throw QdrantFilterError.rawFilterNotAnObject + } + return object } - private static func scalar(_ value: String, column: String, type: QdrantFieldType) throws -> Any { - let trimmed = value.trimmingCharacters(in: .whitespaces) - switch type { - case .integer: - guard let number = Int64(trimmed) else { throw QdrantFilterError.notANumber(column: column, value: value) } - return NSNumber(value: number) - case .bool: - switch trimmed.lowercased() { - case "true", "1": return true - case "false", "0": return false - default: throw QdrantFilterError.notABoolean(column: column, value: value) - } - case .json: - if let number = Int64(trimmed) { return NSNumber(value: number) } - if trimmed == "true" { return true } - if trimmed == "false" { return false } - return value + private static func idCondition(op: String, value: String) throws -> [String: Any] { + let parts = value.split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty } + guard !parts.isEmpty else { throw QdrantFilterError.emptyList(column: QdrantSchema.idColumn) } + let ids = try parts.map { part -> Any in + guard let id = QdrantPointID.json(from: part) else { throw QdrantFilterError.notAPointID(value: part) } + return id + } + switch op { + case "=", "IN": + return ["has_id": ids] + case "!=", "<>", "NOT IN": + return negated([["has_id": ids]]) default: - return value + throw QdrantFilterError.unsupportedOperator(op) } } + private static func list(_ value: String, column: String) throws -> [String] { + let parts = value.split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty } + guard !parts.isEmpty else { throw QdrantFilterError.emptyList(column: column) } + return parts + } + private static func rangeKey(_ op: String) -> String { switch op { case ">": return "gt" @@ -240,7 +269,12 @@ public enum QdrantFilterBuilder { return String(joinedValue.dropLast(suffix.count)) } - private static func negated(_ condition: [String: Any]) -> [String: Any] { - ["must_not": [condition]] + static func negated(_ conditions: [[String: Any]]) -> [String: Any] { + ["must_not": conditions] + } + + static func allOf(_ filter: [String: Any]?, _ condition: [String: Any]) -> [String: Any] { + guard let filter else { return ["must": [condition]] } + return ["must": [filter, condition]] } } diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantJSON.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantJSON.swift index bee609b383..57192f0dec 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantJSON.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantJSON.swift @@ -1,10 +1,6 @@ import Foundation public enum QdrantJSON { - public static func dictionary(_ value: Any?) -> [String: Any]? { - value as? [String: Any] - } - public static func data(_ object: Any, pretty: Bool = false) throws -> Data { guard JSONSerialization.isValidJSONObject(object) else { throw QdrantError.malformedResponse(String(localized: "Request body is not valid JSON.")) @@ -20,6 +16,11 @@ public enum QdrantJSON { String(data: try data(object, pretty: pretty), encoding: .utf8) ?? "{}" } + public static func parse(_ text: String) -> Any? { + guard let data = text.trimmingCharacters(in: .whitespacesAndNewlines).data(using: .utf8) else { return nil } + return try? JSONSerialization.jsonObject(with: data, options: [.fragmentsAllowed]) + } + public static func isBoolean(_ number: NSNumber) -> Bool { CFGetTypeID(number) == CFBooleanGetTypeID() } @@ -43,29 +44,4 @@ public enum QdrantJSON { return String(describing: value as Any) } } - - /// A payload is schemaless, so a cell is written back as the type its column holds. Parsing every - /// cell as JSON would turn a string field holding `42` into a number and break a `match` on it. - public static func parsedValue(_ text: String, fieldType: QdrantFieldType) -> Any { - let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) - switch fieldType { - case .keyword, .text, .string, .uuid, .datetime: - return text - case .integer: - return Int64(trimmed) ?? Double(trimmed).map { $0 as Any } ?? text - case .float: - return Double(trimmed) ?? text - case .bool: - switch trimmed.lowercased() { - case "true": return true - case "false": return false - default: return text - } - case .geo, .object, .array, .json: - guard let data = trimmed.data(using: .utf8), - let parsed = try? JSONSerialization.jsonObject(with: data, options: [.fragmentsAllowed]) - else { return text } - return parsed - } - } } diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantPoint.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantPoint.swift index c3846c1466..b22222e959 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantPoint.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantPoint.swift @@ -1,8 +1,7 @@ import Foundation -/// A point id is an unsigned integer or a UUID, and Qdrant refuses one in the other's form, so the -/// grid's text is turned back into the kind it came from. public enum QdrantPointID { + // Qdrant refuses an integer id spelled as a string and the other way round. public static func json(from text: String) -> Any? { let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { return nil } @@ -12,16 +11,12 @@ public enum QdrantPointID { guard UUID(uuidString: trimmed) != nil else { return nil } return trimmed.lowercased() } - - public static func text(_ value: Any?) -> String? { - QdrantJSON.displayText(value) - } } public struct QdrantPoint: @unchecked Sendable { public let id: Any? public let payload: [String: Any] - /// `nil` when the request left vectors out; an array for an unnamed vector; a map by name otherwise. + // Nil when the request left vectors out, an array for an unnamed vector alone, a map by name otherwise. public let vector: Any? public let score: Double? @@ -32,7 +27,7 @@ public struct QdrantPoint: @unchecked Sendable { self.score = score } - public var idText: String? { QdrantPointID.text(id) } + public var idText: String? { QdrantJSON.displayText(id) } public var payloadKeys: [String] { payload.keys.sorted() } @@ -47,8 +42,7 @@ public struct QdrantPoint: @unchecked Sendable { ) } - /// The point list of any read endpoint: `scroll` and `query` wrap it in `points`, retrieve and the - /// older `search` answer the array itself. `nil` when the result is not a list of points. + // `scroll` and `query` wrap the list in `points`; retrieve and `search` answer the array itself. public static func list(fromResponse json: Any?) -> [QdrantPoint]? { guard let result = (json as? [String: Any])?["result"] else { return nil } let items: [Any] @@ -63,13 +57,9 @@ public struct QdrantPoint: @unchecked Sendable { return points.count == items.count ? points : nil } - public func vectorValue(for column: String) -> Any? { - guard let vector else { return nil } - if column == QdrantSchema.vectorColumn { - if vector is [Any] { return vector } - return (vector as? [String: Any])?[""] - } - let name = String(column.dropFirst(QdrantSchema.vectorColumn.count + 1)) + public func vectorValue(forColumn column: String) -> Any? { + guard let vector, let name = QdrantSchema.vectorName(ofColumn: column) else { return nil } + if name.isEmpty, vector is [Any] { return vector } return (vector as? [String: Any])?[name] } @@ -78,7 +68,7 @@ public struct QdrantPoint: @unchecked Sendable { return idText } if QdrantSchema.isVectorColumn(column) { - return QdrantJSON.displayText(vectorValue(for: column)) + return QdrantJSON.displayText(vectorValue(forColumn: column)) } if column == QdrantSchema.scoreColumn { return score.map { String($0) } @@ -90,7 +80,6 @@ public struct QdrantPoint: @unchecked Sendable { columns.map(cell(for:)) } - /// The columns a console result shows: what came back, in the order the collection lists it. public static func columns(of points: [QdrantPoint], collection: QdrantCollection?) -> [String] { var columns = [QdrantSchema.idColumn] if points.contains(where: { $0.score != nil }) { @@ -101,18 +90,14 @@ public struct QdrantPoint: @unchecked Sendable { columns += declared.filter { returned.contains($0) && !columns.contains($0) } columns += returned.subtracting(declared).subtracting(columns).sorted() - var vectorColumns: [String] = [] + var vectorColumns: Set = [] for point in points { if point.vector is [Any] { - vectorColumns.append(QdrantSchema.vectorColumn) + vectorColumns.insert(QdrantSchema.vectorColumn) } else if let named = point.vector as? [String: Any] { - vectorColumns += named.keys.map { "\(QdrantSchema.vectorColumn).\($0)" } + vectorColumns.formUnion(named.keys.map { QdrantVectorSpec(name: $0, kind: .dense(size: nil)).column }) } } - var seen = Set(columns) - for column in vectorColumns.sorted() where seen.insert(column).inserted { - columns.append(column) - } - return columns + return columns + vectorColumns.subtracting(columns).sorted() } } diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantQuery.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantQuery.swift deleted file mode 100644 index ad7f1d9c09..0000000000 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantQuery.swift +++ /dev/null @@ -1,294 +0,0 @@ -import Foundation - -public struct QdrantSortSpec: Sendable, Equatable { - public let column: String - public let ascending: Bool - - public init(column: String, ascending: Bool) { - self.column = column - self.ascending = ascending - } -} - -public struct QdrantBrowseRequest: Sendable, Equatable { - public let collection: String - public let offset: Int - public let limit: Int - public let sort: QdrantSortSpec? - public let filters: [QdrantFilterSpec] - public let logicMode: String - - public init( - collection: String, - offset: Int, - limit: Int, - sort: QdrantSortSpec?, - filters: [QdrantFilterSpec], - logicMode: String - ) { - self.collection = collection - self.offset = max(offset, 0) - self.limit = max(limit, 1) - self.sort = sort - self.filters = filters - self.logicMode = logicMode - } -} - -/// A grid page travels from `buildFilteredQuery` to `execute` as tagged text, because the page is -/// built before the driver knows the collection's field types the filter JSON depends on. -public enum QdrantBrowseQuery { - public static let tag = "QDRANT_BROWSE:" - - public static func encode(_ request: QdrantBrowseRequest) -> String { - var payload: [String: Any] = [ - "collection": request.collection, - "offset": request.offset, - "limit": request.limit, - "logicMode": request.logicMode, - "filters": request.filters.map { filter -> [String: Any] in - var encoded: [String: Any] = ["column": filter.column, "op": filter.op, "value": filter.value] - if let secondValue = filter.secondValue { - encoded["secondValue"] = secondValue - } - return encoded - } - ] - if let sort = request.sort { - payload["sort"] = ["column": sort.column, "ascending": sort.ascending] - } - let body = (try? QdrantJSON.data(payload)) ?? Data() - return tag + body.base64EncodedString() - } - - public static func parse(_ query: String) -> QdrantBrowseRequest? { - guard query.hasPrefix(tag), - let data = Data(base64Encoded: String(query.dropFirst(tag.count))), - let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], - let collection = json["collection"] as? String - else { return nil } - let sort = (json["sort"] as? [String: Any]).flatMap { item -> QdrantSortSpec? in - guard let column = item["column"] as? String else { return nil } - return QdrantSortSpec(column: column, ascending: item["ascending"] as? Bool ?? true) - } - let filters = (json["filters"] as? [[String: Any]] ?? []).compactMap { item -> QdrantFilterSpec? in - guard let column = item["column"] as? String, let op = item["op"] as? String else { return nil } - return QdrantFilterSpec( - column: column, - op: op, - value: item["value"] as? String ?? "", - secondValue: item["secondValue"] as? String - ) - } - return QdrantBrowseRequest( - collection: collection, - offset: json["offset"] as? Int ?? 0, - limit: json["limit"] as? Int ?? 100, - sort: sort, - filters: filters, - logicMode: json["logicMode"] as? String ?? "AND" - ) - } - - public static func isTagged(_ query: String) -> Bool { - query.hasPrefix(tag) - } - - /// The body of one page. `order_by` is only on the query endpoint, which takes a numeric offset; - /// an unsorted page goes through `scroll`, which pages by point id instead. - public static func queryBody( - for request: QdrantBrowseRequest, - filter: [String: Any]?, - sort: QdrantSortSpec - ) -> [String: Any] { - var body: [String: Any] = [ - "query": ["order_by": [ - "key": QdrantSchema.payloadKey(forColumn: sort.column), - "direction": sort.ascending ? "asc" : "desc" - ]], - "limit": request.limit, - "offset": request.offset, - "with_payload": true, - "with_vector": true - ] - if let filter { - body["filter"] = filter - } - return body - } - - public static func scrollBody( - filter: [String: Any]?, - limit: Int, - cursor: Any?, - withData: Bool, - withVector: Bool? = nil - ) -> [String: Any] { - var body: [String: Any] = [ - "limit": limit, - "with_payload": withData, - "with_vector": withVector ?? withData - ] - if let filter { - body["filter"] = filter - } - if let cursor { - body["offset"] = cursor - } - return body - } -} - -public struct QdrantWriteRequest: Sendable, Equatable { - public let method: String - public let path: String - public let body: String? - - public init(method: String, path: String, body: String?) { - self.method = method - self.path = path - self.body = body - } -} - -/// A grid save is shown in the review sheet and runs as the request line the console would send, -/// so what the user approves, what the read-only gate classifies and what runs are one string. -public enum QdrantWriteCodec { - public static func encode(_ request: QdrantWriteRequest) -> String { - guard let body = request.body else { return "\(request.method) \(request.path)" } - return "\(request.method) \(request.path)\n\(body)" - } -} - -public struct QdrantCellChange: Sendable, Equatable { - public let column: String - public let newText: String? - - public init(column: String, newText: String?) { - self.column = column - self.newText = newText - } -} - -public struct QdrantTrackedChange: Sendable, Equatable { - public enum Kind: String, Sendable, Equatable { - case insert - case update - case delete - } - - public let kind: Kind - public let id: String? - public let values: [String: String?] - public let cellChanges: [QdrantCellChange] - - public init(kind: Kind, id: String?, values: [String: String?] = [:], cellChanges: [QdrantCellChange] = []) { - self.kind = kind - self.id = id - self.values = values - self.cellChanges = cellChanges - } -} - -public enum QdrantSkipReason: String, Sendable, Equatable { - case missingID - case invalidID - case noEditableColumns - case payloadNotEncodable -} - -public struct QdrantSkippedChange: Sendable, Equatable { - public let kind: QdrantTrackedChange.Kind - public let reason: QdrantSkipReason -} - -public struct QdrantWriteBatch: Sendable, Equatable { - public let requests: [QdrantWriteRequest] - public let skipped: [QdrantSkippedChange] -} - -public enum QdrantStatementGenerator { - /// `uuid` supplies the id of an inserted row the user left blank. - public static func generate( - collection: QdrantCollection, - changes: [QdrantTrackedChange], - uuid: () -> String = { UUID().uuidString.lowercased() } - ) -> QdrantWriteBatch { - let base = "/collections/\(QdrantPathEncoding.segment(collection.name))/points" - var requests: [QdrantWriteRequest] = [] - var skipped: [QdrantSkippedChange] = [] - - for change in changes { - switch outcome(for: change, collection: collection, base: base, uuid: uuid) { - case .success(let request): - requests.append(request) - case .failure(let reason): - skipped.append(QdrantSkippedChange(kind: change.kind, reason: reason)) - } - } - return QdrantWriteBatch(requests: requests, skipped: skipped) - } - - private enum Outcome { - case success(QdrantWriteRequest) - case failure(QdrantSkipReason) - } - - private static func outcome( - for change: QdrantTrackedChange, - collection: QdrantCollection, - base: String, - uuid: () -> String - ) -> Outcome { - let immutable = Set(QdrantSchema.immutableColumns(for: collection)) - switch change.kind { - case .insert: - let idText = change.id.flatMap { $0.isEmpty ? nil : $0 } ?? uuid() - guard let id = QdrantPointID.json(from: idText) else { return .failure(.invalidID) } - let payload = payloadObject( - change.values.filter { !immutable.contains($0.key) }, - collection: collection - ) - // `insert_only` keeps an id that already exists from being overwritten, vector and all. - let body: [String: Any] = [ - "points": [["id": id, "vector": [String: Any](), "payload": payload]], - "update_mode": "insert_only" - ] - return request("PUT", "\(base)?wait=true", body) - case .update: - guard let idText = change.id, !idText.isEmpty else { return .failure(.missingID) } - guard let id = QdrantPointID.json(from: idText) else { return .failure(.invalidID) } - var patch: [String: String?] = [:] - for cell in change.cellChanges where !immutable.contains(cell.column) { - patch[cell.column] = cell.newText - } - guard !patch.isEmpty else { return .failure(.noEditableColumns) } - let body: [String: Any] = ["payload": payloadObject(patch, collection: collection), "points": [id]] - return request("POST", "\(base)/payload?wait=true", body) - case .delete: - guard let idText = change.id, !idText.isEmpty else { return .failure(.missingID) } - guard let id = QdrantPointID.json(from: idText) else { return .failure(.invalidID) } - return request("POST", "\(base)/delete?wait=true", ["points": [id]]) - } - } - - private static func request(_ method: String, _ path: String, _ body: [String: Any]) -> Outcome { - guard let text = try? QdrantJSON.text(body) else { return .failure(.payloadNotEncodable) } - return .success(QdrantWriteRequest(method: method, path: path, body: text)) - } - - private static func payloadObject(_ values: [String: String?], collection: QdrantCollection) -> [String: Any] { - var payload: [String: Any] = [:] - for (column, text) in values { - guard let text else { - payload[QdrantSchema.payloadKey(forColumn: column)] = NSNull() - continue - } - payload[QdrantSchema.payloadKey(forColumn: column)] = QdrantJSON.parsedValue( - text, - fieldType: collection.fieldType(of: column) ?? .string - ) - } - return payload - } -} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift new file mode 100644 index 0000000000..2c6b4e6f41 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift @@ -0,0 +1,164 @@ +import Foundation + +public struct QdrantCell: Sendable, Equatable { + public let column: String + public let text: String? + + public init(column: String, text: String?) { + self.column = column + self.text = text + } +} + +public struct QdrantRowChange: Sendable, Equatable { + public enum Kind: Sendable, Equatable { + case insert + case update + case delete + } + + public let kind: Kind + public let rowIndex: Int + public let id: String? + // An insert's every column, or an update's changed cells. + public let cells: [QdrantCell] + + public init(kind: Kind, rowIndex: Int, id: String?, cells: [QdrantCell] = []) { + self.kind = kind + self.rowIndex = rowIndex + self.id = id + self.cells = cells + } +} + +public struct QdrantRowWrite: Sendable, Equatable { + public let request: QdrantConsoleRequest + public let rowIndex: Int + // A read that must find no point before the write runs: `insert_only` answers success and writes + // nothing when the id is taken. + public let mustFindNothing: QdrantConsoleRequest? +} + +public struct QdrantRowRefusal: Error, LocalizedError, Equatable, Sendable { + public let rowIndex: Int + public let reason: String + + public var errorDescription: String? { reason } +} + +public enum QdrantRowWriter { + public static func writes( + collection: String, + changes: [QdrantRowChange], + typeName: (String) -> String?, + newID: () -> String = { UUID().uuidString.lowercased() } + ) throws -> [QdrantRowWrite] { + let points = "/collections/\(QdrantPathEncoding.segment(collection))/points" + var writes: [QdrantRowWrite] = [] + for change in changes { + do { + writes += try self.writes(for: change, points: points, typeName: typeName, newID: newID) + } catch let error as QdrantValueError { + throw QdrantRowRefusal(rowIndex: change.rowIndex, reason: error.errorDescription ?? "") + } catch let reason as Reason { + throw QdrantRowRefusal(rowIndex: change.rowIndex, reason: reason.text) + } + } + return writes + } + + private struct Reason: Error { + let text: String + } + + private static func writes( + for change: QdrantRowChange, + points: String, + typeName: (String) -> String?, + newID: () -> String + ) throws -> [QdrantRowWrite] { + switch change.kind { + case .delete: + let id = try pointID(change.id) + return [try write("POST", "\(points)/delete?wait=true", ["points": [id]], change)] + case .insert: + let idText = change.cells.first { $0.column == QdrantSchema.idColumn }?.text.flatMap { $0.isEmpty ? nil : $0 } + let id = try pointID(idText ?? newID()) + let values = try typedValues(change.cells.filter { $0.text != nil }, typeName: typeName) + let point: [String: Any] = ["id": id, "vector": values.vectors, "payload": values.payload] + let lookup = try request("POST", points, ["ids": [id], "with_payload": false, "with_vector": false]) + return [QdrantRowWrite( + request: try request("PUT", "\(points)?wait=true", ["points": [point], "update_mode": "insert_only"]), + rowIndex: change.rowIndex, + mustFindNothing: lookup + )] + case .update: + let id = try pointID(change.id) + guard !change.cells.contains(where: { $0.column == QdrantSchema.idColumn }) else { + throw Reason(text: String(localized: "A point's id cannot change. Insert a new row with the new id instead.")) + } + let values = try typedValues(change.cells, typeName: typeName) + var writes: [QdrantRowWrite] = [] + if !values.payload.isEmpty { + writes.append(try write("POST", "\(points)/payload?wait=true", ["payload": values.payload, "points": [id]], change)) + } + if !values.vectors.isEmpty { + let body: [String: Any] = ["points": [["id": id, "vector": values.vectors]]] + writes.append(try write("PUT", "\(points)/vectors?wait=true", body, change)) + } + if !values.clearedVectors.isEmpty { + let body: [String: Any] = ["points": [id], "vector": values.clearedVectors] + writes.append(try write("POST", "\(points)/vectors/delete?wait=true", body, change)) + } + return writes + } + } + + private struct TypedValues { + var payload: [String: Any] = [:] + var vectors: [String: Any] = [:] + var clearedVectors: [String] = [] + } + + private static func typedValues(_ cells: [QdrantCell], typeName: (String) -> String?) throws -> TypedValues { + var values = TypedValues() + for cell in cells where cell.column != QdrantSchema.idColumn { + guard let type = typeName(cell.column) else { throw unknownType(cell.column) } + if let name = QdrantSchema.vectorName(ofColumn: cell.column) { + guard let kind = QdrantVectorSpec.kind(ofTypeName: type) else { throw unknownType(cell.column) } + if let text = cell.text { + values.vectors[name] = try QdrantValue.vector(text, kind: kind, column: cell.column) + } else { + values.clearedVectors.append(name) + } + continue + } + guard let fieldType = QdrantFieldType(rawValue: type.lowercased()) else { throw unknownType(cell.column) } + let key = QdrantSchema.payloadKey(forColumn: cell.column) + values.payload[key] = try cell.text.map { try QdrantValue.payload($0, type: fieldType, column: cell.column) } ?? NSNull() + } + return values + } + + private static func unknownType(_ column: String) -> Reason { + Reason(text: String(format: String(localized: "TablePro does not know what %@ holds. Reload the collection, then save again."), column)) + } + + private static func pointID(_ text: String?) throws -> Any { + guard let text, !text.isEmpty else { + throw Reason(text: String(localized: "This row has no point id, so Qdrant cannot tell which point to change.")) + } + guard let id = QdrantPointID.json(from: text) else { + throw Reason(text: String(QdrantFilterError.notAPointID(value: text).errorDescription ?? "")) + } + return id + } + + private static func write(_ method: String, _ path: String, _ body: [String: Any], _ change: QdrantRowChange) throws -> QdrantRowWrite { + QdrantRowWrite(request: try request(method, path, body), rowIndex: change.rowIndex, mustFindNothing: nil) + } + + private static func request(_ method: String, _ path: String, _ body: [String: Any]) throws -> QdrantConsoleRequest { + QdrantConsoleRequest(method: method, path: path, body: try QdrantJSON.text(body)) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift index e9d92af337..dc5eaed17e 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift @@ -1,7 +1,6 @@ import Foundation -/// The type a payload column holds. The first seven are Qdrant's payload index schemas; the rest -/// are read off sampled values for a field nobody indexed. +// The first seven are Qdrant's payload index schemas; the rest are read off sampled values. public enum QdrantFieldType: String, Sendable, Equatable, CaseIterable { case keyword case integer @@ -20,6 +19,13 @@ public enum QdrantFieldType: String, Sendable, Equatable, CaseIterable { self == .integer || self == .float || self == .datetime } + public var holdsText: Bool { + switch self { + case .keyword, .text, .string, .uuid, .array, .json: return true + default: return false + } + } + public static func indexed(_ schema: Any?) -> QdrantFieldType? { let name: String? if let object = schema as? [String: Any] { @@ -49,8 +55,6 @@ public enum QdrantFieldType: String, Sendable, Equatable, CaseIterable { } } - /// An integer column that also holds a fraction reads as a float; anything else that disagrees is - /// mixed JSON, written back as whatever the cell parses to. func merged(with other: QdrantFieldType) -> QdrantFieldType { if self == other { return self } if Set([self, other]) == [.integer, .float] { return .float } @@ -59,17 +63,51 @@ public enum QdrantFieldType: String, Sendable, Equatable, CaseIterable { } public struct QdrantVectorSpec: Sendable, Equatable { - /// `nil` for the collection's single unnamed vector. - public let name: String? - public let isSparse: Bool + public enum Kind: Sendable, Equatable { + case dense(size: Int?) + case multi + case sparse + } + + // Empty for the collection's unnamed vector, which Qdrant also accepts under the name "". + public let name: String + public let kind: Kind - public init(name: String?, isSparse: Bool = false) { + public init(name: String, kind: Kind) { self.name = name - self.isSparse = isSparse + self.kind = kind } public var column: String { - name.map { "\(QdrantSchema.vectorColumn).\($0)" } ?? QdrantSchema.vectorColumn + name.isEmpty ? QdrantSchema.vectorColumn : "\(QdrantSchema.vectorColumn).\(name)" + } + + public var typeName: String { + switch kind { + case .dense(let size?): return "vector(\(size))" + case .dense: return "vector" + case .multi: return "multivector" + case .sparse: return "sparse_vector" + } + } + + public static func kind(ofTypeName typeName: String) -> Kind? { + let lower = typeName.lowercased().trimmingCharacters(in: .whitespaces) + switch lower { + case "sparse_vector": return .sparse + case "multivector": return .multi + case "vector": return .dense(size: nil) + default: + guard lower.hasPrefix("vector("), lower.hasSuffix(")") else { return nil } + return .dense(size: Int(lower.dropFirst("vector(".count).dropLast())) + } + } + + static func dense(name: String, config: [String: Any]) -> QdrantVectorSpec { + if config["multivector_config"] is [String: Any] { + return QdrantVectorSpec(name: name, kind: .multi) + } + return QdrantVectorSpec(name: name, kind: .dense(size: config["size"] as? Int)) } } @@ -77,7 +115,7 @@ public struct QdrantPayloadField: Sendable, Equatable { public let name: String public let type: QdrantFieldType public let isIndexed: Bool - /// False while every value seen was null, so the first real value sets the type outright. + // False while every sampled value was null, so the first real value sets the type outright. public let hasTypedValue: Bool public init(name: String, type: QdrantFieldType, isIndexed: Bool, hasTypedValue: Bool = true) { @@ -88,10 +126,21 @@ public struct QdrantPayloadField: Sendable, Equatable { } } +public struct QdrantPayloadIndex: Sendable, Equatable { + public let path: String + public let type: String + + public init(path: String, type: String) { + self.path = path + self.type = type + } +} + public struct QdrantCollection: Sendable, Equatable { public let name: String public let vectors: [QdrantVectorSpec] public let payloadFields: [QdrantPayloadField] + public let indexes: [QdrantPayloadIndex] public let pointsCount: Int? public let info: String @@ -99,25 +148,23 @@ public struct QdrantCollection: Sendable, Equatable { name: String, vectors: [QdrantVectorSpec], payloadFields: [QdrantPayloadField], + indexes: [QdrantPayloadIndex] = [], pointsCount: Int? = nil, info: String = "{}" ) { self.name = name self.vectors = vectors self.payloadFields = payloadFields + self.indexes = indexes self.pointsCount = pointsCount self.info = info } - public func fieldType(of column: String) -> QdrantFieldType? { + public func fieldType(ofColumn column: String) -> QdrantFieldType? { let key = QdrantSchema.payloadKey(forColumn: column) return payloadFields.first { $0.name == key }?.type } - public var vectorColumns: [String] { vectors.map(\.column) } - - /// The collection with every payload key `points` hold that it lacks, typed by their values. An - /// indexed field keeps the index's type. public func including(_ points: [QdrantPoint]) -> QdrantCollection { var fields = payloadFields var position = Dictionary(uniqueKeysWithValues: fields.enumerated().map { ($1.name, $0) }) @@ -137,36 +184,47 @@ public struct QdrantCollection: Sendable, Equatable { fields[index] = QdrantPayloadField(name: key, type: type, isIndexed: false) } } - return QdrantCollection(name: name, vectors: vectors, payloadFields: fields, pointsCount: pointsCount, info: info) + return QdrantCollection( + name: name, vectors: vectors, payloadFields: fields, indexes: indexes, pointsCount: pointsCount, info: info + ) } - /// `GET /collections/{name}` answers the config and the indexed payload fields; the fields nobody - /// indexed are only visible in the points themselves, so a sample of them fills the rest in. - public static func parse(name: String, info json: Any?, sample: [QdrantPoint]) -> QdrantCollection { + // `payload_schema` keys are JSON paths: only a top-level key is a column, while a nested path such as + // `metadata.source` or `tags[]` stays an index on the column that holds it. + public static func parse(name: String, info json: Any?, sample: [QdrantPoint] = []) -> QdrantCollection { let result = (json as? [String: Any])?["result"] as? [String: Any] ?? [:] let params = ((result["config"] as? [String: Any])?["params"] as? [String: Any]) ?? [:] var vectors: [QdrantVectorSpec] = [] - if let named = params["vectors"] as? [String: Any] { - if named["size"] != nil { - vectors.append(QdrantVectorSpec(name: nil)) + if let config = params["vectors"] as? [String: Any] { + if config["size"] != nil { + vectors.append(.dense(name: "", config: config)) } else { - vectors += named.keys.sorted().map { QdrantVectorSpec(name: $0) } + vectors += config.keys.sorted().map { key in + .dense(name: key, config: config[key] as? [String: Any] ?? [:]) + } } } if let sparse = params["sparse_vectors"] as? [String: Any] { - vectors += sparse.keys.sorted().map { QdrantVectorSpec(name: $0, isSparse: true) } + vectors += sparse.keys.sorted().map { QdrantVectorSpec(name: $0, kind: .sparse) } } - let indexed = result["payload_schema"] as? [String: Any] ?? [:] - let fields = indexed.keys.sorted().compactMap { key in - QdrantFieldType.indexed(indexed[key]).map { QdrantPayloadField(name: key, type: $0, isIndexed: true) } + let schema = result["payload_schema"] as? [String: Any] ?? [:] + var indexes: [QdrantPayloadIndex] = [] + var fields: [QdrantPayloadField] = [] + for path in schema.keys.sorted() { + guard let type = QdrantFieldType.indexed(schema[path]) else { continue } + indexes.append(QdrantPayloadIndex(path: path, type: type.rawValue)) + if let key = QdrantKeyPath.topLevelKey(ofIndexPath: path), !fields.contains(where: { $0.name == key }) { + fields.append(QdrantPayloadField(name: key, type: type, isIndexed: true)) + } } return QdrantCollection( name: name, vectors: vectors, payloadFields: fields, + indexes: indexes, pointsCount: result["points_count"] as? Int, info: (try? QdrantJSON.text(result, pretty: true)) ?? "{}" ).including(sample) @@ -183,6 +241,7 @@ public enum QdrantSchema { public static let idColumn = "id" public static let vectorColumn = "vector" public static let scoreColumn = "score" + static let payloadPrefix = "payload." public static func collectionNames(from json: Any?) -> [String] { let result = (json as? [String: Any])?["result"] as? [String: Any] @@ -194,10 +253,14 @@ public enum QdrantSchema { column == vectorColumn || column.hasPrefix("\(vectorColumn).") } - static let payloadPrefix = "payload." + public static func vectorName(ofColumn column: String) -> String? { + if column == vectorColumn { return "" } + guard column.hasPrefix("\(vectorColumn).") else { return nil } + return String(column.dropFirst(vectorColumn.count + 1)) + } - /// A payload key may be `id`, `score` or `vector`, which name the point's own columns, so such a - /// key shows as `payload.` and maps back to its key wherever a column is read or written. + // A payload key named `id`, `score` or `vector...` would take a point column's name, so it shows as + // `payload.` and maps back wherever a column is read or written. public static func isReserved(_ key: String) -> Bool { key == idColumn || key == scoreColumn || isVectorColumn(key) } @@ -212,28 +275,43 @@ public enum QdrantSchema { return isReserved(key) ? key : column } - public static func immutableColumns(for collection: QdrantCollection) -> [String] { - [idColumn] + collection.vectorColumns - } - public static func columns(for collection: QdrantCollection) -> [QdrantColumn] { var columns = [QdrantColumn(name: idColumn, type: "id", isPrimaryKey: true)] columns += collection.payloadFields.map { QdrantColumn(name: column(forPayloadKey: $0.name), type: $0.type.rawValue, isPrimaryKey: false) } - columns += collection.vectors.map { - QdrantColumn(name: $0.column, type: $0.isSparse ? "sparse_vector" : "vector", isPrimaryKey: false) - } + columns += collection.vectors.map { QdrantColumn(name: $0.column, type: $0.typeName, isPrimaryKey: false) } return columns } public static func typeName(of column: String, in collection: QdrantCollection) -> String { + if let known = knownTypeName(of: column, in: collection) { return known } + if column == scoreColumn { return QdrantFieldType.float.rawValue } + return isVectorColumn(column) ? "vector" : QdrantFieldType.json.rawValue + } + + public static func knownTypeName(of column: String, in collection: QdrantCollection) -> String? { if column == idColumn { return "id" } - if column == scoreColumn { return "float" } if let spec = collection.vectors.first(where: { $0.column == column }) { - return spec.isSparse ? "sparse_vector" : "vector" + return spec.typeName + } + return collection.fieldType(ofColumn: column)?.rawValue + } +} + +public enum QdrantKeyPath { + // Qdrant reads a filter key as a JSON path, so `a.b` would address a nested field. Quoting keeps + // any top-level key whole; a key holding a double quote has no spelling Qdrant accepts. + public static func quoted(_ key: String) -> String? { + guard !key.contains("\"") else { return nil } + return "\"\(key)\"" + } + + public static func topLevelKey(ofIndexPath path: String) -> String? { + if path.count >= 2, path.hasPrefix("\""), path.hasSuffix("\"") { + let inner = path.dropFirst().dropLast() + return inner.contains("\"") ? nil : String(inner) } - if isVectorColumn(column) { return "vector" } - return collection.fieldType(of: column)?.rawValue ?? QdrantFieldType.json.rawValue + return path.contains(where: { ".[]\"".contains($0) }) ? nil : path } } diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantValue.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantValue.swift new file mode 100644 index 0000000000..9b180236db --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantValue.swift @@ -0,0 +1,118 @@ +import Foundation + +public enum QdrantValueError: Error, LocalizedError, Equatable, Sendable { + case notAnInteger(column: String, value: String) + case notANumber(column: String, value: String) + case notABoolean(column: String, value: String) + case notAGeoPoint(column: String) + case notAnObject(column: String) + case notAnArray(column: String) + case notAVector(column: String) + case notAMultivector(column: String) + case notASparseVector(column: String) + + public var errorDescription: String? { + switch self { + case .notAnInteger(let column, let value): + return String(format: String(localized: "%@ holds integers, and %@ is not one."), column, value) + case .notANumber(let column, let value): + return String(format: String(localized: "%@ holds numbers, and %@ is not a number."), column, value) + case .notABoolean(let column, let value): + return String(format: String(localized: "%@ holds booleans, so it takes true or false, not %@."), column, value) + case .notAGeoPoint(let column): + return String(format: String(localized: "%@ holds geo points, written as {\"lon\": 13.4, \"lat\": 52.5}."), column) + case .notAnObject(let column): + return String(format: String(localized: "%@ holds JSON objects, written as {\"key\": \"value\"}."), column) + case .notAnArray(let column): + return String(format: String(localized: "%@ holds JSON arrays, written as [1, 2, 3]."), column) + case .notAVector(let column): + return String(format: String(localized: "%@ is a vector, written as an array of numbers such as [0.1, 0.2]."), column) + case .notAMultivector(let column): + return String(format: String(localized: "%@ is a multivector, written as arrays of numbers such as [[0.1, 0.2], [0.3, 0.4]]."), column) + case .notASparseVector(let column): + return String(format: String(localized: "%@ is a sparse vector, written as {\"indices\": [1, 5], \"values\": [0.2, 0.7]}."), column) + } + } +} + +// The grid's text becomes the JSON type its column holds. Parsing every cell as JSON would turn a +// keyword holding `42` into a number and stop `match` from finding it. +public enum QdrantValue { + // The grid's boolean menu offers 1 and 0 to an engine without a SQL dialect. + public static func bool(_ text: String) -> Bool? { + switch text.trimmingCharacters(in: .whitespaces).lowercased() { + case "true", "1": return true + case "false", "0": return false + default: return nil + } + } + + public static func integer(_ text: String) -> Int64? { + Int64(text.trimmingCharacters(in: .whitespaces)) + } + + public static func number(_ text: String) -> Double? { + let trimmed = text.trimmingCharacters(in: .whitespaces) + guard let value = Double(trimmed), value.isFinite else { return nil } + return value + } + + public static func payload(_ text: String, type: QdrantFieldType, column: String) throws -> Any { + switch type { + case .keyword, .text, .string, .uuid, .datetime: + return text + case .integer: + guard let value = integer(text) else { throw QdrantValueError.notAnInteger(column: column, value: text) } + return NSNumber(value: value) + case .float: + guard let value = number(text) else { throw QdrantValueError.notANumber(column: column, value: text) } + return NSNumber(value: value) + case .bool: + guard let value = bool(text) else { throw QdrantValueError.notABoolean(column: column, value: text) } + return value + case .geo: + guard let point = QdrantJSON.parse(text) as? [String: Any], + point["lon"] is NSNumber, point["lat"] is NSNumber + else { throw QdrantValueError.notAGeoPoint(column: column) } + return point + case .object: + guard let object = QdrantJSON.parse(text) as? [String: Any] else { + throw QdrantValueError.notAnObject(column: column) + } + return object + case .array: + guard let array = QdrantJSON.parse(text) as? [Any] else { throw QdrantValueError.notAnArray(column: column) } + return array + case .json: + return QdrantJSON.parse(text) ?? text + } + } + + public static func vector(_ text: String, kind: QdrantVectorSpec.Kind, column: String) throws -> Any { + let parsed = QdrantJSON.parse(text) + switch kind { + case .dense: + guard let values = numbers(parsed), !values.isEmpty else { throw QdrantValueError.notAVector(column: column) } + return values + case .multi: + guard let rows = parsed as? [Any], !rows.isEmpty else { throw QdrantValueError.notAMultivector(column: column) } + return try rows.map { row in + guard let values = numbers(row), !values.isEmpty else { throw QdrantValueError.notAMultivector(column: column) } + return values + } + case .sparse: + guard let object = parsed as? [String: Any], + let indices = object["indices"] as? [Any], let values = numbers(object["values"]), + indices.count == values.count, + indices.allSatisfy({ ($0 as? NSNumber).map { !QdrantJSON.isBoolean($0) } ?? false }) + else { throw QdrantValueError.notASparseVector(column: column) } + return ["indices": indices, "values": values] + } + } + + private static func numbers(_ value: Any?) -> [NSNumber]? { + guard let array = value as? [Any] else { return nil } + let numbers = array.compactMap { $0 as? NSNumber }.filter { !QdrantJSON.isBoolean($0) } + return numbers.count == array.count ? numbers : nil + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantClient.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantClient.swift index 75a2c7ed4f..28e1f0c544 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantClient.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantClient.swift @@ -2,21 +2,12 @@ import Foundation public struct QdrantScrollPage: @unchecked Sendable { public let points: [QdrantPoint] - /// The id the next page starts from, `nil` once the collection is exhausted. + // Nil once the collection is exhausted. public let nextCursor: Any? } -public enum QdrantCursorLookup: @unchecked Sendable { - /// Scroll from this id; `nil` is the start of the collection. - case at(Any?) - /// The collection ends before the requested offset. - case pastEnd -} - public final class QdrantClient: @unchecked Sendable { public static let sampleSize = 100 - /// How many ids one request skips while walking to a page no cursor is held for. - public static let skipBatchSize = 1_000 public let settings: QdrantConnectionSettings private let transport: QdrantTransport @@ -43,28 +34,31 @@ public final class QdrantClient: @unchecked Sendable { } public func connect(requestTimeout: @Sendable () -> TimeInterval) async throws { - let root = try await send(method: "GET", path: "/", timeoutInterval: requestTimeout()) - try throwIfFailed(root) - if let json = QdrantJSON.dictionary(root.json), let found = json["version"] as? String { + let root = try await checked(send(method: "GET", path: "/", timeoutInterval: requestTimeout())) + if let found = (root.json as? [String: Any])?["version"] as? String { lock.withLock { version = found } } - try throwIfFailed(try await send(method: "GET", path: "/collections", timeoutInterval: requestTimeout())) + _ = try await checked(send(method: "GET", path: "/collections", timeoutInterval: requestTimeout())) } - /// `/healthz` answers without the key, so a revoked key would leave the session looking live. + // `/healthz` answers without the key, so a revoked key would leave the session looking live. public func ping() async throws { - try throwIfFailed(try await send(method: "GET", path: "/collections")) + _ = try await checked(send(method: "GET", path: "/collections")) } public func collectionNames() async throws -> [String] { - let response = try await send(method: "GET", path: "/collections") - try throwIfFailed(response) - return QdrantSchema.collectionNames(from: response.json) + QdrantSchema.collectionNames(from: try await checked(send(method: "GET", path: "/collections")).json) + } + + // The config and indexed fields alone: enough for a row count, the DDL and the index list. + public func collectionInfo(named name: String) async throws -> QdrantCollection { + let info = try await checked(send(method: "GET", path: collectionPath(name))) + return QdrantCollection.parse(name: name, info: info.json) } + // Fields nobody indexed are only visible in the points, so a sample of them fills in the columns. public func collection(named name: String) async throws -> QdrantCollection { - let info = try await send(method: "GET", path: collectionPath(name)) - try throwIfFailed(info) + let info = try await checked(send(method: "GET", path: collectionPath(name))) let sample = try await scroll( collection: name, body: ["limit": Self.sampleSize, "with_payload": true, "with_vector": false] @@ -98,50 +92,28 @@ public final class QdrantClient: @unchecked Sendable { return count } - /// Walks to the cursor of `offset` from the nearest one already known, reading ids only. - public func cursor( - collection: String, - filter: [String: Any]?, - offset: Int, - from known: (offset: Int, cursor: Any?) - ) async throws -> QdrantCursorLookup { - var position = known.offset - var cursor = known.cursor - while position < offset { - try Task.checkCancellation() - let step = min(offset - position, Self.skipBatchSize) - let page = try await scroll( - collection: collection, - body: QdrantBrowseQuery.scrollBody(filter: filter, limit: step, cursor: cursor, withData: false) - ) - position += page.points.count - guard let next = page.nextCursor else { return .pastEnd } - cursor = next - } - return .at(cursor) - } - public func execute(_ request: QdrantConsoleRequest) async throws -> QdrantHTTPResponse { - let response = try await send(method: request.method, path: request.path, body: request.body.map { Data($0.utf8) }) - try throwIfFailed(response) - return response - } - - public func send(method: String, path: String, body: Data? = nil) async throws -> QdrantHTTPResponse { - try await send(method: method, path: path, body: body, timeoutInterval: timeout()) + try await checked(send(method: request.method, path: request.path, body: request.body.map { Data($0.utf8) })) } private func post(_ path: String, _ body: [String: Any]) async throws -> QdrantHTTPResponse { - let response = try await send(method: "POST", path: path, body: try QdrantJSON.data(body)) - try throwIfFailed(response) - return response + try await checked(send(method: "POST", path: path, body: try QdrantJSON.data(body))) } private func collectionPath(_ name: String) -> String { "/collections/\(QdrantPathEncoding.segment(name))" } - private func send(method: String, path: String, body: Data? = nil, timeoutInterval: TimeInterval) async throws -> QdrantHTTPResponse { + private func send(method: String, path: String, body: Data? = nil) async throws -> QdrantHTTPResponse { + try await send(method: method, path: path, body: body, timeoutInterval: timeout()) + } + + private func send( + method: String, + path: String, + body: Data? = nil, + timeoutInterval: TimeInterval + ) async throws -> QdrantHTTPResponse { guard let url = QdrantPathEncoding.resolve(path, against: try settings.baseURL()) else { throw QdrantError.configuration(String(format: String(localized: "Invalid path: %@"), path)) } @@ -159,9 +131,10 @@ public final class QdrantClient: @unchecked Sendable { )) } - public func throwIfFailed(_ response: QdrantHTTPResponse) throws { + private func checked(_ response: QdrantHTTPResponse) throws -> QdrantHTTPResponse { guard (200..<300).contains(response.statusCode) else { throw QdrantError.from(status: response.statusCode, body: response.body) } + return response } } diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift index e7e67dbbd0..0367ca8100 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift @@ -1,18 +1,18 @@ import Foundation -/// The grid asks for a page by row offset and `scroll` answers with the id the next page starts -/// from. This remembers that id for every page already read, so paging forward costs one request -/// and a jump costs one id-only walk from the nearest page before it. -public final class QdrantCursorCache: @unchecked Sendable { +// `scroll` pages by point id, so a row offset is reached by walking ids from the nearest cursor +// already known. Every cursor a walk passes is kept, so paging on or jumping nearby costs one request. +public final class QdrantPagingCache: @unchecked Sendable { private static let maxQueries = 32 private let lock = NSLock() private var cursors: [String: [Int: Any]] = [:] private var order: [String] = [] + private var checkedSorts: Set = [] public init() {} - public func nearest(at offset: Int, key: String) -> (offset: Int, cursor: Any?) { + public func nearestCursor(at offset: Int, key: String) -> (offset: Int, cursor: Any?) { lock.withLock { guard let known = cursors[key], let best = known.keys.filter({ $0 <= offset }).max() @@ -21,7 +21,7 @@ public final class QdrantCursorCache: @unchecked Sendable { } } - public func store(_ cursor: Any, at offset: Int, key: String) { + public func storeCursor(_ cursor: Any, at offset: Int, key: String) { lock.withLock { if cursors[key] == nil { order.append(key) @@ -33,51 +33,57 @@ public final class QdrantCursorCache: @unchecked Sendable { } } + public func isSortChecked(_ key: String) -> Bool { + lock.withLock { checkedSorts.contains(key) } + } + + public func markSortChecked(_ key: String) { + lock.withLock { _ = checkedSorts.insert(key) } + } + public func removeAll() { lock.withLock { cursors.removeAll() order.removeAll() + checkedSorts.removeAll() } } } public extension QdrantClient { + static let walkBatchSize = 10_000 + func scrollPage( collection: String, filter: [String: Any]?, offset: Int, limit: Int, - cache: QdrantCursorCache + cache: QdrantPagingCache ) async throws -> [QdrantPoint] { - let filterKey = filter.flatMap { try? QdrantJSON.text($0) } ?? "" - let key = "\(collection)\u{0}\(filterKey)" - let start: Any? - switch try await cursor( - collection: collection, - filter: filter, - offset: offset, - from: cache.nearest(at: offset, key: key) - ) { - case .pastEnd: - return [] - case .at(let cursor): - start = cursor - if offset > 0, let cursor { - cache.store(cursor, at: offset, key: key) - } + let key = "\(collection)\u{0}\(filter.flatMap { try? QdrantJSON.text($0) } ?? "")" + var (position, cursor) = cache.nearestCursor(at: offset, key: key) + while position < offset { + try Task.checkCancellation() + let page = try await scroll( + collection: collection, + body: Self.scrollBody(filter: filter, limit: min(offset - position, Self.walkBatchSize), cursor: cursor, withData: false) + ) + guard let next = page.nextCursor else { return [] } + position += page.points.count + cursor = next + cache.storeCursor(next, at: position, key: key) } let page = try await scroll( collection: collection, - body: QdrantBrowseQuery.scrollBody(filter: filter, limit: limit, cursor: start, withData: true) + body: Self.scrollBody(filter: filter, limit: limit, cursor: cursor, withData: true) ) if let next = page.nextCursor { - cache.store(next, at: offset + page.points.count, key: key) + cache.storeCursor(next, at: offset + page.points.count, key: key) } return page.points } - /// Every point of a collection a page at a time, for export. Without vectors it reads payloads only, - /// which is how an export learns every payload key before it writes its header. + // Without vectors this reads payloads only, which is how an export learns every key before its header. func pointPages( collection: String, pageSize: Int, @@ -91,7 +97,7 @@ public extension QdrantClient { try Task.checkCancellation() let page = try await self.scroll( collection: collection, - body: QdrantBrowseQuery.scrollBody( + body: Self.scrollBody( filter: nil, limit: max(pageSize, 1), cursor: cursor, withData: true, withVector: withVector ) ) @@ -108,4 +114,25 @@ public extension QdrantClient { continuation.onTermination = { @Sendable _ in task.cancel() } } } + + static func scrollBody( + filter: [String: Any]?, + limit: Int, + cursor: Any?, + withData: Bool, + withVector: Bool? = nil + ) -> [String: Any] { + var body: [String: Any] = [ + "limit": limit, + "with_payload": withData, + "with_vector": withVector ?? withData + ] + if let filter { + body["filter"] = filter + } + if let cursor { + body["offset"] = cursor + } + return body + } } diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantSortedPager.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantSortedPager.swift new file mode 100644 index 0000000000..63a27a92e0 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantSortedPager.swift @@ -0,0 +1,60 @@ +import Foundation + +// `order_by` walks Qdrant's range index, which holds only points with a value in it and holds a point +// once per value. So the grid's rows are the ordered points first, then the points without a value in +// id order, and a field holding a list in some point is refused: no offset can page it one row per point. +public extension QdrantClient { + func sortedPage( + collection: String, + filter: [String: Any]?, + column: String, + key: String, + ascending: Bool, + offset: Int, + limit: Int, + cache: QdrantPagingCache + ) async throws -> [QdrantPoint] { + let hasValue: [String: Any] = ["key": key, "range": [String: Any]()] + let ordered = QdrantFilterBuilder.allOf(filter, hasValue) + + let checkKey = "\(collection)\u{0}\(key)\u{0}\((try? QdrantJSON.text(ordered)) ?? "")" + if !cache.isSortChecked(checkKey) { + let listed = try await count( + collection: collection, + filter: QdrantFilterBuilder.allOf(ordered, ["key": key, "values_count": ["gt": 1]]) + ) + guard listed == 0 else { + throw QdrantError.unsupported(String( + format: String(localized: "%@ holds a list in %d points, and Qdrant sorts such a point once per value. Sort by a field that holds one value per point."), + column, listed + )) + } + cache.markSortChecked(checkKey) + } + + let orderedCount = try await count(collection: collection, filter: ordered) + var points: [QdrantPoint] = [] + if offset < orderedCount { + var body: [String: Any] = [ + "query": ["order_by": ["key": key, "direction": ascending ? "asc" : "desc"]], + "limit": min(limit, orderedCount - offset), + "offset": offset, + "with_payload": true, + "with_vector": true + ] + if let filter { + body["filter"] = filter + } + points = try await query(collection: collection, body: body) + } + guard points.count < limit else { return points } + let withoutValue = QdrantFilterBuilder.allOf(filter, QdrantFilterBuilder.negated([hasValue])) + return points + (try await scrollPage( + collection: collection, + filter: withoutValue, + offset: max(offset - orderedCount, 0), + limit: limit - points.count, + cache: cache + )) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantTransport.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantTransport.swift index b596180e49..b92ec486fa 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantTransport.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantTransport.swift @@ -34,116 +34,9 @@ public struct QdrantHTTPResponse: Sendable, Equatable { } } +// The URLSession transport lives in the plugin: the app links this module for the query classifier, +// and an Objective-C class compiled into both would be registered twice in one process. public protocol QdrantTransport: Sendable { func send(_ request: QdrantHTTPRequest) async throws -> QdrantHTTPResponse func cancelAll() } - -public final class URLSessionQdrantTransport: QdrantTransport, @unchecked Sendable { - private let session: URLSession - private let lock = NSLock() - private var inFlight: [ObjectIdentifier: URLSessionTask] = [:] - - public init( - configuration: URLSessionConfiguration = .ephemeral, - resourceTimeout: TimeInterval, - skipTLSVerify: Bool = false - ) { - configuration.timeoutIntervalForResource = resourceTimeout - session = URLSession( - configuration: configuration, - delegate: skipTLSVerify ? TrustingTLSDelegate() : nil, - delegateQueue: nil - ) - } - - deinit { - session.invalidateAndCancel() - } - - public func cancelAll() { - let tasks = lock.withLock { Array(inFlight.values) } - tasks.forEach { $0.cancel() } - } - - public func send(_ request: QdrantHTTPRequest) async throws -> QdrantHTTPResponse { - var urlRequest = URLRequest(url: request.url) - urlRequest.httpMethod = request.method - urlRequest.httpBody = request.body - urlRequest.timeoutInterval = request.timeoutInterval - for (name, value) in request.headers { - urlRequest.setValue(value, forHTTPHeaderField: name) - } - - let tracker = TaskTracker(transport: self) - defer { tracker.finish() } - do { - let (data, response) = try await session.data(for: urlRequest, delegate: tracker) - guard let http = response as? HTTPURLResponse else { - throw QdrantError.transport(String(localized: "Qdrant answered with something other than HTTP.")) - } - return QdrantHTTPResponse(statusCode: http.statusCode, body: data) - } catch let error as URLError where error.code == .cancelled { - throw QdrantError.cancelled - } catch let error as QdrantError { - throw error - } catch let error as URLError { - throw QdrantError.transport(error.localizedDescription) - } - } - - fileprivate func register(_ task: URLSessionTask) { - lock.withLock { inFlight[ObjectIdentifier(task)] = task } - } - - fileprivate func unregister(_ task: URLSessionTask) { - lock.withLock { inFlight[ObjectIdentifier(task)] = nil } - } -} - -private final class TrustingTLSDelegate: NSObject, URLSessionDelegate { - func urlSession( - _ session: URLSession, - didReceive challenge: URLAuthenticationChallenge, - completionHandler: @escaping (URLSession.AuthChallengeDisposition, URLCredential?) -> Void - ) { - guard challenge.protectionSpace.authenticationMethod == NSURLAuthenticationMethodServerTrust, - let trust = challenge.protectionSpace.serverTrust - else { - completionHandler(.performDefaultHandling, nil) - return - } - completionHandler(.useCredential, URLCredential(trust: trust)) - } -} - -private final class TaskTracker: NSObject, URLSessionTaskDelegate, @unchecked Sendable { - private weak var transport: URLSessionQdrantTransport? - private let lock = NSLock() - private var task: URLSessionTask? - - init(transport: URLSessionQdrantTransport) { - self.transport = transport - } - - func urlSession(_ session: URLSession, didCreateTask task: URLSessionTask) { - lock.withLock { self.task = task } - transport?.register(task) - } - - // A redirect would carry the API key to wherever it points. - func urlSession( - _ session: URLSession, - task: URLSessionTask, - willPerformHTTPRedirection response: HTTPURLResponse, - newRequest request: URLRequest, - completionHandler: @escaping (URLRequest?) -> Void - ) { - completionHandler(nil) - } - - func finish() { - guard let task = lock.withLock({ task }) else { return } - transport?.unregister(task) - } -} diff --git a/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateGraphQL.swift b/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateGraphQL.swift index 0705167481..7fa048ddc5 100644 --- a/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateGraphQL.swift +++ b/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateGraphQL.swift @@ -131,10 +131,8 @@ public enum WeaviateConsoleParser { let inlineBody = parts.count > 2 ? String(parts[2]).trimmingCharacters(in: .whitespacesAndNewlines) : "" - return WeaviateConsoleRequest( - method: method, - path: path, - body: body ?? (inlineBody.isEmpty ? nil : inlineBody) - ) + // A body can open on the request line and go on below it, as `POST /v1/graphql {`. + let joined = [inlineBody, body ?? ""].filter { !$0.isEmpty }.joined(separator: "\n") + return WeaviateConsoleRequest(method: method, path: path, body: joined.isEmpty ? nil : joined) } } diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift index ee12db647e..7e7a426526 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift @@ -7,7 +7,7 @@ struct QdrantClientTests { func pagingForward() async throws { let transport = scrollingTransport(count: 25) let client = makeClient(transport: transport) - let cache = QdrantCursorCache() + let cache = QdrantPagingCache() let first = try await client.scrollPage(collection: "docs", filter: nil, offset: 0, limit: 10, cache: cache) let second = try await client.scrollPage(collection: "docs", filter: nil, offset: 10, limit: 10, cache: cache) @@ -18,24 +18,29 @@ struct QdrantClientTests { #expect(bodyObject(transport.recorded[1])["offset"] as? Int == 11) } - @Test("A jump with no cursor walks ids only, then reads the page") + @Test("A jump walks ids only and keeps every cursor it passes, so a nearby jump starts from one") func jump() async throws { - let transport = scrollingTransport(count: 25) + let transport = scrollingTransport(count: 25_000) let client = makeClient(transport: transport) + let cache = QdrantPagingCache() - let page = try await client.scrollPage(collection: "docs", filter: nil, offset: 20, limit: 10, cache: QdrantCursorCache()) + let page = try await client.scrollPage(collection: "docs", filter: nil, offset: 20_000, limit: 10, cache: cache) + #expect(page.compactMap(\.idText) == (20_001...20_010).map(String.init)) + let walks = transport.recorded.dropLast().map(bodyObject) + #expect(walks.map { $0["limit"] as? Int } == [10_000, 10_000]) + #expect(walks.allSatisfy { $0["with_payload"] as? Bool == false && $0["with_vector"] as? Bool == false }) - #expect(page.compactMap(\.idText) == (21...25).map(String.init)) - let walk = bodyObject(transport.recorded[0]) - #expect(walk["limit"] as? Int == 20) - #expect(walk["with_payload"] as? Bool == false) - #expect(walk["with_vector"] as? Bool == false) + let before = transport.recorded.count + let nearby = try await client.scrollPage(collection: "docs", filter: nil, offset: 15_000, limit: 10, cache: cache) + #expect(nearby.first?.idText == "15001") + #expect(bodyObject(transport.recorded[before])["offset"] as? Int == 10_001) + #expect(transport.recorded.count == before + 2) } @Test("An offset past the end answers an empty page") func pastEnd() async throws { let client = makeClient(transport: scrollingTransport(count: 5)) - let page = try await client.scrollPage(collection: "docs", filter: nil, offset: 10, limit: 10, cache: QdrantCursorCache()) + let page = try await client.scrollPage(collection: "docs", filter: nil, offset: 10, limit: 10, cache: QdrantPagingCache()) #expect(page.isEmpty) } @@ -59,19 +64,76 @@ struct QdrantClientTests { _ = try await client.collection(named: "nope") } #expect(transport.recorded.first?.headers["api-key"] == "secret") + #expect(QdrantError.from(status: 401, body: Data("Invalid api-key".utf8)) == .authentication("Invalid api-key")) + } + + @Test("Collection info is one GET, without the payload sample the columns need") + func infoOnly() async throws { + let transport = RecordingTransport { _ in + jsonResponse(["result": ["points_count": 12, "config": ["params": ["vectors": ["size": 2]]]], "status": "ok"]) + } + let collection = try await makeClient(transport: transport).collectionInfo(named: "docs") + #expect(collection.pointsCount == 12) + #expect(transport.recorded.map(\.method) == ["GET"]) } +} - @Test("A rejected key is an authentication error") - func authentication() { - let error = QdrantError.from(status: 401, body: Data("Invalid api-key".utf8)) - #expect(error == .authentication("Invalid api-key")) +// Ordered points 1...6 by `price`, then 7...9 without a price. +struct QdrantSortedPagerTests { + private func transport(listed: Int = 0) -> RecordingTransport { + RecordingTransport { request in + let body = bodyObject(request) + let text = canonical(body) + switch request.url.path { + case "/collections/docs/points/count": + if text.contains("values_count") { return jsonResponse(["result": ["count": listed]]) } + return jsonResponse(["result": ["count": 6]]) + case "/collections/docs/points/query": + let offset = body["offset"] as? Int ?? 0 + let limit = body["limit"] as? Int ?? 10 + let ids = Array((1...6).dropFirst(offset).prefix(limit)) + return jsonResponse(["result": ["points": ids.map { ["id": $0, "payload": ["price": $0]] }]]) + case "/collections/docs/points/scroll": + let limit = body["limit"] as? Int ?? 10 + let start = body["offset"] as? Int ?? 7 + let ids = Array(start.. [String] { + try await client.sortedPage( + collection: "docs", filter: nil, column: "price", key: "\"price\"", ascending: true, + offset: offset, limit: limit, cache: cache + ).compactMap(\.idText) + } + + @Test("Points with a value come first in order, then the points without one, and every point is reachable") + func orderedThenRemainder() async throws { + let transport = transport() + let client = makeClient(transport: transport) + let cache = QdrantPagingCache() + + #expect(try await page(client, offset: 0, limit: 4, cache: cache) == ["1", "2", "3", "4"]) + #expect(try await page(client, offset: 4, limit: 4, cache: cache) == ["5", "6", "7", "8"]) + #expect(try await page(client, offset: 8, limit: 4, cache: cache) == ["9"]) + + let query = try #require(transport.recorded.first { $0.url.path.hasSuffix("/query") }) + #expect(canonical(bodyObject(query)["query"] ?? [:]) == #"{"order_by":{"direction":"asc","key":"\"price\""}}"#) + let remainder = try #require(transport.recorded.first { $0.url.path.hasSuffix("/scroll") }) + #expect(canonical(bodyObject(remainder)["filter"] ?? [:]) + == #"{"must":[{"must_not":[{"key":"\"price\"","range":{}}]}]}"#) + #expect(transport.recorded.filter { canonical(bodyObject($0)).contains("values_count") }.count == 1) + } + + @Test("A field holding a list in some point is refused, since Qdrant would show that point once per value") + func listedValuesRefused() async throws { + let client = makeClient(transport: transport(listed: 2)) + await #expect(throws: QdrantError.self) { + _ = try await page(client, offset: 0, limit: 4, cache: QdrantPagingCache()) + } } } diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantConsoleTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantConsoleTests.swift new file mode 100644 index 0000000000..8619ca221e --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantConsoleTests.swift @@ -0,0 +1,92 @@ +import Foundation +@testable import TableProQdrantCore +import Testing + +struct QdrantConsoleTests { + @Test("The console reads a request line with its body below, beside, or starting beside and going on below") + func parser() { + let below = QdrantConsoleParser.parse("POST /collections/docs/points/query\n{\"limit\": 3}") + #expect(below == QdrantConsoleRequest(method: "POST", path: "/collections/docs/points/query", body: "{\"limit\": 3}")) + let inline = QdrantConsoleParser.parse("post /collections/docs/points/count {\"exact\": true}") + #expect(inline?.method == "POST") + #expect(inline?.body == "{\"exact\": true}") + let split = QdrantConsoleParser.parse("POST /collections/docs/points/query {\n \"query\": [0.1, 0.2], \"limit\": 5 }") + #expect(split?.body == "{\n\"query\": [0.1, 0.2], \"limit\": 5 }") + let crlf = QdrantConsoleParser.parse("GET /collections\r\n") + #expect(crlf == QdrantConsoleRequest(method: "GET", path: "/collections", body: nil)) + #expect(QdrantConsoleParser.parse("DELETE\t/collections/docs")?.method == "DELETE") + #expect(QdrantConsoleParser.parse("SELECT 1") == nil) + #expect(QdrantConsoleParser.parse("GET collections") == nil) + } + + @Test("Reads, writes and removals are told apart by verb and route") + func requestKinds() { + func tier(_ method: String, _ path: String) -> QdrantRequestKind.Tier { + QdrantRequestKind.of(method: method, path: path).tier + } + #expect(tier("POST", "/collections/docs/points/query") == .read) + #expect(tier("POST", "/collections/docs/points") == .read) + #expect(tier("POST", "/collections/docs/facet") == .read) + #expect(tier("POST", "/collections/docs/points/facet") == .write) + #expect(tier("GET", "/collections") == .read) + #expect(tier("PUT", "/collections/docs/points?wait=true") == .write) + #expect(tier("POST", "/collections/docs/points/payload") == .write) + #expect(tier("POST", "/collections/docs/points/delete?wait=true") == .remove) + #expect(tier("POST", "/collections/docs/points/payload/clear") == .remove) + #expect(tier("POST", "/collections/docs/points/batch") == .remove) + #expect(tier("DELETE", "/collections/docs") == .remove) + } + + @Test("A snapshot touches the server's disk, and a dot segment is never read as another route") + func serverFilesAndDotSegments() { + #expect(QdrantRequestKind.of(method: "POST", path: "/collections/docs/snapshots").touchesServerFiles) + #expect(!QdrantRequestKind.of(method: "GET", path: "/collections").touchesServerFiles) + let dotted = QdrantRequestKind.of(method: "POST", path: "/collections/docs/points/query/../delete") + #expect(dotted == QdrantRequestKind(tier: .write, touchesServerFiles: true)) + #expect(QdrantPathEncoding.segments(of: "/collections/%2e%2e/x") == nil) + } + + @Test("A request path keeps its query string and never leaves the node") + func pathResolution() throws { + let base = try #require(URL(string: "http://localhost:6333")) + let url = QdrantPathEncoding.resolve("/collections/docs/points?wait=true", against: base) + #expect(url?.absoluteString == "http://localhost:6333/collections/docs/points?wait=true") + #expect(QdrantPathEncoding.resolve("//evil.example/x", against: base) == nil) + #expect(QdrantPathEncoding.resolve("/collections/docs/points/query/../delete", against: base) == nil) + } + + @Test("Browse requests round-trip through their tag with every sort column") + func browseRoundTrip() throws { + let request = QdrantBrowseRequest( + collection: "docs", + offset: 200, + limit: 100, + sorts: [QdrantSortSpec(column: "pop", ascending: false), QdrantSortSpec(column: "city", ascending: true)], + filters: [QdrantFilterSpec(column: "count", op: "BETWEEN", value: "1,5", secondValue: "5")], + logicMode: "OR" + ) + #expect(QdrantBrowseQuery.parse(QdrantBrowseQuery.encode(request)) == request) + } + + @Test("A sort Qdrant cannot honor is refused rather than ignored") + func sortPlans() throws { + #expect(try QdrantSortPlan.resolve([]) == .byID) + #expect(try QdrantSortPlan.resolve([QdrantSortSpec(column: "id", ascending: true)]) == .byID) + #expect(try QdrantSortPlan.resolve([QdrantSortSpec(column: "pop", ascending: false)]) + == .byPayload(column: "pop", key: "\"pop\"", ascending: false)) + #expect(throws: QdrantError.self) { try QdrantSortPlan.resolve([QdrantSortSpec(column: "id", ascending: false)]) } + #expect(throws: QdrantError.self) { try QdrantSortPlan.resolve([QdrantSortSpec(column: "vector.image", ascending: true)]) } + #expect(throws: QdrantError.self) { + try QdrantSortPlan.resolve([QdrantSortSpec(column: "pop", ascending: true), QdrantSortSpec(column: "city", ascending: true)]) + } + } + + @Test("Drop, truncate and export are request lines or tags the driver reads back") + func operations() { + #expect(QdrantOperations.deleteCollection(named: "my docs", objectType: "TABLE") == "DELETE /collections/my%20docs") + #expect(QdrantOperations.deleteCollection(named: "docs", objectType: "VIEW") == nil) + #expect(QdrantOperations.deleteAllPoints(collection: "docs") + == "POST /collections/docs/points/delete?wait=true\n{\"filter\":{}}") + #expect(QdrantOperations.decodeExport(QdrantOperations.encodeExport(collection: "docs")) == "docs") + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift index 14e50cbec6..a4043cedeb 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift @@ -10,60 +10,93 @@ struct QdrantFilterBuilderTests { )) } - @Test("Equality matches a keyword as text and an integer as a number") + @Test("Equality matches a keyword as text, an integer as a number and a boolean as true or false") func equality() throws { - #expect(try condition("city", "=", "Berlin") == #"{"key":"city","match":{"value":"Berlin"}}"#) - #expect(try condition("count", "=", "7") == #"{"key":"count","match":{"value":7}}"#) - #expect(try condition("active", "=", "true") == #"{"key":"active","match":{"value":true}}"#) + #expect(try condition("city", "=", "Berlin") == #"{"key":"\"city\"","match":{"value":"Berlin"}}"#) + #expect(try condition("count", "=", "7") == #"{"key":"\"count\"","match":{"value":7}}"#) + #expect(try condition("active", "=", "1") == #"{"key":"\"active\"","match":{"value":true}}"#) + #expect(throws: QdrantValueError.notAnInteger(column: "count", value: "x")) { try condition("count", "=", "x") } } - @Test("A float has no exact match, so equality is the closed range around it") - func floatEquality() throws { - #expect(try condition("pop", "=", "3.5") == #"{"key":"pop","range":{"gte":3.5,"lte":3.5}}"#) + @Test("A key holding a dot stays one top-level key, and one holding a quote is refused") + func keyQuoting() throws { + #expect(try condition("a.b", "=", "x") == #"{"key":"\"a.b\"","match":{"value":"x"}}"#) + let collection = QdrantCollection(name: "docs", vectors: [], payloadFields: [ + QdrantPayloadField(name: "q\"x", type: .string, isIndexed: false) + ]) + #expect(throws: QdrantFilterError.keyNotAddressable(column: "q\"x")) { + try QdrantFilterBuilder.condition(for: QdrantFilterSpec(column: "q\"x", op: "=", value: "1"), collection: collection) + } + } + + @Test("A float and a datetime have no exact match, so equality is the closed range around the value") + func rangeEquality() throws { + #expect(try condition("pop", "=", "3.5") == #"{"key":"\"pop\"","range":{"gte":3.5,"lte":3.5}}"#) + #expect(try condition("seen", "=", "2024-01-02T00:00:00Z") + == #"{"key":"\"seen\"","range":{"gte":"2024-01-02T00:00:00Z","lte":"2024-01-02T00:00:00Z"}}"#) } @Test("Not equal nests the match under must_not") func notEqual() throws { - #expect(try condition("city", "!=", "Berlin") == #"{"must_not":[{"key":"city","match":{"value":"Berlin"}}]}"#) + #expect(try condition("city", "!=", "Berlin") == #"{"must_not":[{"key":"\"city\"","match":{"value":"Berlin"}}]}"#) } - @Test("Comparisons and BETWEEN become a range") + @Test("Comparisons and BETWEEN become a range, and text is refused") func ranges() throws { - #expect(try condition("pop", ">", "2") == #"{"key":"pop","range":{"gt":2}}"#) - #expect(try condition("count", "BETWEEN", "1,5", "5") == #"{"key":"count","range":{"gte":1,"lte":5}}"#) - } - - @Test("A comparison on a string field is refused, not sent") - func comparisonOnText() { + #expect(try condition("pop", ">", "2") == #"{"key":"\"pop\"","range":{"gt":2}}"#) + #expect(try condition("count", "BETWEEN", "1,5", "5") == #"{"key":"\"count\"","range":{"gte":1,"lte":5}}"#) #expect(throws: QdrantFilterError.comparisonNeedsNumberOrDate(column: "note", op: ">")) { try condition("note", ">", "a") } } - @Test("IN and NOT IN use match any and except") - func lists() throws { - #expect(try condition("city", "IN", "Berlin, Paris") == #"{"key":"city","match":{"any":["Berlin","Paris"]}}"#) - #expect(try condition("count", "NOT IN", "1,2") == #"{"key":"count","match":{"except":[1,2]}}"#) + @Test("IN is the equality of each value: match any for strings or integers, ranges for floats") + func inLists() throws { + #expect(try condition("city", "IN", "Berlin, Paris") == #"{"key":"\"city\"","match":{"any":["Berlin","Paris"]}}"#) + #expect(try condition("count", "IN", "1,2") == #"{"key":"\"count\"","match":{"any":[1,2]}}"#) + #expect(try condition("pop", "IN", "1.5, 2.5") + == #"{"should":[{"key":"\"pop\"","range":{"gte":1.5,"lte":1.5}},{"key":"\"pop\"","range":{"gte":2.5,"lte":2.5}}]}"#) + #expect(try condition("active", "IN", "true") + == #"{"should":[{"key":"\"active\"","match":{"value":true}}]}"#) + } + + @Test("NOT IN excludes every value the way != excludes one") + func notInLists() throws { + #expect(try condition("count", "NOT IN", "1,2") + == #"{"must_not":[{"key":"\"count\"","match":{"value":1}},{"key":"\"count\"","match":{"value":2}}]}"#) + #expect(try condition("pop", "NOT IN", "1.5") == #"{"must_not":[{"key":"\"pop\"","range":{"gte":1.5,"lte":1.5}}]}"#) + } + + @Test("A geo point cannot be matched whole") + func geoEquality() { + #expect(throws: QdrantFilterError.wholeValueNotMatchable(column: "home", type: "geo")) { + try condition("home", "IN", "1") + } } - @Test("CONTAINS needs a full-text index, since Qdrant matches nothing without one") + @Test("CONTAINS matches words in any text field, indexed or not, and refuses numbers") func contains() throws { - #expect(try condition("body", "CONTAINS", "vector") == #"{"key":"body","match":{"text":"vector"}}"#) - #expect(throws: QdrantFilterError.textMatchNeedsTextIndex(column: "city", op: "CONTAINS")) { - try condition("city", "CONTAINS", "Ber") + #expect(try condition("body", "CONTAINS", "vector") == #"{"key":"\"body\"","match":{"text":"vector"}}"#) + #expect(try condition("note", "CONTAINS", "Ber") == #"{"key":"\"note\"","match":{"text":"Ber"}}"#) + #expect(throws: QdrantFilterError.textMatchNeedsText(column: "count", type: "integer")) { + try condition("count", "CONTAINS", "1") } } - @Test("Null and empty checks") + @Test("IS NULL matches a missing key as well as null, and IS EMPTY adds the empty string") func nullness() throws { - #expect(try condition("note", "IS NULL") == #"{"is_null":{"key":"note"}}"#) - #expect(try condition("tags", "IS NOT EMPTY") == #"{"must_not":[{"is_empty":{"key":"tags"}}]}"#) + #expect(try condition("note", "IS NULL") == #"{"is_empty":{"key":"\"note\""}}"#) + #expect(try condition("note", "IS NOT NULL") == #"{"must_not":[{"is_empty":{"key":"\"note\""}}]}"#) + #expect(try condition("note", "IS EMPTY") + == #"{"should":[{"is_empty":{"key":"\"note\""}},{"key":"\"note\"","match":{"value":""}}]}"#) + #expect(try condition("count", "IS EMPTY") == #"{"is_empty":{"key":"\"count\""}}"#) } @Test("The id column filters with has_id, keeping integer and UUID ids apart") func ids() throws { let uuid = "5c56c793-69f3-4fbf-87e6-c4bf54c28c26" #expect(try condition("id", "IN", "1, \(uuid)") == #"{"has_id":[1,"\#(uuid)"]}"#) + #expect(try condition("id", "!=", "1") == #"{"must_not":[{"has_id":[1]}]}"#) #expect(throws: QdrantFilterError.notAPointID(value: "abc")) { try condition("id", "=", "abc") } } diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift new file mode 100644 index 0000000000..38a506c1c1 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift @@ -0,0 +1,117 @@ +import Foundation +@testable import TableProQdrantCore +import Testing + +struct QdrantRowWriteTests { + private let types: [String: String] = [ + "id": "id", "city": "keyword", "count": "integer", "pop": "float", "active": "bool", + "note": "string", "meta": "object", "payload.id": "keyword", + "vector": "vector(2)", "vector.text": "multivector", "vector.sp": "sparse_vector" + ] + + private func writes(_ changes: [QdrantRowChange]) throws -> [String] { + try QdrantRowWriter.writes( + collection: "docs", + changes: changes, + typeName: { types[$0] }, + newID: { "00000000-0000-0000-0000-000000000001" } + ).map(\.request.text) + } + + private func refusal(_ change: QdrantRowChange) -> QdrantRowRefusal? { + do { + _ = try writes([change]) + return nil + } catch { + return error as? QdrantRowRefusal + } + } + + @Test("A payload edit is a set-payload request typed by the column the grid read") + func payloadUpdate() throws { + let change = QdrantRowChange(kind: .update, rowIndex: 0, id: "7", cells: [ + QdrantCell(column: "note", text: "42"), + QdrantCell(column: "count", text: "42"), + QdrantCell(column: "active", text: "1"), + QdrantCell(column: "payload.id", text: "ext"), + QdrantCell(column: "city", text: nil) + ]) + #expect(try writes([change]) == [ + "POST /collections/docs/points/payload?wait=true\n" + + #"{"payload":{"active":true,"city":null,"count":42,"id":"ext","note":"42"},"points":[7]}"# + ]) + } + + @Test("Vector edits update or remove vectors by name, the unnamed one under the empty name") + func vectorUpdate() throws { + let change = QdrantRowChange(kind: .update, rowIndex: 0, id: "7", cells: [ + QdrantCell(column: "vector", text: "[0.5, 1]"), + QdrantCell(column: "vector.sp", text: #"{"indices":[1],"values":[0.5]}"#), + QdrantCell(column: "vector.text", text: nil) + ]) + #expect(try writes([change]) == [ + "PUT /collections/docs/points/vectors?wait=true\n" + + #"{"points":[{"id":7,"vector":{"":[0.5,1],"sp":{"indices":[1],"values":[0.5]}}}]}"#, + "POST /collections/docs/points/vectors/delete?wait=true\n" + #"{"points":[7],"vector":["text"]}"# + ]) + } + + @Test("A value that does not fit its column refuses the row instead of changing the field's type") + func typeRefusals() { + let count = QdrantRowChange(kind: .update, rowIndex: 3, id: "7", cells: [QdrantCell(column: "count", text: "4.5")]) + #expect(refusal(count) == QdrantRowRefusal( + rowIndex: 3, reason: QdrantValueError.notAnInteger(column: "count", value: "4.5").errorDescription ?? "" + )) + let vector = QdrantRowChange(kind: .update, rowIndex: 1, id: "7", cells: [QdrantCell(column: "vector", text: "[\"a\"]")]) + #expect(refusal(vector)?.rowIndex == 1) + let sparse = QdrantRowChange(kind: .update, rowIndex: 1, id: "7", cells: [QdrantCell(column: "vector.sp", text: "[1]")]) + #expect(refusal(sparse)?.rowIndex == 1) + } + + @Test("A column whose type the grid did not report is refused, never written as text") + func unknownType() { + let change = QdrantRowChange(kind: .update, rowIndex: 2, id: "7", cells: [QdrantCell(column: "late", text: "43")]) + #expect(refusal(change)?.rowIndex == 2) + } + + @Test("A delete names the point, and a row without an id or with a changed id is refused") + func delete() throws { + let uuid = "5c56c793-69f3-4fbf-87e6-c4bf54c28c26" + #expect(try writes([QdrantRowChange(kind: .delete, rowIndex: 0, id: uuid)]) == [ + "POST /collections/docs/points/delete?wait=true\n{\"points\":[\"\(uuid)\"]}" + ]) + #expect(refusal(QdrantRowChange(kind: .delete, rowIndex: 4, id: nil))?.rowIndex == 4) + let idChange = QdrantRowChange(kind: .update, rowIndex: 5, id: "1", cells: [QdrantCell(column: "id", text: "2")]) + #expect(refusal(idChange)?.rowIndex == 5) + } + + @Test("An insert carries its vectors, keeps an existing id untouched, and first checks that the id is free") + func insert() throws { + let change = QdrantRowChange(kind: .insert, rowIndex: 0, id: nil, cells: [ + QdrantCell(column: "id", text: ""), + QdrantCell(column: "city", text: "Hanoi"), + QdrantCell(column: "active", text: "true"), + QdrantCell(column: "note", text: nil), + QdrantCell(column: "vector", text: "[1,2]") + ]) + let write = try #require(try QdrantRowWriter.writes( + collection: "docs", changes: [change], typeName: { types[$0] }, + newID: { "00000000-0000-0000-0000-000000000001" } + ).first) + #expect(write.request.text == "PUT /collections/docs/points?wait=true\n" + + #"{"points":[{"id":"00000000-0000-0000-0000-000000000001","payload":{"active":true,"city":"Hanoi"},"vector":{"":[1,2]}}],"update_mode":"insert_only"}"#) + #expect(write.mustFindNothing?.text == "POST /collections/docs/points\n" + + #"{"ids":["00000000-0000-0000-0000-000000000001"],"with_payload":false,"with_vector":false}"#) + } + + @Test("Typed values: text stays text, JSON columns parse, and malformed structures are refused") + func payloadValues() throws { + #expect(try QdrantValue.payload("42", type: .keyword, column: "c") as? String == "42") + #expect(try QdrantValue.payload("0", type: .bool, column: "c") as? Bool == false) + #expect(try QdrantValue.payload(#"["a"]"#, type: .array, column: "c") as? [String] == ["a"]) + #expect(try QdrantValue.payload("42", type: .json, column: "c") as? Int == 42) + #expect(try QdrantValue.payload("plain", type: .json, column: "c") as? String == "plain") + #expect(throws: QdrantValueError.notAnObject(column: "c")) { try QdrantValue.payload("not json", type: .object, column: "c") } + #expect(throws: QdrantValueError.notAGeoPoint(column: "c")) { try QdrantValue.payload("{\"x\":1}", type: .geo, column: "c") } + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaAndWriteTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaAndWriteTests.swift deleted file mode 100644 index 190a771a8d..0000000000 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaAndWriteTests.swift +++ /dev/null @@ -1,198 +0,0 @@ -import Foundation -@testable import TableProQdrantCore -import Testing - -struct QdrantSchemaAndWriteTests { - private func info(vectors: Any, payloadSchema: [String: Any] = [:]) -> [String: Any] { - ["result": [ - "points_count": 4, - "config": ["params": ["vectors": vectors]], - "payload_schema": payloadSchema - ]] - } - - private func point(_ json: String) throws -> QdrantPoint { - let object = try JSONSerialization.jsonObject(with: Data(json.utf8)) - return try #require(QdrantPoint.parse(object)) - } - - @Test("Columns are the id, indexed fields, sampled fields, then vectors") - func columns() throws { - let sample = [ - try point(#"{"id":1,"payload":{"city":"Berlin","count":3,"meta":{"x":1}}}"#), - try point(#"{"id":2,"payload":{"count":3.5,"tags":["a"]}}"#) - ] - let collection = QdrantCollection.parse( - name: "docs", - info: info(vectors: ["size": 4, "distance": "Cosine"], payloadSchema: ["city": ["data_type": "keyword"]]), - sample: sample - ) - #expect(QdrantSchema.columns(for: collection).map(\.name) == ["id", "city", "count", "meta", "tags", "vector"]) - #expect(collection.fieldType(of: "city") == .keyword) - #expect(collection.fieldType(of: "count") == .float) - #expect(collection.fieldType(of: "meta") == .object) - #expect(collection.pointsCount == 4) - #expect(QdrantSchema.immutableColumns(for: collection) == ["id", "vector"]) - } - - @Test("A payload key named like a point column shows as payload. and maps back") - func reservedPayloadKeys() throws { - let row = try point(#"{"id":7,"payload":{"id":"external-1","vector":"text","city":"Berlin"},"vector":[1,2]}"#) - let collection = QdrantCollection.parse(name: "docs", info: info(vectors: ["size": 2]), sample: [row]) - let columns = QdrantSchema.columns(for: collection).map(\.name) - #expect(columns == ["id", "city", "payload.id", "payload.vector", "vector"]) - #expect(row.row(columns: columns) == ["7", "Berlin", "external-1", "text", "[1,2]"]) - - let condition = try QdrantFilterBuilder.condition( - for: QdrantFilterSpec(column: "payload.id", op: "=", value: "external-1"), - collection: collection - ) - #expect(canonical(condition) == #"{"key":"id","match":{"value":"external-1"}}"#) - - let batch = QdrantStatementGenerator.generate(collection: collection, changes: [ - QdrantTrackedChange(kind: .update, id: "7", cellChanges: [QdrantCellChange(column: "payload.id", newText: "external-2")]) - ]) - #expect(batch.requests.first?.body == #"{"payload":{"id":"external-2"},"points":[7]}"#) - } - - @Test("Keys outside the sample join the collection, and a key first seen as null takes its later type") - func includingLaterPoints() throws { - let sample = [try point(#"{"id":1,"payload":{"city":"Berlin","rank":null}}"#)] - let collection = QdrantCollection.parse(name: "docs", info: info(vectors: ["size": 2]), sample: sample) - let widened = collection.including([try point(#"{"id":2,"payload":{"late":true,"rank":3}}"#)]) - #expect(widened.payloadFields.map(\.name) == ["city", "rank", "late"]) - #expect(widened.fieldType(of: "rank") == .integer) - #expect(widened.fieldType(of: "late") == .bool) - } - - @Test("Named and sparse vectors each get a read-only column") - func namedVectors() throws { - var json = info(vectors: ["image": ["size": 2], "text": ["size": 3]]) - var result = try #require(json["result"] as? [String: Any]) - var config = try #require(result["config"] as? [String: Any]) - var params = try #require(config["params"] as? [String: Any]) - params["sparse_vectors"] = ["keywords": [String: Any]()] - config["params"] = params - result["config"] = config - json["result"] = result - - let collection = QdrantCollection.parse(name: "docs", info: json, sample: []) - #expect(collection.vectorColumns == ["vector.image", "vector.text", "vector.keywords"]) - - let row = try point(#"{"id":"5c56c793-69f3-4fbf-87e6-c4bf54c28c26","vector":{"image":[0.5,1],"keywords":{"indices":[1],"values":[0.5]}}}"#) - #expect(row.cell(for: "id") == "5c56c793-69f3-4fbf-87e6-c4bf54c28c26") - #expect(row.cell(for: "vector.image") == "[0.5,1]") - #expect(row.cell(for: "vector.keywords") == #"{"indices":[1],"values":[0.5]}"#) - #expect(row.cell(for: "vector.text") == nil) - } - - @Test("A query result shows the score after the id") - func queryColumns() throws { - let json = try JSONSerialization.jsonObject(with: Data( - #"{"result":{"points":[{"id":1,"score":0.9,"payload":{"city":"Berlin"},"vector":[1,2]}]},"status":"ok"}"#.utf8 - )) - let points = try #require(QdrantPoint.list(fromResponse: json)) - let columns = QdrantPoint.columns(of: points, collection: nil) - #expect(columns == ["id", "score", "city", "vector"]) - #expect(points[0].row(columns: columns) == ["1", "0.9", "Berlin", "[1,2]"]) - #expect(QdrantPoint.list(fromResponse: ["result": ["count": 3]]) == nil) - } - - @Test("A payload edit is a set-payload request typed by the column") - func update() throws { - let batch = QdrantStatementGenerator.generate(collection: sampleCollection, changes: [ - QdrantTrackedChange(kind: .update, id: "7", cellChanges: [ - QdrantCellChange(column: "note", newText: "42"), - QdrantCellChange(column: "count", newText: "42"), - QdrantCellChange(column: "vector", newText: "[1]") - ]) - ]) - let request = try #require(batch.requests.first) - #expect(request.method == "POST") - #expect(request.path == "/collections/docs/points/payload?wait=true") - #expect(request.body == #"{"payload":{"count":42,"note":"42"},"points":[7]}"#) - } - - @Test("A delete names the point, and a row without an id is skipped, not sent") - func delete() throws { - let uuid = "5c56c793-69f3-4fbf-87e6-c4bf54c28c26" - let batch = QdrantStatementGenerator.generate(collection: sampleCollection, changes: [ - QdrantTrackedChange(kind: .delete, id: uuid), - QdrantTrackedChange(kind: .delete, id: nil), - QdrantTrackedChange(kind: .update, id: "1", cellChanges: [QdrantCellChange(column: "id", newText: "2")]) - ]) - #expect(batch.requests.map(QdrantWriteCodec.encode) == [ - "POST /collections/docs/points/delete?wait=true\n{\"points\":[\"\(uuid)\"]}" - ]) - #expect(batch.skipped.map(\.reason) == [.missingID, .noEditableColumns]) - } - - @Test("An insert never overwrites an existing id and gets a UUID when left blank") - func insert() throws { - let batch = QdrantStatementGenerator.generate( - collection: sampleCollection, - changes: [QdrantTrackedChange(kind: .insert, id: nil, values: ["city": "Hanoi", "active": "true"])], - uuid: { "00000000-0000-0000-0000-000000000001" } - ) - let request = try #require(batch.requests.first) - #expect(request.method == "PUT") - #expect(request.path == "/collections/docs/points?wait=true") - #expect(request.body == #"{"points":[{"id":"00000000-0000-0000-0000-000000000001","payload":{"active":true,"city":"Hanoi"},"vector":{}}],"update_mode":"insert_only"}"#) - } - - @Test("Browse requests round-trip through their tag") - func browseRoundTrip() throws { - let request = QdrantBrowseRequest( - collection: "docs", - offset: 200, - limit: 100, - sort: QdrantSortSpec(column: "pop", ascending: false), - filters: [QdrantFilterSpec(column: "count", op: "BETWEEN", value: "1,5", secondValue: "5")], - logicMode: "OR" - ) - #expect(QdrantBrowseQuery.parse(QdrantBrowseQuery.encode(request)) == request) - } - - @Test("A sorted page uses order_by with a numeric offset") - func sortedBody() { - let request = QdrantBrowseRequest(collection: "docs", offset: 50, limit: 25, sort: nil, filters: [], logicMode: "AND") - let body = QdrantBrowseQuery.queryBody(for: request, filter: nil, sort: QdrantSortSpec(column: "pop", ascending: false)) - #expect(canonical(body) == #"{"limit":25,"offset":50,"query":{"order_by":{"direction":"desc","key":"pop"}},"with_payload":true,"with_vector":true}"#) - } - - @Test("The console reads a request line with its body below or beside it") - func console() { - let below = QdrantConsoleParser.parse("POST /collections/docs/points/query\n{\"limit\": 3}") - #expect(below == QdrantConsoleRequest(method: "POST", path: "/collections/docs/points/query", body: "{\"limit\": 3}")) - let inline = QdrantConsoleParser.parse("post /collections/docs/points/count {\"exact\": true}") - #expect(inline?.method == "POST") - #expect(inline?.body == "{\"exact\": true}") - #expect(QdrantConsoleParser.parse("SELECT 1") == nil) - #expect(QdrantConsoleParser.parse("GET collections") == nil) - } - - @Test("Only the read endpoints count as reads") - func readEndpoints() { - #expect(QdrantConsoleRequest(method: "POST", path: "/collections/docs/points/query", body: nil).isRead) - #expect(QdrantConsoleRequest(method: "POST", path: "/collections/docs/points", body: nil).isRead) - #expect(!QdrantConsoleRequest(method: "POST", path: "/collections/docs/points/delete?wait=true", body: nil).isRead) - #expect(!QdrantConsoleRequest(method: "PUT", path: "/collections/docs/points", body: nil).isRead) - } - - @Test("Point ids parse back to the kind Qdrant expects") - func pointIDs() { - #expect(QdrantPointID.json(from: "42") as? NSNumber == 42) - #expect(QdrantPointID.json(from: "5C56C793-69F3-4FBF-87E6-C4BF54C28C26") as? String == "5c56c793-69f3-4fbf-87e6-c4bf54c28c26") - #expect(QdrantPointID.json(from: "-1") == nil) - #expect(QdrantPointID.json(from: "abc") == nil) - } - - @Test("A cell is written back as its column's type") - func parsedValues() { - #expect(QdrantJSON.parsedValue("42", fieldType: .keyword) as? String == "42") - #expect(QdrantJSON.parsedValue("42", fieldType: .integer) as? Int64 == 42) - #expect(QdrantJSON.parsedValue("false", fieldType: .bool) as? Bool == false) - #expect((QdrantJSON.parsedValue(#"["a"]"#, fieldType: .array) as? [String]) == ["a"]) - #expect(QdrantJSON.parsedValue("not json", fieldType: .object) as? String == "not json") - } -} diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift new file mode 100644 index 0000000000..a1eba2ef8b --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift @@ -0,0 +1,128 @@ +import Foundation +@testable import TableProQdrantCore +import Testing + +struct QdrantSchemaTests { + private func info(vectors: Any, sparse: [String: Any]? = nil, payloadSchema: [String: Any] = [:]) -> [String: Any] { + var params: [String: Any] = ["vectors": vectors] + params["sparse_vectors"] = sparse + return ["result": ["points_count": 4, "config": ["params": params], "payload_schema": payloadSchema]] + } + + private func point(_ json: String) throws -> QdrantPoint { + let object = try JSONSerialization.jsonObject(with: Data(json.utf8)) + return try #require(QdrantPoint.parse(object)) + } + + @Test("Columns are the id, indexed fields, sampled fields, then vectors typed by their size") + func columns() throws { + let sample = [ + try point(#"{"id":1,"payload":{"city":"Berlin","count":3,"meta":{"x":1}}}"#), + try point(#"{"id":2,"payload":{"count":3.5,"tags":["a"]}}"#) + ] + let collection = QdrantCollection.parse( + name: "docs", + info: info(vectors: ["size": 4, "distance": "Cosine"], payloadSchema: ["city": ["data_type": "keyword"]]), + sample: sample + ) + let columns = QdrantSchema.columns(for: collection) + #expect(columns.map(\.name) == ["id", "city", "count", "meta", "tags", "vector"]) + #expect(columns.map(\.type) == ["id", "keyword", "float", "object", "array", "vector(4)"]) + #expect(collection.pointsCount == 4) + } + + @Test("A nested index path is an index, not a column that would always read NULL") + func nestedIndexPaths() throws { + let collection = QdrantCollection.parse( + name: "docs", + info: info(vectors: ["size": 2], payloadSchema: [ + "metadata.source": ["data_type": "keyword"], + "tags[]": ["data_type": "keyword"], + "\"a.b\"": ["data_type": "integer"], + "rank": ["data_type": "integer"] + ]), + sample: [try point(#"{"id":1,"payload":{"metadata":{"source":"a.pdf"},"tags":["x"]}}"#)] + ) + #expect(QdrantSchema.columns(for: collection).map(\.name) == ["id", "a.b", "rank", "metadata", "tags", "vector"]) + #expect(collection.indexes.map(\.path) == ["\"a.b\"", "metadata.source", "rank", "tags[]"]) + #expect(collection.fieldType(ofColumn: "metadata") == .object) + } + + @Test("A payload key named like a point column shows as payload. and maps back") + func reservedPayloadKeys() throws { + let row = try point(#"{"id":7,"payload":{"id":"external-1","vector":"text","city":"Berlin"},"vector":[1,2]}"#) + let collection = QdrantCollection.parse(name: "docs", info: info(vectors: ["size": 2]), sample: [row]) + let columns = QdrantSchema.columns(for: collection).map(\.name) + #expect(columns == ["id", "city", "payload.id", "payload.vector", "vector"]) + #expect(row.row(columns: columns) == ["7", "Berlin", "external-1", "text", "[1,2]"]) + #expect(QdrantSchema.payloadKey(forColumn: "payload.id") == "id") + #expect(QdrantSchema.payloadKey(forColumn: "payload.city") == "payload.city") + } + + @Test("Keys outside the sample join the collection, and a key first seen as null takes its later type") + func includingLaterPoints() throws { + let sample = [try point(#"{"id":1,"payload":{"city":"Berlin","rank":null}}"#)] + let collection = QdrantCollection.parse(name: "docs", info: info(vectors: ["size": 2]), sample: sample) + let widened = collection.including([try point(#"{"id":2,"payload":{"late":true,"rank":3}}"#)]) + #expect(widened.payloadFields.map(\.name) == ["city", "rank", "late"]) + #expect(widened.fieldType(ofColumn: "rank") == .integer) + #expect(widened.fieldType(ofColumn: "late") == .bool) + } + + @Test("Named, multi and sparse vectors each get a column typed by their kind") + func namedVectors() throws { + let collection = QdrantCollection.parse( + name: "docs", + info: info( + vectors: ["image": ["size": 2], "text": ["size": 3, "multivector_config": ["comparator": "max_sim"]]], + sparse: ["keywords": [String: Any]()] + ) + ) + let columns = QdrantSchema.columns(for: collection) + #expect(columns.map(\.name) == ["id", "vector.image", "vector.text", "vector.keywords"]) + #expect(columns.map(\.type) == ["id", "vector(2)", "multivector", "sparse_vector"]) + + let row = try point(#"{"id":"5c56c793-69f3-4fbf-87e6-c4bf54c28c26","vector":{"image":[0.5,1],"keywords":{"indices":[1],"values":[0.5]}}}"#) + #expect(row.cell(for: "id") == "5c56c793-69f3-4fbf-87e6-c4bf54c28c26") + #expect(row.cell(for: "vector.image") == "[0.5,1]") + #expect(row.cell(for: "vector.keywords") == #"{"indices":[1],"values":[0.5]}"#) + #expect(row.cell(for: "vector.text") == nil) + } + + @Test("An unnamed vector beside sparse ones comes back under the empty name") + func unnamedBesideSparse() throws { + let row = try point(#"{"id":1,"vector":{"":[1,2],"sp":{"indices":[0],"values":[1]}}}"#) + #expect(row.cell(for: "vector") == "[1,2]") + #expect(QdrantPoint.columns(of: [row], collection: nil) == ["id", "vector", "vector.sp"]) + } + + @Test("A vector type name maps back to its kind") + func vectorKinds() { + #expect(QdrantVectorSpec.kind(ofTypeName: "vector(384)") == .dense(size: 384)) + #expect(QdrantVectorSpec.kind(ofTypeName: "VECTOR") == .dense(size: nil)) + #expect(QdrantVectorSpec.kind(ofTypeName: "multivector") == .multi) + #expect(QdrantVectorSpec.kind(ofTypeName: "sparse_vector") == .sparse) + #expect(QdrantVectorSpec.kind(ofTypeName: "keyword") == nil) + } + + @Test("A query result shows the score after the id") + func queryColumns() throws { + let json = try JSONSerialization.jsonObject(with: Data( + #"{"result":{"points":[{"id":1,"score":0.9,"payload":{"city":"Berlin"},"vector":[1,2]}]},"status":"ok"}"#.utf8 + )) + let points = try #require(QdrantPoint.list(fromResponse: json)) + let columns = QdrantPoint.columns(of: points, collection: nil) + #expect(columns == ["id", "score", "city", "vector"]) + #expect(points[0].row(columns: columns) == ["1", "0.9", "Berlin", "[1,2]"]) + #expect(QdrantPoint.list(fromResponse: ["result": ["count": 3]]) == nil) + #expect(QdrantPoint.list(fromResponse: ["result": [Any]()])?.isEmpty == true) + } + + @Test("Point ids parse back to the kind Qdrant expects") + func pointIDs() { + #expect(QdrantPointID.json(from: "42") as? NSNumber == 42) + #expect(QdrantPointID.json(from: "5C56C793-69F3-4FBF-87E6-C4BF54C28C26") as? String == "5c56c793-69f3-4fbf-87e6-c4bf54c28c26") + #expect(QdrantPointID.json(from: "-1") == nil) + #expect(QdrantPointID.json(from: "abc") == nil) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantTestSupport.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantTestSupport.swift index 7c907a2f13..3ac74609d2 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantTestSupport.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantTestSupport.swift @@ -3,18 +3,15 @@ import Foundation final class RecordingTransport: QdrantTransport, @unchecked Sendable { private let lock = NSLock() - private var handler: (QdrantHTTPRequest) throws -> QdrantHTTPResponse - private(set) var requests: [QdrantHTTPRequest] = [] + private let handler: (QdrantHTTPRequest) throws -> QdrantHTTPResponse + private var requests: [QdrantHTTPRequest] = [] init(_ handler: @escaping (QdrantHTTPRequest) throws -> QdrantHTTPResponse) { self.handler = handler } func send(_ request: QdrantHTTPRequest) async throws -> QdrantHTTPResponse { - let handler = lock.withLock { - requests.append(request) - return self.handler - } + lock.withLock { requests.append(request) } return try handler(request) } @@ -34,37 +31,42 @@ func bodyObject(_ request: QdrantHTTPRequest) -> [String: Any] { func makeClient(apiKey: String = "", transport: RecordingTransport) -> QdrantClient { QdrantClient( - settings: QdrantConnectionSettings(host: "localhost", port: 6_333, usesTLS: false, apiKey: apiKey, skipTLSVerify: false), + settings: QdrantConnectionSettings(host: "localhost", port: 6_333, usesTLS: false, apiKey: apiKey), transport: transport, timeout: { 5 } ) } -/// A collection of integer ids 1...count answering `scroll` the way Qdrant does: from `offset` -/// (an id) up to `limit` points, with the next id as the cursor. +func scrollResponse(_ ids: [Int], next: Int?) -> QdrantHTTPResponse { + let points = ids.map { ["id": $0, "payload": ["n": $0]] as [String: Any] } + return jsonResponse(["result": ["points": points, "next_page_offset": next.map { $0 as Any } ?? NSNull()], "status": "ok"]) +} + +// Integer ids 1...count, answering `scroll` from `offset` (an id) the way Qdrant does. func scrollingTransport(count: Int) -> RecordingTransport { RecordingTransport { request in let body = bodyObject(request) let limit = body["limit"] as? Int ?? 10 - let start = (body["offset"] as? Int) ?? 1 + let start = body["offset"] as? Int ?? 1 let ids = Array(start.. [ConnectionField] { fieldType: .secure, section: .authentication, hidesPassword: true - ).withHidesUsername(true), - ConnectionField( - id: QdrantFieldID.skipTLSVerify, - label: String(localized: "Skip TLS Verification"), - defaultValue: "false", - fieldType: .toggle, - section: .advanced - ) + ).withHidesUsername(true) ] } @@ -98,6 +91,13 @@ let qdrantCompletions: [CompletionEntry] = [ POST /collections/docs/points/count {"exact": true} """ + ), + CompletionEntry( + label: "POST /facet", + insertText: """ + POST /collections/docs/facet + {"key": "city"} + """ ) ] @@ -107,5 +107,5 @@ let qdrantColumnTypes: [String: [String]] = [ "Boolean": ["bool"], "Date": ["datetime"], "Structured": ["geo", "object", "array", "json"], - "Vector": ["vector", "sparse_vector"] + "Vector": ["vector", "multivector", "sparse_vector"] ] diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift index 4102b56433..cf57e7d4eb 100644 --- a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift @@ -39,18 +39,15 @@ extension QdrantPluginDriver { let collection = try await collection(request.collection) let filter = try QdrantFilterBuilder.filter(request.filters, logicMode: request.logicMode, collection: collection) let points: [QdrantPoint] - if let sort = request.sort { - points = try await client.query( - collection: request.collection, - body: QdrantBrowseQuery.queryBody(for: request, filter: filter, sort: sort) - ) - } else { + switch try QdrantSortPlan.resolve(request.sorts) { + case .byID: points = try await client.scrollPage( - collection: request.collection, - filter: filter, - offset: request.offset, - limit: request.limit, - cache: cursors + collection: request.collection, filter: filter, offset: request.offset, limit: request.limit, cache: paging + ) + case .byPayload(let column, let key, let ascending): + points = try await client.sortedPage( + collection: request.collection, filter: filter, column: column, key: key, ascending: ascending, + offset: request.offset, limit: request.limit, cache: paging ) } let columns = QdrantSchema.columns(for: collection).map(\.name) @@ -63,16 +60,17 @@ extension QdrantPluginDriver { started: Date ) async throws -> PluginQueryResult { let response = try await client.execute(request) - if !request.isRead { + if request.kind.tier != .read { forgetAfterWrite() } - if let points = QdrantPoint.list(fromResponse: response.json), !points.isEmpty { - let name = Self.collectionName(in: request.path) + // An empty point list is a grid with no rows, which is what a save's existence check counts. + let route = QdrantPathEncoding.segments(of: request.path) ?? [] + if let points = QdrantPoint.list(fromResponse: response.json), !points.isEmpty || route.contains("points") { + let name = route.count >= 2 && route[0] == "collections" ? Self.collectionName(in: request.path) : nil let collection = name.flatMap(rememberedCollection) - let columns = QdrantPoint.columns(of: points, collection: collection) return render( points, - columns: columns, + columns: QdrantPoint.columns(of: points, collection: collection), collection: collection ?? QdrantCollection(name: name ?? "", vectors: [], payloadFields: []), started: started ) @@ -95,12 +93,16 @@ extension QdrantPluginDriver { PluginQueryResult( columns: columns, columnTypeNames: columns.map { QdrantSchema.typeName(of: $0, in: collection) }, - rows: points.map { point in point.row(columns: columns).map { $0.map(PluginCellValue.text) ?? .null } }, + rows: points.map(cells(columns: columns)), rowsAffected: 0, executionTime: Date().timeIntervalSince(started) ) } + private func cells(columns: [String]) -> (QdrantPoint) -> [PluginCellValue] { + { point in point.row(columns: columns).map { $0.map(PluginCellValue.text) ?? .null } } + } + private func renderJSON(_ response: QdrantHTTPResponse, started: Date) -> PluginQueryResult { let text: String if let json = response.json, JSONSerialization.isValidJSONObject(json), @@ -120,8 +122,7 @@ extension QdrantPluginDriver { // MARK: - Export - /// Walks the collection a page at a time instead of buffering it whole, which the protocol's - /// default `streamRows` would do by running `execute` once. + // The protocol's default `streamRows` runs `execute` once and buffers the whole collection. func streamRows(query: String) -> AsyncThrowingStream { AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in let task = Task { @@ -146,14 +147,14 @@ extension QdrantPluginDriver { } } + // The grid's columns come from a sample; an export carries every key, so it first reads payloads + // alone, which skips the vectors that make up most of a page. private func streamCollection( _ name: String, continuation: AsyncThrowingStream.Continuation ) async throws { let client = try requireClient() var collection = try await collection(name, refresh: true) - // The grid's columns come from a sample; an export has to carry every key, so it reads - // payloads alone first, which skips the vectors that make up most of a page. for try await points in client.pointPages(collection: name, pageSize: Self.exportPageSize, withVector: false) { collection = collection.including(points) } @@ -164,9 +165,7 @@ extension QdrantPluginDriver { estimatedRowCount: collection.pointsCount ))) for try await points in client.pointPages(collection: name, pageSize: Self.exportPageSize) { - continuation.yield(.rows(points.map { point in - point.row(columns: columns).map { $0.map(PluginCellValue.text) ?? .null } - })) + continuation.yield(.rows(points.map(cells(columns: columns)))) } } } diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift index d18a73f8aa..7cb996c4d1 100644 --- a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift @@ -1,5 +1,4 @@ import Foundation -import os import TableProPluginKit import TableProQdrantCore @@ -17,8 +16,7 @@ extension QdrantPluginDriver { } func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { - let collection = try await collection(table, refresh: true) - return QdrantSchema.columns(for: collection).map { column in + QdrantSchema.columns(for: try await collection(table, refresh: true)).map { column in PluginColumnInfo( name: column.name, dataType: column.type, @@ -29,30 +27,21 @@ extension QdrantPluginDriver { } func fetchImmutableColumns(table: String, schema: String?) async throws -> Set { - Set(QdrantSchema.immutableColumns(for: try await collection(table))) + [QdrantSchema.idColumn] } func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { - let collection = try await collection(table) - var indexes = [ - PluginIndexInfo( - name: QdrantSchema.idColumn, - columns: [QdrantSchema.idColumn], - isUnique: true, - isPrimary: true, - type: "PRIMARY KEY" - ) - ] - indexes += collection.payloadFields.filter(\.isIndexed).map { field in - PluginIndexInfo( - name: field.name, - columns: [field.name], - isUnique: false, - isPrimary: false, - type: field.type.rawValue - ) + let collection = try await requireClient().collectionInfo(named: table) + let primary = PluginIndexInfo( + name: QdrantSchema.idColumn, + columns: [QdrantSchema.idColumn], + isUnique: true, + isPrimary: true, + type: "PRIMARY KEY" + ) + return [primary] + collection.indexes.map { index in + PluginIndexInfo(name: index.path, columns: [index.path], isUnique: false, isPrimary: false, type: index.type) } - return indexes } func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { @@ -60,7 +49,7 @@ extension QdrantPluginDriver { } func fetchTableDDL(table: String, schema: String?) async throws -> String { - try await collection(table, refresh: true).info + try await requireClient().collectionInfo(named: table).info } func fetchViewDefinition(view: String, schema: String?) async throws -> String { @@ -72,7 +61,7 @@ extension QdrantPluginDriver { } func fetchApproximateRowCount(table: String, schema: String?) async throws -> Int? { - try await collection(table, refresh: true).pointsCount + try await requireClient().collectionInfo(named: table).pointsCount } func fetchFilteredRowCount(table: String, queryFilters: [PluginQueryFilter], logicMode: String) async throws -> Int? { @@ -101,8 +90,7 @@ extension QdrantPluginDriver { ) } - /// Qdrant orders by one payload key only, so the first sortable column wins. The id is the - /// order a page already has, and a vector has no order. + // Every sort column travels with the page, so one Qdrant cannot honor is refused instead of dropped. func buildFilteredQuery( table: String, schema: String?, @@ -114,17 +102,15 @@ extension QdrantPluginDriver { offset: Int, columnKinds: [String: PluginColumnKind] ) -> String? { - let sort = sortColumns.lazy.compactMap { sort -> QdrantSortSpec? in - guard sort.columnIndex >= 0, sort.columnIndex < columns.count else { return nil } - let column = columns[sort.columnIndex] - guard column != QdrantSchema.idColumn, !QdrantSchema.isVectorColumn(column) else { return nil } - return QdrantSortSpec(column: column, ascending: sort.ascending) - }.first + let sorts = sortColumns.compactMap { sort -> QdrantSortSpec? in + guard columns.indices.contains(sort.columnIndex) else { return nil } + return QdrantSortSpec(column: columns[sort.columnIndex], ascending: sort.ascending) + } return QdrantBrowseQuery.encode(QdrantBrowseRequest( collection: table, offset: offset, limit: limit, - sort: sort, + sorts: sorts, filters: specs(filters), logicMode: logicMode )) @@ -136,66 +122,82 @@ extension QdrantPluginDriver { } } - func generateStatements( + // Typed by what the grid read each column as, so a cache a console write or a reconnect cleared + // can never turn a number into text. + func generateRowWrites( table: String, + schema: String?, columns: [String], primaryKeyColumns: [String], changes: [PluginRowChange], insertedRowData: [Int: [PluginCellValue]], deletedRowIndices: Set, - insertedRowIndices: Set - ) -> [(statement: String, parameters: [PluginCellValue])]? { - let collection = rememberedCollection(table) - ?? QdrantCollection(name: table, vectors: [], payloadFields: []) + insertedRowIndices: Set, + context: PluginRowWriteContext + ) throws -> [PluginRowWrite]? { + let known = rememberedCollection(table) + let typeNames = context.columnTypeNames let tracked = changes.compactMap { change in - trackedChange( - change, - columns: columns, - insertedRowData: insertedRowData, - deletedRowIndices: deletedRowIndices, - insertedRowIndices: insertedRowIndices - ) + rowChange(change, columns: columns, insertedRowData: insertedRowData, + deletedRowIndices: deletedRowIndices, insertedRowIndices: insertedRowIndices) } - let batch = QdrantStatementGenerator.generate(collection: collection, changes: tracked) - for skipped in batch.skipped { - Self.logger.warning( - "Skipped a \(skipped.kind.rawValue, privacy: .public) on \(table, privacy: .private): \(skipped.reason.rawValue, privacy: .public)" - ) + do { + return try QdrantRowWriter.writes(collection: table, changes: tracked) { column in + typeNames[column] ?? known.flatMap { QdrantSchema.knownTypeName(of: column, in: $0) } + }.map { write in + PluginRowWrite( + statement: write.request.text, + rowIndices: [write.rowIndex], + lookup: write.mustFindNothing.map { + PluginRowLookup(query: $0.text, parameters: [], rowCount: 0, pinsRows: false) + } + ) + } + } catch let refusal as QdrantRowRefusal { + throw PluginRowWriteRefusal(rowIndex: refusal.rowIndex, reason: refusal.reason) } - return batch.requests.map { (QdrantWriteCodec.encode($0), []) } } - private func trackedChange( + func generateRowWrites( + table: String, + schema: String?, + columns: [String], + primaryKeyColumns: [String], + changes: [PluginRowChange], + insertedRowData: [Int: [PluginCellValue]], + deletedRowIndices: Set, + insertedRowIndices: Set + ) throws -> [PluginRowWrite]? { + try generateRowWrites( + table: table, schema: schema, columns: columns, primaryKeyColumns: primaryKeyColumns, changes: changes, + insertedRowData: insertedRowData, deletedRowIndices: deletedRowIndices, + insertedRowIndices: insertedRowIndices, context: PluginRowWriteContext() + ) + } + + private func rowChange( _ change: PluginRowChange, columns: [String], insertedRowData: [Int: [PluginCellValue]], deletedRowIndices: Set, insertedRowIndices: Set - ) -> QdrantTrackedChange? { + ) -> QdrantRowChange? { switch change.type { case .insert: guard insertedRowIndices.contains(change.rowIndex) else { return nil } - var values: [String: String?] = [:] + let cells: [QdrantCell] if let row = insertedRowData[change.rowIndex] { - for (index, column) in columns.enumerated() where index < row.count { - guard let text = row[index].asText else { continue } - values[column] = text - } + cells = zip(columns, row).map { QdrantCell(column: $0, text: $1.asText) } } else { - for cell in change.cellChanges { - guard let text = cell.newValue.asText else { continue } - values[cell.columnName] = text - } + cells = change.cellChanges.map { QdrantCell(column: $0.columnName, text: $0.newValue.asText) } } - return QdrantTrackedChange(kind: .insert, id: values[QdrantSchema.idColumn] ?? nil, values: values) + return QdrantRowChange(kind: .insert, rowIndex: change.rowIndex, id: nil, cells: cells) case .update: - let cells = change.cellChanges.map { - QdrantCellChange(column: $0.columnName, newText: $0.newValue.asText) - } - return QdrantTrackedChange(kind: .update, id: pointID(from: change, columns: columns), cellChanges: cells) + let cells = change.cellChanges.map { QdrantCell(column: $0.columnName, text: $0.newValue.asText) } + return QdrantRowChange(kind: .update, rowIndex: change.rowIndex, id: pointID(from: change, columns: columns), cells: cells) case .delete: guard deletedRowIndices.contains(change.rowIndex) else { return nil } - return QdrantTrackedChange(kind: .delete, id: pointID(from: change, columns: columns)) + return QdrantRowChange(kind: .delete, rowIndex: change.rowIndex, id: pointID(from: change, columns: columns)) } } diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift index 22edbe01e6..357ce143bd 100644 --- a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift @@ -10,7 +10,7 @@ internal final class QdrantPluginDriver: PluginDatabaseDriver, @unchecked Sendab private let lock = NSLock() private var client: QdrantClient? private var cachedCollections: [String: QdrantCollection] = [:] - let cursors = QdrantCursorCache() + let paging = QdrantPagingCache() let queryTimeout = HttpQueryTimeoutBox() init(config: DriverConnectionConfig) { @@ -36,8 +36,6 @@ internal final class QdrantPluginDriver: PluginDatabaseDriver, @unchecked Sendab usesTLS: config.ssl.isEnabled, fields: config.additionalFields ) - let skipTLS = settings.skipTLSVerify - || (config.ssl.isEnabled && !config.ssl.verifiesCertificate) let deadline = PluginConnectDeadline(milliseconds: PluginConnectTimeout.milliseconds( in: config.additionalFields, default: Int(HttpQueryTimeout().requestTimeoutInterval * 1_000) @@ -47,7 +45,7 @@ internal final class QdrantPluginDriver: PluginDatabaseDriver, @unchecked Sendab settings: settings, transport: URLSessionQdrantTransport( resourceTimeout: HttpQueryTimeout.sessionResourceTimeout, - skipTLSVerify: skipTLS + tls: config.ssl.isEnabled ? try QdrantTLS.make(from: config.ssl) : .systemDefault ), timeout: { timeout.requestTimeoutInterval } ) @@ -56,7 +54,7 @@ internal final class QdrantPluginDriver: PluginDatabaseDriver, @unchecked Sendab self.client = client cachedCollections.removeAll() } - cursors.removeAll() + paging.removeAll() } func disconnect() { @@ -65,7 +63,7 @@ internal final class QdrantPluginDriver: PluginDatabaseDriver, @unchecked Sendab client = nil cachedCollections.removeAll() } - cursors.removeAll() + paging.removeAll() } func ping() async throws { @@ -100,10 +98,10 @@ internal final class QdrantPluginDriver: PluginDatabaseDriver, @unchecked Sendab return collection } - /// A write moves rows between pages and may add payload keys, so both caches start over. + // A write moves points between pages and may add payload keys. func forgetAfterWrite() { lock.withLock { cachedCollections.removeAll() } - cursors.removeAll() + paging.removeAll() } func defaultExportQuery(table: String) -> String? { diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantURLSessionTransport.swift b/apple/Plugins/QdrantDriverPlugin/QdrantURLSessionTransport.swift new file mode 100644 index 0000000000..1e9220f72e --- /dev/null +++ b/apple/Plugins/QdrantDriverPlugin/QdrantURLSessionTransport.swift @@ -0,0 +1,222 @@ +import Foundation +import Security +import TableProPluginKit +import TableProQdrantCore +import TableProTLSClientIdentity + +struct QdrantTLS: @unchecked Sendable { + enum Verification: Sendable, Equatable { + case none + case caOnly + case full + } + + let verification: Verification + let anchorDER: Data? + let clientCredential: URLCredential? + + static let systemDefault = QdrantTLS(verification: .full, anchorDER: nil, clientCredential: nil) + + // Preferred and Required encrypt without checking the certificate, as the SSL Mode picker says. + static func make(from ssl: SSLConfiguration) throws -> QdrantTLS { + let verification: Verification + switch ssl.mode { + case .disabled, .preferred, .required: verification = .none + case .verifyCa: verification = .caOnly + case .verifyIdentity: verification = .full + } + let anchor = try anchorCertificate(at: ssl.caCertificatePath, for: verification) + if verification == .caOnly, anchor == nil { + throw QdrantError.configuration(String(localized: """ + Verify CA needs a CA certificate. On the connection's Network tab, choose the CA certificate \ + that signed the server's certificate, or set SSL Mode to Verify Identity. + """)) + } + return QdrantTLS(verification: verification, anchorDER: anchor, clientCredential: try clientCredential(for: ssl)) + } + + private static func anchorCertificate(at path: String, for verification: Verification) throws -> Data? { + let trimmed = path.trimmingCharacters(in: .whitespaces) + guard verification != .none, !trimmed.isEmpty else { return nil } + guard let data = try? Data(contentsOf: URL(fileURLWithPath: trimmed)), + let der = PEMCertificateDecoder.certificateDER(from: data), + SecCertificateCreateWithData(nil, der as CFData) != nil + else { + throw QdrantError.configuration(String( + format: String(localized: "The CA certificate at %@ could not be read as a PEM or DER certificate."), + trimmed + )) + } + return der + } + + private static func clientCredential(for ssl: SSLConfiguration) throws -> URLCredential? { + let certificatePath = ssl.clientCertificatePath.trimmingCharacters(in: .whitespaces) + let keyPath = ssl.clientKeyPath.trimmingCharacters(in: .whitespaces) + guard ssl.isEnabled, !certificatePath.isEmpty else { return nil } + guard !keyPath.isEmpty else { + throw QdrantError.configuration(String(localized: """ + A client certificate needs its client key. On the connection's Network tab, choose the \ + client key, or clear the client certificate. + """)) + } + do { + return try TLSClientIdentity.credential( + certificateFile: URL(fileURLWithPath: certificatePath), + privateKeyFile: URL(fileURLWithPath: keyPath) + ) + } catch let failure as TLSClientIdentityError { + throw QdrantError.configuration(failure.message(certificatePath: certificatePath, keyPath: keyPath)) + } + } +} + +final class URLSessionQdrantTransport: QdrantTransport, @unchecked Sendable { + private let session: URLSession + private let tls: QdrantTLS + private let lock = NSLock() + private var inFlight: [ObjectIdentifier: URLSessionTask] = [:] + + // No session delegate: URLSession then hands the trust and client certificate challenges to the + // per-task delegate, which can say why it refused instead of surfacing a bare cancel. + init(resourceTimeout: TimeInterval, tls: QdrantTLS) { + let configuration = URLSessionConfiguration.ephemeral + configuration.timeoutIntervalForResource = resourceTimeout + session = URLSession(configuration: configuration) + self.tls = tls + } + + deinit { + session.invalidateAndCancel() + } + + func cancelAll() { + let tasks = lock.withLock { Array(inFlight.values) } + tasks.forEach { $0.cancel() } + } + + func send(_ request: QdrantHTTPRequest) async throws -> QdrantHTTPResponse { + var urlRequest = URLRequest(url: request.url) + urlRequest.httpMethod = request.method + urlRequest.httpBody = request.body + urlRequest.timeoutInterval = request.timeoutInterval + for (name, value) in request.headers { + urlRequest.setValue(value, forHTTPHeaderField: name) + } + + let delegate = QdrantTaskDelegate(transport: self, tls: tls) + defer { delegate.finish() } + do { + let (data, response) = try await session.data(for: urlRequest, delegate: delegate) + guard let http = response as? HTTPURLResponse else { + throw QdrantError.transport(String(localized: "Qdrant answered with something other than HTTP.")) + } + return QdrantHTTPResponse(statusCode: http.statusCode, body: data) + } catch let error as URLError where error.code == .cancelled { + if let refusal = delegate.refusal { + throw QdrantError.transport(refusal) + } + throw QdrantError.cancelled + } catch let error as QdrantError { + throw error + } catch let error as URLError { + throw QdrantError.transport(error.localizedDescription) + } + } + + fileprivate func register(_ task: URLSessionTask) { + lock.withLock { inFlight[ObjectIdentifier(task)] = task } + } + + fileprivate func unregister(_ task: URLSessionTask) { + lock.withLock { inFlight[ObjectIdentifier(task)] = nil } + } +} + +private final class QdrantTaskDelegate: NSObject, URLSessionTaskDelegate, @unchecked Sendable { + private weak var transport: URLSessionQdrantTransport? + private let tls: QdrantTLS + private let lock = NSLock() + private var task: URLSessionTask? + private var refusalMessage: String? + + init(transport: URLSessionQdrantTransport, tls: QdrantTLS) { + self.transport = transport + self.tls = tls + } + + var refusal: String? { lock.withLock { refusalMessage } } + + func urlSession(_ session: URLSession, didCreateTask task: URLSessionTask) { + lock.withLock { self.task = task } + transport?.register(task) + } + + func urlSession( + _ session: URLSession, + task: URLSessionTask, + didReceive challenge: URLAuthenticationChallenge, + completionHandler: @escaping (URLSession.AuthChallengeDisposition, URLCredential?) -> Void + ) { + switch challenge.protectionSpace.authenticationMethod { + case NSURLAuthenticationMethodServerTrust: + guard let trust = challenge.protectionSpace.serverTrust else { + completionHandler(.performDefaultHandling, nil) + return + } + if let failure = evaluate(trust) { + lock.withLock { refusalMessage = failure } + completionHandler(.cancelAuthenticationChallenge, nil) + } else if tls.verification == .full, tls.anchorDER == nil { + completionHandler(.performDefaultHandling, nil) + } else { + completionHandler(.useCredential, URLCredential(trust: trust)) + } + case NSURLAuthenticationMethodClientCertificate: + if let credential = tls.clientCredential { + completionHandler(.useCredential, credential) + } else { + completionHandler(.performDefaultHandling, nil) + } + default: + completionHandler(.performDefaultHandling, nil) + } + } + + // A redirect would carry the API key to wherever it points. + func urlSession( + _ session: URLSession, + task: URLSessionTask, + willPerformHTTPRedirection response: HTTPURLResponse, + newRequest request: URLRequest, + completionHandler: @escaping (URLRequest?) -> Void + ) { + completionHandler(nil) + } + + func finish() { + guard let task = lock.withLock({ task }) else { return } + transport?.unregister(task) + } + + private func evaluate(_ trust: SecTrust) -> String? { + switch tls.verification { + case .none: + return nil + case .full where tls.anchorDER == nil: + return nil + case .caOnly, .full: + if let der = tls.anchorDER, let anchor = SecCertificateCreateWithData(nil, der as CFData) { + SecTrustSetAnchorCertificates(trust, [anchor] as CFArray) + SecTrustSetAnchorCertificatesOnly(trust, true) + } + if tls.verification == .caOnly { + SecTrustSetPolicies(trust, SecPolicyCreateBasicX509()) + } + var error: CFError? + guard !SecTrustEvaluateWithError(trust, &error) else { return nil } + let detail = error.map { CFErrorCopyDescription($0) as String } ?? "" + return String(format: String(localized: "The server's certificate is not trusted. %@"), detail) + } + } +} diff --git a/apple/TablePro/Core/DataWrite/DataWriteExecutor.swift b/apple/TablePro/Core/DataWrite/DataWriteExecutor.swift index d0bf6b9918..b0421096b8 100644 --- a/apple/TablePro/Core/DataWrite/DataWriteExecutor.swift +++ b/apple/TablePro/Core/DataWrite/DataWriteExecutor.swift @@ -289,13 +289,16 @@ enum DataWriteExecutor { logger.error( "Lookup on '\(table, privacy: .private(mask: .hash))' found \(rows.count, privacy: .public) rows, expected \(lookup.rowCount, privacy: .public)" ) - throw DataWriteError.changeRefused( - table: table, - kind: step.rowKind, - reason: rows.count < lookup.rowCount - ? String(localized: "No row holds the values it was loaded with. Another client may have changed or deleted it.") - : String(localized: "Other rows hold exactly the same values, so the change cannot reach this row alone.") - ) + let reason: String + if lookup.rowCount == 0 { + // A lookup that expects no row is an insert checking its key is free. + reason = String(localized: "A row with this key already exists, so the insert would not add one.") + } else if rows.count < lookup.rowCount { + reason = String(localized: "No row holds the values it was loaded with. Another client may have changed or deleted it.") + } else { + reason = String(localized: "Other rows hold exactly the same values, so the change cannot reach this row alone.") + } + throw DataWriteError.changeRefused(table: table, kind: step.rowKind, reason: reason) } // The row may have moved between the lookup and the write, or another row may now hold its old values, so a save diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift index d48af46cf2..8914415785 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift @@ -32,6 +32,7 @@ extension PluginMetadataRegistry { supportsDropIndex: false, supportsModifyPrimaryKey: false, supportsOpportunisticTLS: false, + verifiesServerWithSystemTrust: true, supportsPrincipalConnectionLimit: false ), schema: PluginMetadataSnapshot.SchemaInfo( @@ -40,7 +41,7 @@ extension PluginMetadataRegistry { tableEntityName: "Collections", containerEntityName: "Cluster", defaultPrimaryKeyColumn: "id", - immutableColumns: ["id", "vector"], + immutableColumns: ["id"], systemDatabaseNames: [], systemSchemaNames: [], fileExtensions: [], @@ -87,6 +88,13 @@ private let qdrantCompletions: [CompletionEntry] = [ POST /collections/docs/points/count {"exact": true} """ + ), + CompletionEntry( + label: "POST /facet", + insertText: """ + POST /collections/docs/facet + {"key": "city"} + """ ) ] @@ -96,7 +104,7 @@ private let qdrantColumnTypes: [String: [String]] = [ "Boolean": ["bool"], "Date": ["datetime"], "Structured": ["geo", "object", "array", "json"], - "Vector": ["vector", "sparse_vector"] + "Vector": ["vector", "multivector", "sparse_vector"] ] private func qdrantConnectionFields() -> [ConnectionField] { @@ -108,13 +116,6 @@ private func qdrantConnectionFields() -> [ConnectionField] { fieldType: .secure, section: .authentication, hidesPassword: true - ).withHidesUsername(true), - ConnectionField( - id: "qdSkipTLSVerify", - label: String(localized: "Skip TLS Verification"), - defaultValue: "false", - fieldType: .toggle, - section: .advanced - ) + ).withHidesUsername(true) ] } diff --git a/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift b/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift index 75827bca81..e766cb1c27 100644 --- a/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift +++ b/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift @@ -5,6 +5,7 @@ import Foundation import TableProPluginKit +import TableProQdrantCore import TableProRedisCore import TableProSQLGrammar import TableProUtilities @@ -1087,11 +1088,11 @@ private extension QueryClassifier { /// followed by a JSON body, and the body is the caller's data: scanning the whole statement /// let `POST /collections/c/documents/import` carrying `"note": "/multi_search"` in a field /// read as a search, which is a read-only mode and MCP gate bypass. A URL path holds no raw - /// space, so the path ends at the first one: that keeps a body written on the header line out - /// of it, and stops `POST /collections/c /multi_search` from ending in a read path. + /// whitespace, so the path ends at the first one: that keeps a body written on the header line + /// out of it, and stops `POST /collections/c /multi_search` from ending in a read path. static func typesenseRequestLine(_ trimmed: String) -> (verb: String, path: String) { - let header = trimmed.split(separator: "\n", maxSplits: 1).first.map(String.init) ?? "" - let parts = header.split(separator: " ", maxSplits: 1, omittingEmptySubsequences: true) + let header = trimmed.firstIndex(where: \.isNewline).map { String(trimmed[..<$0]) } ?? trimmed + let parts = header.split(maxSplits: 1, omittingEmptySubsequences: true, whereSeparator: \.isWhitespace) guard let verb = parts.first else { return ("", "/") } guard parts.count == 2 else { return (String(verb).uppercased(), "/") } @@ -1161,12 +1162,15 @@ private extension QueryClassifier { /// The driver takes a body from the rest of the request line as well as from the lines below /// it, so the gate has to read the same two places. static func weaviateConsoleBody(_ trimmed: String) -> String { - let lines = trimmed.split(separator: "\n", maxSplits: 1, omittingEmptySubsequences: false) - let header = lines.first.map(String.init) ?? "" - let following = lines.count > 1 ? String(lines[1]) : "" + let newline = trimmed.firstIndex(where: \.isNewline) + let header = newline.map { trimmed[..<$0] } ?? trimmed[...] + let following = newline.map { trimmed[trimmed.index(after: $0)...] } ?? "" let parts = header.split(maxSplits: 2, omittingEmptySubsequences: true, whereSeparator: \.isWhitespace) - let inline = parts.count > 2 ? String(parts[2]) : "" - return inline.isEmpty ? following : inline + let inline = parts.count > 2 ? parts[2] : "" + return [inline, following] + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + .joined(separator: "\n") } static func weaviateClassification(_ trimmed: String) -> QueryClassification { @@ -1209,42 +1213,19 @@ private extension QueryClassifier { return QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: false) } - /// POSTs under `/collections//` that only read. Retrieving points by id is a POST to `points`. - static let qdrantReadRoutes: Set<[String]> = [ - ["POINTS"], ["POINTS", "SCROLL"], ["POINTS", "SEARCH"], ["POINTS", "QUERY"], ["POINTS", "COUNT"], - ["POINTS", "RECOMMEND"], ["POINTS", "DISCOVER"], ["POINTS", "FACET"], - ["POINTS", "SEARCH", "BATCH"], ["POINTS", "QUERY", "BATCH"], ["POINTS", "RECOMMEND", "BATCH"], - ["POINTS", "DISCOVER", "BATCH"], ["POINTS", "SEARCH", "GROUPS"], ["POINTS", "QUERY", "GROUPS"], - ["POINTS", "RECOMMEND", "GROUPS"], ["POINTS", "SEARCH", "MATRIX", "PAIRS"], - ["POINTS", "SEARCH", "MATRIX", "OFFSETS"], - ] - - /// A snapshot is written to the server's disk, and recovering one fetches a URL from the server. - /// A batch can carry deletes, and a POST ending in `delete` or `clear` removes points or payload. + // The driver runs what `QdrantConsoleParser` parses, so the gate reads the same request. static func qdrantClassification(_ trimmed: String) -> QueryClassification { - if trimmed.hasPrefix("QDRANT_BROWSE:") || trimmed.hasPrefix("QDRANT_EXPORT:") { + if QdrantOperations.isTagged(trimmed) { return .safe } - let (verb, path) = typesenseRequestLine(trimmed) - guard !verb.isEmpty else { return .safe } - guard let route = typesenseRoute(of: path) else { - return QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: true) - } - let reachesFilesystem = route.contains("SNAPSHOTS") - if verb == "GET" || verb == "HEAD" { - return QueryClassification(tier: .safe, reachesFilesystemOrExecutesCode: reachesFilesystem) + guard let request = QdrantConsoleParser.parse(trimmed) else { return .safe } + let kind = request.kind + let tier: QueryTier = switch kind.tier { + case .read: .safe + case .write: .write + case .remove: .destructive } - let isCollectionRoute = route.count >= 3 && route[0] == "COLLECTIONS" - let subroute = isCollectionRoute ? Array(route.dropFirst(2)) : [] - if verb == "POST", isCollectionRoute, qdrantReadRoutes.contains(subroute) { - return QueryClassification(tier: .safe, reachesFilesystemOrExecutesCode: reachesFilesystem) - } - let removes = verb == "DELETE" - || route.last == "DELETE" || route.last == "CLEAR" || subroute == ["POINTS", "BATCH"] - return QueryClassification( - tier: removes ? .destructive : .write, - reachesFilesystemOrExecutesCode: reachesFilesystem - ) + return QueryClassification(tier: tier, reachesFilesystemOrExecutesCode: kind.touchesServerFiles) } /// A grid page is a tagged spec and anything else a request line read the way the driver reads diff --git a/apple/TableProTests/Core/DataWrite/DataWriteExecutorLookupTests.swift b/apple/TableProTests/Core/DataWrite/DataWriteExecutorLookupTests.swift index e5bce29c79..cc8ad67a43 100644 --- a/apple/TableProTests/Core/DataWrite/DataWriteExecutorLookupTests.swift +++ b/apple/TableProTests/Core/DataWrite/DataWriteExecutorLookupTests.swift @@ -55,16 +55,22 @@ private final class LookupDriver: PluginDriverStub, @unchecked Sendable { } struct DataWriteExecutorLookupTests { - private func step(_ sql: String, rowCount: Int = 1, check: Bool = true) -> DataWriteStep { + private func step( + _ sql: String, + rowCount: Int = 1, + check: Bool = true, + rowKind: RowWriteKind = .delete, + pinsRows: Bool = true + ) -> DataWriteStep { DataWriteStep( kind: .rowWrite, statement: ParameterizedStatement(sql: sql, parameters: ["a"]), tableName: "events", - rowKind: .delete, + rowKind: rowKind, lookup: RowLookup( query: ParameterizedStatement(sql: "LOOKUP \(sql)", parameters: ["a"]), rowCount: rowCount, - pinsRows: true, + pinsRows: pinsRows, check: check ? ParameterizedStatement(sql: "CHECK \(sql)", parameters: ["a"]) : nil ) ) @@ -115,6 +121,20 @@ struct DataWriteExecutorLookupTests { #expect(driver.writes.isEmpty) } + @Test("An insert whose key a row already holds refuses the save, saying the key is taken, before anything is written") + func takenKeyRefusesInsertBeforeAnyWrite() async { + let driver = LookupDriver(found: [1]) + + await #expect(throws: DataWriteError.changeRefused( + table: "events", + kind: .insert, + reason: String(localized: "A row with this key already exists, so the insert would not add one.") + )) { + try await run([step("INSERT keyed", rowCount: 0, check: false, rowKind: .insert, pinsRows: false)], on: driver) + } + #expect(driver.writes.isEmpty) + } + @Test("Each write reads its lookup again right before it runs, after the writes ahead of it") func lookupRunsAgainBeforeEachWrite() async { let driver = LookupDriver(found: [1, 1, 1, 2]) diff --git a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift index e54fec8fc0..44ab29ab76 100644 --- a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift @@ -895,6 +895,10 @@ struct QueryClassifierNonSqlTests { #expect(QueryClassifier.isWriteQuery("mutation { delete { Article } }", databaseType: .weaviate)) #expect(QueryClassifier.isWriteQuery("POST /v1/objects {}", databaseType: .weaviate)) #expect(QueryClassifier.classifyTier("DELETE /v1/objects/abc", databaseType: .weaviate) == .destructive) + #expect(QueryClassifier.classifyTier("DELETE /v1/schema/Article", databaseType: .weaviate) == .destructive) + #expect(QueryClassifier.classifyTier("DELETE\t/v1/schema/Article", databaseType: .weaviate) == .destructive) + let mutation = "POST /v1/graphql {\n\"query\": \"mutation { delete { Article } }\" }" + #expect(QueryClassifier.isWriteQuery(mutation, databaseType: .weaviate)) let search = "WEAVIATE_SEARCH:e30=" #expect(!QueryClassifier.isWriteQuery(search, databaseType: .weaviate)) } @@ -913,6 +917,9 @@ struct QueryClassifierNonSqlTests { #expect(QueryClassifier.classifyTier("POST /collections/docs/points/batch {}", databaseType: .qdrant) == .destructive) #expect(QueryClassifier.classifyTier("DELETE /collections/docs", databaseType: .qdrant) == .destructive) #expect(QueryClassifier.isWriteQuery("POST /collections/docs/points/query/../delete {}", databaseType: .qdrant)) + #expect(!QueryClassifier.isWriteQuery("POST /collections/docs/facet {\"key\": \"city\"}", databaseType: .qdrant)) + #expect(QueryClassifier.classifyTier("DELETE\t/collections/docs", databaseType: .qdrant) == .destructive) + #expect(!QueryClassifier.isWriteQuery("POST /collections/docs/points/query {\n\"limit\": 3}", databaseType: .qdrant)) } @Test("A Qdrant snapshot reaches the server's disk") diff --git a/apple/changelog.d/3972.added.md b/apple/changelog.d/3972.added.md index 7ece2164bb..0ad2c302b7 100644 --- a/apple/changelog.d/3972.added.md +++ b/apple/changelog.d/3972.added.md @@ -1 +1 @@ -Qdrant support: browse collections, filter points, edit payloads, and run REST requests such as vector queries. +Qdrant support: browse collections, filter and sort points, edit payloads and vectors, and run REST requests. diff --git a/apple/project.yml b/apple/project.yml index 327bd99b94..3041a2272d 100644 --- a/apple/project.yml +++ b/apple/project.yml @@ -408,7 +408,7 @@ targets: - package: Sparkle - package: Yams - package: TableProCore - products: [TableProAnalytics, TableProAutocomplete, TableProConnectionLibrary, TableProGeometry, TableProGoogleCloud, TableProImport, TableProLogRedaction, TableProNumberFormatting, TableProRedisCore, TableProSQLGrammar, TableProSQLVocabulary, TableProSSHTransport, TableProSyncTransport, TableProTabular, TableProTabularIO, TableProTheme] + products: [TableProAnalytics, TableProAutocomplete, TableProConnectionLibrary, TableProGeometry, TableProGoogleCloud, TableProImport, TableProLogRedaction, TableProNumberFormatting, TableProQdrantCore, TableProRedisCore, TableProSQLGrammar, TableProSQLVocabulary, TableProSSHTransport, TableProSyncTransport, TableProTabular, TableProTabularIO, TableProTheme] - package: TableProMacCore products: [TableProMCP, TableProSSHConfig, TableProUtilities] # Writes nothing for a version CHANGELOG.md has no section for, so a 0.0.0-dev build ships no notes. @@ -1306,6 +1306,8 @@ targets: dependencies: - package: TableProCore product: TableProQdrantCore + - package: TableProCore + product: TableProTLSClientIdentity WeaviateDriverPlugin: templates: [DriverPlugin] diff --git a/apple/scripts/probes/check-qdrant.sh b/apple/scripts/probes/check-qdrant.sh index 9b459f86c7..1a4b551f33 100755 --- a/apple/scripts/probes/check-qdrant.sh +++ b/apple/scripts/probes/check-qdrant.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # -# Run TableProQdrantCore against a live Qdrant node: cursor paging, every filter kind, order_by, -# set-payload, insert_only inserts with no vector, delete, truncate, export and drop. The probe -# creates and drops a collection named tablepro_probe. +# Run TableProQdrantCore against a live Qdrant node: cursor paging, every filter kind, sorting with +# keyless points, typed payload and vector writes, the insert id check, facet, export, truncate and +# drop. The probe creates and drops a collection named tablepro_probe. # # Usage: # scripts/probes/check-qdrant.sh [host] [port] diff --git a/apple/scripts/probes/qdrant-probe.swift b/apple/scripts/probes/qdrant-probe.swift index ce567d0370..3a4d70a334 100644 --- a/apple/scripts/probes/qdrant-probe.swift +++ b/apple/scripts/probes/qdrant-probe.swift @@ -1,7 +1,7 @@ import Foundation import TableProQdrantCore -// Exercises TableProQdrantCore against a live node: the facts the driver relies on, measured. +// Measures, against a live node, the Qdrant behavior TableProQdrantCore hard-codes. // Run through check-qdrant.sh, which builds it against this checkout's TableProCore. var failures = 0 @@ -10,104 +10,163 @@ var failures = 0 if !ok { failures += 1 } } +final class ProbeTransport: QdrantTransport, @unchecked Sendable { + func send(_ request: QdrantHTTPRequest) async throws -> QdrantHTTPResponse { + var urlRequest = URLRequest(url: request.url) + urlRequest.httpMethod = request.method + urlRequest.httpBody = request.body + request.headers.forEach { urlRequest.setValue($1, forHTTPHeaderField: $0) } + let (data, response) = try await URLSession.shared.data(for: urlRequest) + return QdrantHTTPResponse(statusCode: (response as? HTTPURLResponse)?.statusCode ?? 0, body: data) + } + + func cancelAll() {} +} + let env = ProcessInfo.processInfo.environment let settings = QdrantConnectionSettings( host: env["QDRANT_HOST"] ?? "127.0.0.1", - port: Int(env["QDRANT_PORT"] ?? "") ?? 6333, + port: Int(env["QDRANT_PORT"] ?? "") ?? 6_333, usesTLS: false, - apiKey: env["QDRANT_API_KEY"] ?? "", - skipTLSVerify: false + apiKey: env["QDRANT_API_KEY"] ?? "" ) -let client = QdrantClient(settings: settings, transport: URLSessionQdrantTransport(resourceTimeout: 30), timeout: { 10 }) +let client = QdrantClient(settings: settings, transport: ProbeTransport(), timeout: { 10 }) try await client.connect(requestTimeout: { 10 }) print("version", client.serverVersion ?? "nil") -_ = try? await client.execute(QdrantConsoleRequest(method: "DELETE", path: "/collections/tablepro_probe", body: nil)) -_ = try await client.execute(QdrantConsoleRequest(method: "PUT", path: "/collections/tablepro_probe", body: #"{"vectors":{"size":2,"distance":"Dot"}}"#)) +let name = "tablepro_probe" +let base = "/collections/\(name)" +func request(_ text: String?) -> QdrantConsoleRequest { + guard let text, let request = QdrantConsoleParser.parse(text) else { fatalError("not a request: \(text ?? "nil")") } + return request +} +func run(_ text: String?) async throws -> QdrantHTTPResponse { + try await client.execute(request(text)) +} + +_ = try? await run("DELETE \(base)") +_ = try await run(#"PUT \#(base) {"vectors":{"size":2,"distance":"Dot"},"sparse_vectors":{"sp":{}}}"#) var points: [[String: Any]] = [] -for i in 1...250 { points.append(["id": i, "vector": [Double(i), 1], "payload": ["n": i, "city": i % 2 == 0 ? "even" : "odd", "body": "hello world \(i)"]]) } -_ = try await client.execute(QdrantConsoleRequest(method: "PUT", path: "/collections/tablepro_probe/points?wait=true", body: try QdrantJSON.text(["points": points]))) -_ = try await client.execute(QdrantConsoleRequest(method: "PUT", path: "/collections/tablepro_probe/index?wait=true", body: #"{"field_name":"n","field_schema":"integer"}"#)) -_ = try await client.execute(QdrantConsoleRequest(method: "PUT", path: "/collections/tablepro_probe/index?wait=true", body: #"{"field_name":"body","field_schema":"text"}"#)) - -check(try await client.collectionNames().contains("tablepro_probe"), "lists collection") -let collection = try await client.collection(named: "tablepro_probe") -print("columns", QdrantSchema.columns(for: collection).map { "\($0.name):\($0.type)" }) +for i in 1...250 { + var payload: [String: Any] = ["city": i % 2 == 0 ? "even" : "odd", "body": "Hello World \(i)", "f": Double(i) + 0.5] + if i <= 200 { payload["n"] = i } + if i == 7 { payload["note"] = NSNull() } + if i == 8 { payload["note"] = "" } + payload["meta"] = ["source": "doc\(i).pdf"] + payload["a.b"] = "dotted" + points.append(["id": i, "vector": ["": [Double(i), 1]], "payload": payload]) +} +_ = try await run("PUT \(base)/points?wait=true\n" + (try QdrantJSON.text(["points": points]))) +for (field, schema) in [("n", "integer"), ("f", "float"), ("city", "keyword"), ("meta.source", "keyword")] { + _ = try await run(#"PUT \#(base)/index?wait=true {"field_name":"\#(field)","field_schema":"\#(schema)"}"#) +} + +check(try await client.collectionNames().contains(name), "lists collection") +let collection = try await client.collection(named: name) +let columns = QdrantSchema.columns(for: collection) +print("columns", columns.map { "\($0.name):\($0.type)" }) check(collection.pointsCount == 250, "points_count") +check(!columns.contains { $0.name == "meta.source" } && collection.indexes.contains { $0.path == "meta.source" }, "nested index path is an index, not a column") +check(columns.contains { $0.name == "vector" && $0.type == "vector(2)" } && columns.contains { $0.name == "vector.sp" }, "vector columns typed") -let cache = QdrantCursorCache() -let p0 = try await client.scrollPage(collection: "tablepro_probe", filter: nil, offset: 0, limit: 100, cache: cache) -let p1 = try await client.scrollPage(collection: "tablepro_probe", filter: nil, offset: 100, limit: 100, cache: cache) -let p2 = try await client.scrollPage(collection: "tablepro_probe", filter: nil, offset: 200, limit: 100, cache: cache) +let cache = QdrantPagingCache() +let p0 = try await client.scrollPage(collection: name, filter: nil, offset: 0, limit: 100, cache: cache) +let p1 = try await client.scrollPage(collection: name, filter: nil, offset: 100, limit: 100, cache: cache) +let p2 = try await client.scrollPage(collection: name, filter: nil, offset: 200, limit: 100, cache: cache) check(p0.first?.idText == "1" && p1.first?.idText == "101" && p2.count == 50 && p2.last?.idText == "250", "paging forward") -let jump = try await client.scrollPage(collection: "tablepro_probe", filter: nil, offset: 170, limit: 10, cache: QdrantCursorCache()) +let jump = try await client.scrollPage(collection: name, filter: nil, offset: 170, limit: 10, cache: QdrantPagingCache()) check(jump.map { $0.idText ?? "" } == (171...180).map(String.init), "jump walks to offset") -check(try await client.scrollPage(collection: "tablepro_probe", filter: nil, offset: 400, limit: 10, cache: cache).isEmpty, "past end") +check(try await client.scrollPage(collection: name, filter: nil, offset: 400, limit: 10, cache: cache).isEmpty, "past end") func filtered(_ specs: [QdrantFilterSpec], _ mode: String = "AND") async throws -> Int { - let f = try QdrantFilterBuilder.filter(specs, logicMode: mode, collection: collection) - return try await client.count(collection: "tablepro_probe", filter: f) + let filter = try QdrantFilterBuilder.filter(specs, logicMode: mode, collection: collection) + return try await client.count(collection: name, filter: filter) } check(try await filtered([QdrantFilterSpec(column: "city", op: "=", value: "even")]) == 125, "match keyword") check(try await filtered([QdrantFilterSpec(column: "n", op: "BETWEEN", value: "10,19", secondValue: "19")]) == 10, "between") -check(try await filtered([QdrantFilterSpec(column: "n", op: ">", value: "240"), QdrantFilterSpec(column: "city", op: "!=", value: "even")]) == 5, "and with not") -check(try await filtered([QdrantFilterSpec(column: "n", op: "<", value: "3"), QdrantFilterSpec(column: "n", op: ">", value: "248")], "OR") == 4, "or") -check(try await filtered([QdrantFilterSpec(column: "body", op: "CONTAINS", value: "hello")]) == 250, "full text") +check(try await filtered([QdrantFilterSpec(column: "n", op: ">", value: "190"), QdrantFilterSpec(column: "city", op: "!=", value: "even")]) == 5, "and with not") +check(try await filtered([QdrantFilterSpec(column: "n", op: "<", value: "3"), QdrantFilterSpec(column: "n", op: ">", value: "198")], "OR") == 4, "or") +check(try await filtered([QdrantFilterSpec(column: "body", op: "CONTAINS", value: "world")]) == 250, "contains without a text index matches words") check(try await filtered([QdrantFilterSpec(column: "id", op: "IN", value: "1,2,3")]) == 3, "has_id") check(try await filtered([QdrantFilterSpec(column: "n", op: "IN", value: "1,2,300")]) == 2, "match any") -check(try await filtered([QdrantFilterSpec(column: "missing", op: "IS NULL", value: "")]) == 0, "is_null") -check(try await filtered([QdrantFilterSpec(column: "missing", op: "IS EMPTY", value: "")]) == 250, "is_empty") - -let sortReq = QdrantBrowseRequest(collection: "tablepro_probe", offset: 5, limit: 3, sort: QdrantSortSpec(column: "n", ascending: false), filters: [], logicMode: "AND") -let sorted = try await client.query(collection: "tablepro_probe", body: QdrantBrowseQuery.queryBody(for: sortReq, filter: nil, sort: sortReq.sort!)) -check(sorted.map { $0.idText ?? "" } == ["245", "244", "243"], "order_by with offset") - -let batch = QdrantStatementGenerator.generate(collection: collection, changes: [ - QdrantTrackedChange(kind: .update, id: "1", cellChanges: [QdrantCellChange(column: "city", newText: "edited"), QdrantCellChange(column: "n", newText: "1000")]), - QdrantTrackedChange(kind: .delete, id: "2"), - QdrantTrackedChange(kind: .insert, id: "999", values: ["city": "new"]), - QdrantTrackedChange(kind: .insert, id: "3", values: ["city": "clobber"]) -]) -for request in batch.requests { - let text = QdrantWriteCodec.encode(request) - _ = try await client.execute(QdrantConsoleParser.parse(text)!) +check(try await filtered([QdrantFilterSpec(column: "f", op: "IN", value: "1.5, 2.5")]) == 2, "IN on floats") +check(try await filtered([QdrantFilterSpec(column: "f", op: "NOT IN", value: "1.5")]) == 249, "NOT IN on floats") +check(try await filtered([QdrantFilterSpec(column: "n", op: "IS NULL", value: "")]) == 50, "IS NULL counts missing keys") +check(try await filtered([QdrantFilterSpec(column: "note", op: "IS NULL", value: "")]) == 249, "IS NULL counts null and missing") +check(try await filtered([QdrantFilterSpec(column: "note", op: "IS EMPTY", value: "")]) == 250, "IS EMPTY adds the empty string") +check(try await filtered([QdrantFilterSpec(column: "a.b", op: "=", value: "dotted")]) == 250, "dotted top-level key") + +func sorted(_ offset: Int, _ limit: Int, ascending: Bool = false) async throws -> [String] { + try await client.sortedPage( + collection: name, filter: nil, column: "n", key: "\"n\"", ascending: ascending, + offset: offset, limit: limit, cache: cache + ).map { $0.idText ?? "" } } -let after = try await client.query(collection: "tablepro_probe", body: ["query": NSNull(), "filter": ["must": [["has_id": [1, 2, 3, 999]]]], "with_payload": true, "limit": 10] as [String: Any]) -let byID = Dictionary(uniqueKeysWithValues: after.map { ($0.idText ?? "", $0) }) +check(try await sorted(5, 3) == ["195", "194", "193"], "order_by with offset") +check(try await sorted(198, 4) == ["2", "1", "201", "202"], "points without the key follow the ordered ones") +check(try await sorted(248, 10) == ["249", "250"], "every point reachable when sorted") +_ = try await run(#"POST \#(base)/points/payload?wait=true {"payload":{"n":[1,2]},"points":[3]}"#) +do { + _ = try await client.sortedPage(collection: name, filter: nil, column: "n", key: "\"n\"", ascending: true, offset: 0, limit: 5, cache: QdrantPagingCache()) + check(false, "list values refuse the sort") +} catch { + check(error.localizedDescription.contains("list"), "list values refuse the sort") +} +_ = try await run(#"POST \#(base)/points/payload?wait=true {"payload":{"n":3},"points":[3]}"#) + +let types = Dictionary(uniqueKeysWithValues: columns.map { ($0.name, $0.type) }) +let writes = try QdrantRowWriter.writes(collection: name, changes: [ + QdrantRowChange(kind: .update, rowIndex: 0, id: "1", cells: [ + QdrantCell(column: "city", text: "edited"), QdrantCell(column: "n", text: "1000"), + QdrantCell(column: "vector", text: "[9, 9]"), QdrantCell(column: "vector.sp", text: #"{"indices":[2],"values":[0.5]}"#) + ]), + QdrantRowChange(kind: .delete, rowIndex: 1, id: "2"), + QdrantRowChange(kind: .insert, rowIndex: 2, id: nil, cells: [ + QdrantCell(column: "id", text: "999"), QdrantCell(column: "city", text: "new"), QdrantCell(column: "vector", text: "[1, 2]") + ]) +], typeName: { types[$0] }) +for write in writes { + if let lookup = write.mustFindNothing { + check(QdrantPoint.list(fromResponse: try await run(lookup.text).json)?.isEmpty == true, "insert lookup finds a free id") + } + _ = try await run(write.request.text) +} +let taken = try QdrantRowWriter.writes(collection: name, changes: [ + QdrantRowChange(kind: .insert, rowIndex: 0, id: nil, cells: [QdrantCell(column: "id", text: "3")]) +], typeName: { types[$0] }) +check(QdrantPoint.list(fromResponse: try await run(taken[0].mustFindNothing?.text).json)?.count == 1, "insert lookup finds a taken id") + +let after = try await run(#"POST \#(base)/points {"ids":[1,2,999],"with_payload":true,"with_vector":true}"#) +let byID = Dictionary(uniqueKeysWithValues: (QdrantPoint.list(fromResponse: after.json) ?? []).map { ($0.idText ?? "", $0) }) check(byID["1"]?.cell(for: "city") == "edited" && byID["1"]?.cell(for: "n") == "1000", "set payload typed") +check(byID["1"]?.cell(for: "vector") == "[9,9]" && byID["1"]?.cell(for: "vector.sp") == #"{"indices":[2],"values":[0.5]}"#, "vectors updated") check(byID["2"] == nil, "delete") -check(byID["999"]?.cell(for: "city") == "new", "insert") -check(byID["3"]?.cell(for: "city") == "odd", "insert_only keeps existing") +check(byID["999"]?.cell(for: "city") == "new" && byID["999"]?.cell(for: "vector") == "[1,2]", "insert with vector") +let cleared = try QdrantRowWriter.writes(collection: name, changes: [ + QdrantRowChange(kind: .update, rowIndex: 0, id: "1", cells: [QdrantCell(column: "vector.sp", text: nil)]) +], typeName: { types[$0] }) +_ = try await run(cleared[0].request.text) +let afterClear = QdrantPoint.list(fromResponse: try await run(#"POST \#(base)/points {"ids":[1],"with_vector":true}"#).json) +check(afterClear?.first?.cell(for: "vector.sp") == nil, "NULL removes a vector") -let console = try await client.execute(QdrantConsoleParser.parse("POST /collections/tablepro_probe/points/query\n{\"query\": [1, 0], \"limit\": 2, \"with_vector\": true, \"with_payload\": true}")!) +let facet = request(#"POST \#(base)/facet {"key":"city"}"#) +check(facet.kind.tier == .read, "facet is a read") +check(((try await client.execute(facet).json as? [String: Any])?["result"]) != nil, "facet route answers") + +let console = try await run("POST \(base)/points/query {\"query\": [1, 0],\n\"limit\": 2, \"with_vector\": true, \"with_payload\": true}") let hits = QdrantPoint.list(fromResponse: console.json) ?? [] -print("query columns", QdrantPoint.columns(of: hits, collection: collection), hits.first?.row(columns: QdrantPoint.columns(of: hits, collection: collection)) ?? []) -check(hits.count == 2 && hits[0].score != nil, "vector query has scores") +check(hits.count == 2 && hits[0].score != nil, "vector query with a body over two lines has scores") -_ = try await client.execute(QdrantConsoleRequest(method: "PUT", path: "/collections/tablepro_probe/points?wait=true", body: #"{"points":[{"id":5000,"vector":[1,1],"payload":{"late":true,"id":"ext-1"}}]}"#)) var widened = collection -for try await page in client.pointPages(collection: "tablepro_probe", pageSize: 60, withVector: false) { - check(page.allSatisfy { $0.vector == nil }, "payload-only pass has no vectors") +for try await page in client.pointPages(collection: name, pageSize: 60, withVector: false) { widened = widened.including(page) } -let wcols = QdrantSchema.columns(for: widened).map(\.name) -print("export columns", wcols) -check(wcols.contains("late") && wcols.contains("payload.id"), "late and reserved keys found") -let lateFilter = try QdrantFilterBuilder.filter([QdrantFilterSpec(column: "payload.id", op: "=", value: "ext-1")], logicMode: "AND", collection: widened) -check(try await client.count(collection: "tablepro_probe", filter: lateFilter) == 1, "filter on payload.id") var exported = 0 -for try await page in client.pointPages(collection: "tablepro_probe", pageSize: 60) { exported += page.count } -check(exported == 251, "export all") - -let truncate = QdrantOperations.deleteAllPoints(collection: "tablepro_probe")! -_ = try await client.execute(QdrantConsoleParser.parse(truncate)!) -check(try await client.count(collection: "tablepro_probe", filter: nil) == 0, "truncate") -_ = try await client.execute(QdrantConsoleParser.parse(QdrantOperations.deleteCollection(named: "tablepro_probe", objectType: "TABLE")!)!) -check(!(try await client.collectionNames().contains("tablepro_probe")), "drop") -do { - _ = try await client.collection(named: "tablepro_probe_missing") - check(false, "missing collection reports an error") -} catch { - check(error.localizedDescription.contains("doesn't exist"), "missing collection reports an error") -} +for try await page in client.pointPages(collection: name, pageSize: 60) { exported += page.count } +check(exported == 250, "export all") + +_ = try await run(QdrantOperations.deleteAllPoints(collection: name)) +check(try await client.count(collection: name, filter: nil) == 0, "truncate") +_ = try await run(QdrantOperations.deleteCollection(named: name, objectType: "TABLE")) +check(!(try await client.collectionNames().contains(name)), "drop") exit(failures == 0 ? 0 : 1) From 339937c69fd8544f492eeb97a9adb81a20a1457a Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 20:55:11 +0700 Subject: [PATCH 05/20] fix(plugin-qdrant): keep indexed lists as lists and refuse sorts Qdrant cannot order --- .../TableProQdrantCore/QdrantSchema.swift | 56 +++++++++++++++---- .../Wire/QdrantSortedPager.swift | 31 ++++++---- .../QdrantClientTests.swift | 14 +++++ .../QdrantSchemaTests.swift | 42 +++++++++++++- ...luginMetadataRegistry+QdrantDefaults.swift | 1 + apple/scripts/probes/qdrant-probe.swift | 6 ++ 6 files changed, 127 insertions(+), 23 deletions(-) diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift index dc5eaed17e..a32c7ecb5d 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift @@ -113,16 +113,34 @@ public struct QdrantVectorSpec: Sendable, Equatable { public struct QdrantPayloadField: Sendable, Equatable { public let name: String + // What a filter compares: an index types each element of a list, so this is the element's type. public let type: QdrantFieldType public let isIndexed: Bool // False while every sampled value was null, so the first real value sets the type outright. public let hasTypedValue: Bool + public let holdsLists: Bool + public let holdsScalars: Bool - public init(name: String, type: QdrantFieldType, isIndexed: Bool, hasTypedValue: Bool = true) { + public init( + name: String, + type: QdrantFieldType, + isIndexed: Bool, + hasTypedValue: Bool = true, + holdsLists: Bool = false, + holdsScalars: Bool = false + ) { self.name = name self.type = type self.isIndexed = isIndexed self.hasTypedValue = hasTypedValue + self.holdsLists = holdsLists + self.holdsScalars = holdsScalars + } + + // What a cell holds and is written back as. An indexed field holding lists keeps them lists. + public var cellType: QdrantFieldType { + guard isIndexed, holdsLists else { return type } + return holdsScalars ? .json : .array } } @@ -161,8 +179,16 @@ public struct QdrantCollection: Sendable, Equatable { } public func fieldType(ofColumn column: String) -> QdrantFieldType? { + field(ofColumn: column)?.type + } + + public func cellType(ofColumn column: String) -> QdrantFieldType? { + field(ofColumn: column)?.cellType + } + + private func field(ofColumn column: String) -> QdrantPayloadField? { let key = QdrantSchema.payloadKey(forColumn: column) - return payloadFields.first { $0.name == key }?.type + return payloadFields.first { $0.name == key } } public func including(_ points: [QdrantPoint]) -> QdrantCollection { @@ -179,7 +205,15 @@ public struct QdrantCollection: Sendable, Equatable { continue } let field = fields[index] - guard !field.isIndexed, let observed else { continue } + guard let observed else { continue } + if field.isIndexed { + fields[index] = QdrantPayloadField( + name: key, type: field.type, isIndexed: true, + holdsLists: field.holdsLists || observed == .array, + holdsScalars: field.holdsScalars || observed != .array + ) + continue + } let type = field.hasTypedValue ? field.type.merged(with: observed) : observed fields[index] = QdrantPayloadField(name: key, type: type, isIndexed: false) } @@ -197,7 +231,8 @@ public struct QdrantCollection: Sendable, Equatable { var vectors: [QdrantVectorSpec] = [] if let config = params["vectors"] as? [String: Any] { - if config["size"] != nil { + // A named vector may itself be called `size`; only the unnamed one has a number there. + if config["size"] is NSNumber { vectors.append(.dense(name: "", config: config)) } else { vectors += config.keys.sorted().map { key in @@ -260,9 +295,10 @@ public enum QdrantSchema { } // A payload key named `id`, `score` or `vector...` would take a point column's name, so it shows as - // `payload.` and maps back wherever a column is read or written. + // `payload.`. A key already starting with `payload.` is prefixed too, so every column maps back + // to exactly one key. public static func isReserved(_ key: String) -> Bool { - key == idColumn || key == scoreColumn || isVectorColumn(key) + key == idColumn || key == scoreColumn || isVectorColumn(key) || key.hasPrefix(payloadPrefix) } public static func column(forPayloadKey key: String) -> String { @@ -270,15 +306,13 @@ public enum QdrantSchema { } public static func payloadKey(forColumn column: String) -> String { - guard column.hasPrefix(payloadPrefix) else { return column } - let key = String(column.dropFirst(payloadPrefix.count)) - return isReserved(key) ? key : column + column.hasPrefix(payloadPrefix) ? String(column.dropFirst(payloadPrefix.count)) : column } public static func columns(for collection: QdrantCollection) -> [QdrantColumn] { var columns = [QdrantColumn(name: idColumn, type: "id", isPrimaryKey: true)] columns += collection.payloadFields.map { - QdrantColumn(name: column(forPayloadKey: $0.name), type: $0.type.rawValue, isPrimaryKey: false) + QdrantColumn(name: column(forPayloadKey: $0.name), type: $0.cellType.rawValue, isPrimaryKey: false) } columns += collection.vectors.map { QdrantColumn(name: $0.column, type: $0.typeName, isPrimaryKey: false) } return columns @@ -295,7 +329,7 @@ public enum QdrantSchema { if let spec = collection.vectors.first(where: { $0.column == column }) { return spec.typeName } - return collection.fieldType(ofColumn: column)?.rawValue + return collection.cellType(ofColumn: column)?.rawValue } } diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantSortedPager.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantSortedPager.swift index 63a27a92e0..2271467404 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantSortedPager.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantSortedPager.swift @@ -16,9 +16,25 @@ public extension QdrantClient { ) async throws -> [QdrantPoint] { let hasValue: [String: Any] = ["key": key, "range": [String: Any]()] let ordered = QdrantFilterBuilder.allOf(filter, hasValue) + func orderBody(offset: Int, limit: Int, withData: Bool) -> [String: Any] { + var body: [String: Any] = [ + "query": ["order_by": ["key": key, "direction": ascending ? "asc" : "desc"]], + "limit": limit, + "offset": offset, + "with_payload": withData, + "with_vector": withData + ] + if let filter { + body["filter"] = filter + } + return body + } let checkKey = "\(collection)\u{0}\(key)\u{0}\((try? QdrantJSON.text(ordered)) ?? "")" if !cache.isSortChecked(checkKey) { + // A field with no range index matches no point, so without asking Qdrant to order by it + // every point would land in the remainder and the page would look sorted by id. + _ = try await query(collection: collection, body: orderBody(offset: 0, limit: 1, withData: false)) let listed = try await count( collection: collection, filter: QdrantFilterBuilder.allOf(ordered, ["key": key, "values_count": ["gt": 1]]) @@ -35,17 +51,10 @@ public extension QdrantClient { let orderedCount = try await count(collection: collection, filter: ordered) var points: [QdrantPoint] = [] if offset < orderedCount { - var body: [String: Any] = [ - "query": ["order_by": ["key": key, "direction": ascending ? "asc" : "desc"]], - "limit": min(limit, orderedCount - offset), - "offset": offset, - "with_payload": true, - "with_vector": true - ] - if let filter { - body["filter"] = filter - } - points = try await query(collection: collection, body: body) + points = try await query( + collection: collection, + body: orderBody(offset: offset, limit: min(limit, orderedCount - offset), withData: true) + ) } guard points.count < limit else { return points } let withoutValue = QdrantFilterBuilder.allOf(filter, QdrantFilterBuilder.negated([hasValue])) diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift index 7e7a426526..4eb10b12e2 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift @@ -129,6 +129,20 @@ struct QdrantSortedPagerTests { #expect(transport.recorded.filter { canonical(bodyObject($0)).contains("values_count") }.count == 1) } + @Test("A field Qdrant cannot order by is refused, never shown in id order as if sorted") + func unorderableFieldRefused() async throws { + let transport = RecordingTransport { request in + guard request.url.path.hasSuffix("/query") else { return jsonResponse(["result": ["count": 0]]) } + return jsonResponse(["status": ["error": "Wrong input: No range index for `order_by` key: `city`."]], status: 400) + } + await #expect(throws: QdrantError.api(status: 400, message: "Wrong input: No range index for `order_by` key: `city`.")) { + _ = try await makeClient(transport: transport).sortedPage( + collection: "docs", filter: nil, column: "city", key: "\"city\"", ascending: true, + offset: 0, limit: 4, cache: QdrantPagingCache() + ) + } + } + @Test("A field holding a list in some point is refused, since Qdrant would show that point once per value") func listedValuesRefused() async throws { let client = makeClient(transport: transport(listed: 2)) diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift index a1eba2ef8b..f05cb22745 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift @@ -56,7 +56,47 @@ struct QdrantSchemaTests { #expect(columns == ["id", "city", "payload.id", "payload.vector", "vector"]) #expect(row.row(columns: columns) == ["7", "Berlin", "external-1", "text", "[1,2]"]) #expect(QdrantSchema.payloadKey(forColumn: "payload.id") == "id") - #expect(QdrantSchema.payloadKey(forColumn: "payload.city") == "payload.city") + } + + @Test("Every payload key maps to one column and back, including one that starts with payload.") + func payloadKeysRoundTrip() { + let keys = ["city", "id", "score", "vector", "vector.image", "payload.id", "payload.city", "payload."] + let columns = keys.map(QdrantSchema.column(forPayloadKey:)) + #expect(Set(columns).count == keys.count) + #expect(columns.map(QdrantSchema.payloadKey(forColumn:)) == keys) + #expect(QdrantSchema.column(forPayloadKey: "payload.id") == "payload.payload.id") + } + + @Test("An indexed field holding lists stays a list in its cell, while filters compare its elements") + func indexedLists() throws { + let collection = QdrantCollection.parse( + name: "docs", + info: info(vectors: ["size": 2], payloadSchema: [ + "tags": ["data_type": "keyword"], "mixed": ["data_type": "keyword"], "city": ["data_type": "keyword"] + ]), + sample: [ + try point(#"{"id":1,"payload":{"tags":["a","b"],"mixed":["a"],"city":"Berlin"}}"#), + try point(#"{"id":2,"payload":{"tags":["c"],"mixed":"b"}}"#) + ] + ) + let types = Dictionary(uniqueKeysWithValues: QdrantSchema.columns(for: collection).map { ($0.name, $0.type) }) + #expect(types["tags"] == "array") + #expect(types["mixed"] == "json") + #expect(types["city"] == "keyword") + #expect(collection.fieldType(ofColumn: "tags") == .keyword) + let condition = try QdrantFilterBuilder.condition( + for: QdrantFilterSpec(column: "tags", op: "=", value: "42"), collection: collection + ) + #expect(canonical(condition) == #"{"key":"\"tags\"","match":{"value":"42"}}"#) + } + + @Test("A named vector called size is not mistaken for the unnamed vector's dimension") + func vectorNamedSize() { + let collection = QdrantCollection.parse( + name: "docs", + info: info(vectors: ["size": ["size": 3, "distance": "Cosine"], "text": ["size": 2, "distance": "Dot"]]) + ) + #expect(collection.vectors.map(\.column) == ["vector.size", "vector.text"]) } @Test("Keys outside the sample join the collection, and a key first seen as null takes its later type") diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift index 8914415785..95c510298b 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift @@ -33,6 +33,7 @@ extension PluginMetadataRegistry { supportsModifyPrimaryKey: false, supportsOpportunisticTLS: false, verifiesServerWithSystemTrust: true, + columnsAreSampled: true, supportsPrincipalConnectionLimit: false ), schema: PluginMetadataSnapshot.SchemaInfo( diff --git a/apple/scripts/probes/qdrant-probe.swift b/apple/scripts/probes/qdrant-probe.swift index 3a4d70a334..c43850aedf 100644 --- a/apple/scripts/probes/qdrant-probe.swift +++ b/apple/scripts/probes/qdrant-probe.swift @@ -113,6 +113,12 @@ do { check(error.localizedDescription.contains("list"), "list values refuse the sort") } _ = try await run(#"POST \#(base)/points/payload?wait=true {"payload":{"n":3},"points":[3]}"#) +do { + _ = try await client.sortedPage(collection: name, filter: nil, column: "city", key: "\"city\"", ascending: true, offset: 0, limit: 5, cache: QdrantPagingCache()) + check(false, "a keyword field refuses the sort") +} catch { + check(error.localizedDescription.contains("range index"), "a keyword field refuses the sort") +} let types = Dictionary(uniqueKeysWithValues: columns.map { ($0.name, $0.type) }) let writes = try QdrantRowWriter.writes(collection: name, changes: [ From 79605dc0692324ac38be62b3ce6238119eda6c21 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 21:25:57 +0700 Subject: [PATCH 06/20] fix(plugin-qdrant): insert without overwriting, widen types per page, check the server purpose under Verify CA --- apple/Packages/TableProCore/Package.swift | 2 +- .../TableProQdrantCore/QdrantRowWrites.swift | 32 +++++++++++-- .../TableProQdrantCore/QdrantSchema.swift | 48 +++++++++++++++---- .../TableProQdrantCore/QdrantValue.swift | 7 +++ .../Wire/QdrantScrollPager.swift | 5 ++ .../ServerCertificatePurpose.swift | 29 +++++++++++ .../TableProTrinoCore/TrinoTransport.swift | 4 ++ .../QdrantClientTests.swift | 13 +++++ .../QdrantRowWriteTests.swift | 32 +++++++++++-- .../QdrantSchemaTests.swift | 19 +++++++- .../ServerCertificatePurposeTests.swift | 37 ++++++++++++++ .../QdrantPluginDriver+Execution.swift | 8 +++- .../QdrantPluginDriver+Metadata.swift | 2 +- .../QdrantPluginDriver.swift | 6 ++- .../QdrantURLSessionTransport.swift | 11 +++-- apple/scripts/probes/qdrant-probe.swift | 9 ++-- 16 files changed, 234 insertions(+), 30 deletions(-) create mode 100644 apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/ServerCertificatePurpose.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProTLSClientIdentityTests/ServerCertificatePurposeTests.swift diff --git a/apple/Packages/TableProCore/Package.swift b/apple/Packages/TableProCore/Package.swift index 0489760cef..1b7bd60140 100644 --- a/apple/Packages/TableProCore/Package.swift +++ b/apple/Packages/TableProCore/Package.swift @@ -156,7 +156,7 @@ let package = Package( ), .target( name: "TableProTrinoCore", - dependencies: ["TableProSQLGrammar"], + dependencies: ["TableProSQLGrammar", "TableProTLSClientIdentity"], path: "Sources/TableProTrinoCore" ), .target( diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift index 2c6b4e6f41..797c78c5d9 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift @@ -34,11 +34,22 @@ public struct QdrantRowChange: Sendable, Equatable { public struct QdrantRowWrite: Sendable, Equatable { public let request: QdrantConsoleRequest public let rowIndex: Int - // A read that must find no point before the write runs: `insert_only` answers success and writes - // nothing when the id is taken. + // A read that must find no point before the write runs: an insert that leaves a taken id alone + // still answers success, so without it the save would report a row it never added. public let mustFindNothing: QdrantConsoleRequest? } +public enum QdrantServerVersion { + // Measured: 1.15.5 and later honor `update_filter`; 1.13.6 and 1.14.1 ignore it and overwrite the + // point. `update_mode: insert_only` is ignored the same way on 1.16.3. + public static func insertsWithoutOverwrite(_ version: String?) -> Bool { + guard let version else { return false } + let parts = version.split(separator: ".").prefix(2).map { Int($0.prefix { $0.isNumber }) ?? 0 } + guard parts.count == 2 else { return false } + return parts[0] > 1 || (parts[0] == 1 && parts[1] >= 15) + } +} + public struct QdrantRowRefusal: Error, LocalizedError, Equatable, Sendable { public let rowIndex: Int public let reason: String @@ -50,13 +61,21 @@ public enum QdrantRowWriter { public static func writes( collection: String, changes: [QdrantRowChange], + serverVersion: String?, typeName: (String) -> String?, newID: () -> String = { UUID().uuidString.lowercased() } ) throws -> [QdrantRowWrite] { let points = "/collections/\(QdrantPathEncoding.segment(collection))/points" + let insertsSafely = QdrantServerVersion.insertsWithoutOverwrite(serverVersion) var writes: [QdrantRowWrite] = [] for change in changes { do { + if change.kind == .insert, !insertsSafely { + throw Reason(text: String( + format: String(localized: "Qdrant %@ overwrites a point that already has the id. Insert it from the REST editor, or upgrade Qdrant to 1.15 or later."), + serverVersion ?? "?" + )) + } writes += try self.writes(for: change, points: points, typeName: typeName, newID: newID) } catch let error as QdrantValueError { throw QdrantRowRefusal(rowIndex: change.rowIndex, reason: error.errorDescription ?? "") @@ -88,7 +107,7 @@ public enum QdrantRowWriter { let point: [String: Any] = ["id": id, "vector": values.vectors, "payload": values.payload] let lookup = try request("POST", points, ["ids": [id], "with_payload": false, "with_vector": false]) return [QdrantRowWrite( - request: try request("PUT", "\(points)?wait=true", ["points": [point], "update_mode": "insert_only"]), + request: try request("PUT", "\(points)?wait=true", ["points": [point], "update_filter": Self.matchesNoPoint]), rowIndex: change.rowIndex, mustFindNothing: lookup )] @@ -114,6 +133,9 @@ public enum QdrantRowWriter { } } + // An upsert updates only existing points this filter matches, and it matches none. + private static var matchesNoPoint: [String: Any] { ["must": [["has_id": [Any]()]]] } + private struct TypedValues { var payload: [String: Any] = [:] var vectors: [String: Any] = [:] @@ -133,9 +155,9 @@ public enum QdrantRowWriter { } continue } - guard let fieldType = QdrantFieldType(rawValue: type.lowercased()) else { throw unknownType(cell.column) } + guard let cellType = QdrantCellType(name: type) else { throw unknownType(cell.column) } let key = QdrantSchema.payloadKey(forColumn: cell.column) - values.payload[key] = try cell.text.map { try QdrantValue.payload($0, type: fieldType, column: cell.column) } ?? NSNull() + values.payload[key] = try cell.text.map { try QdrantValue.payload($0, cellType: cellType, column: cell.column) } ?? NSNull() } return values } diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift index a32c7ecb5d..f4ae63e5ba 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift @@ -137,10 +137,11 @@ public struct QdrantPayloadField: Sendable, Equatable { self.holdsScalars = holdsScalars } - // What a cell holds and is written back as. An indexed field holding lists keeps them lists. - public var cellType: QdrantFieldType { - guard isIndexed, holdsLists else { return type } - return holdsScalars ? .json : .array + // What a cell holds and is written back as. An indexed field that holds lists is `json()`, + // which the grid edits as JSON and a save writes as a list, or as one value of the index's type. + public var cellTypeName: String { + guard isIndexed, holdsLists else { return type.rawValue } + return QdrantCellType(element: type, allowsList: true).name } } @@ -182,8 +183,8 @@ public struct QdrantCollection: Sendable, Equatable { field(ofColumn: column)?.type } - public func cellType(ofColumn column: String) -> QdrantFieldType? { - field(ofColumn: column)?.cellType + public func cellTypeName(ofColumn column: String) -> String? { + field(ofColumn: column)?.cellTypeName } private func field(ofColumn column: String) -> QdrantPayloadField? { @@ -191,13 +192,16 @@ public struct QdrantCollection: Sendable, Equatable { return payloadFields.first { $0.name == key } } - public func including(_ points: [QdrantPoint]) -> QdrantCollection { + // A page's points widen the types a sample guessed, so a list first seen on page 2 is not saved as + // text. `addingKeys` is false for a grid page, whose columns stay the ones the table opened with. + public func including(_ points: [QdrantPoint], addingKeys: Bool = true) -> QdrantCollection { var fields = payloadFields var position = Dictionary(uniqueKeysWithValues: fields.enumerated().map { ($1.name, $0) }) for point in points { for key in point.payloadKeys { let observed = point.payload[key].flatMap(QdrantFieldType.observed) guard let index = position[key] else { + guard addingKeys else { continue } position[key] = fields.count fields.append(QdrantPayloadField( name: key, type: observed ?? .json, isIndexed: false, hasTypedValue: observed != nil @@ -312,7 +316,7 @@ public enum QdrantSchema { public static func columns(for collection: QdrantCollection) -> [QdrantColumn] { var columns = [QdrantColumn(name: idColumn, type: "id", isPrimaryKey: true)] columns += collection.payloadFields.map { - QdrantColumn(name: column(forPayloadKey: $0.name), type: $0.cellType.rawValue, isPrimaryKey: false) + QdrantColumn(name: column(forPayloadKey: $0.name), type: $0.cellTypeName, isPrimaryKey: false) } columns += collection.vectors.map { QdrantColumn(name: $0.column, type: $0.typeName, isPrimaryKey: false) } return columns @@ -329,7 +333,7 @@ public enum QdrantSchema { if let spec = collection.vectors.first(where: { $0.column == column }) { return spec.typeName } - return collection.cellType(ofColumn: column)?.rawValue + return collection.cellTypeName(ofColumn: column) } } @@ -349,3 +353,29 @@ public enum QdrantKeyPath { return path.contains(where: { ".[]\"".contains($0) }) ? nil : path } } + +public struct QdrantCellType: Sendable, Equatable { + public let element: QdrantFieldType + public let allowsList: Bool + + public init(element: QdrantFieldType, allowsList: Bool = false) { + self.element = element + self.allowsList = allowsList + } + + public init?(name: String) { + let lower = name.lowercased().trimmingCharacters(in: .whitespaces) + if lower.hasPrefix("json("), lower.hasSuffix(")"), + let element = QdrantFieldType(rawValue: String(lower.dropFirst("json(".count).dropLast())) { + self.init(element: element, allowsList: true) + } else if let element = QdrantFieldType(rawValue: lower) { + self.init(element: element) + } else { + return nil + } + } + + public var name: String { + allowsList ? "json(\(element.rawValue))" : element.rawValue + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantValue.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantValue.swift index 9b180236db..0564a0f46a 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantValue.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantValue.swift @@ -57,6 +57,13 @@ public enum QdrantValue { return value } + public static func payload(_ text: String, cellType: QdrantCellType, column: String) throws -> Any { + if cellType.allowsList, let list = QdrantJSON.parse(text) as? [Any] { + return list + } + return try payload(text, type: cellType.element, column: column) + } + public static func payload(_ text: String, type: QdrantFieldType, column: String) throws -> Any { switch type { case .keyword, .text, .string, .uuid, .datetime: diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift index 0367ca8100..9fa582eccc 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift @@ -21,6 +21,8 @@ public final class QdrantPagingCache: @unchecked Sendable { } } + // A boundary that moved means points were added or removed before it, so every cursor past it + // now starts its page at the wrong row. public func storeCursor(_ cursor: Any, at offset: Int, key: String) { lock.withLock { if cursors[key] == nil { @@ -29,6 +31,9 @@ public final class QdrantPagingCache: @unchecked Sendable { cursors[order.removeFirst()] = nil } } + if let known = cursors[key]?[offset], QdrantJSON.displayText(known) != QdrantJSON.displayText(cursor) { + cursors[key] = cursors[key]?.filter { $0.key < offset } + } cursors[key, default: [:]][offset] = cursor } } diff --git a/apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/ServerCertificatePurpose.swift b/apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/ServerCertificatePurpose.swift new file mode 100644 index 0000000000..26f9ac028e --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/ServerCertificatePurpose.swift @@ -0,0 +1,29 @@ +#if os(macOS) +import Foundation +import Security + +// Verify CA checks the chain with the basic X.509 policy, because Apple's SSL policy refuses a private +// CA's long-lived certificate as not standards compliant. The SSL policy was also what required a +// server certificate, so this puts that one check back: a client-only certificate from the same CA +// must not pass as the server. +public enum ServerCertificatePurpose { + private static let serverAuthentication = Data([0x2B, 0x06, 0x01, 0x05, 0x05, 0x07, 0x03, 0x01]) + private static let anyPurpose = Data([0x55, 0x1D, 0x25, 0x00]) + + public static func permitsServerAuthentication(_ certificate: SecCertificate) -> Bool { + let values = SecCertificateCopyValues(certificate, [kSecOIDExtendedKeyUsage] as CFArray, nil) as? [String: Any] + guard let usage = values?[kSecOIDExtendedKeyUsage as String] as? [String: Any], + let purposes = usage[kSecPropertyKeyValue as String] as? [Any] + else { return true } + return purposes.contains { purpose in + guard let oid = purpose as? Data else { return false } + return oid == serverAuthentication || oid == anyPurpose + } + } + + public static func leafPermitsServerAuthentication(_ trust: SecTrust) -> Bool { + guard let leaf = (SecTrustCopyCertificateChain(trust) as? [SecCertificate])?.first else { return false } + return permitsServerAuthentication(leaf) + } +} +#endif diff --git a/apple/Packages/TableProCore/Sources/TableProTrinoCore/TrinoTransport.swift b/apple/Packages/TableProCore/Sources/TableProTrinoCore/TrinoTransport.swift index 0fed9cb2d2..df41d8b00e 100644 --- a/apple/Packages/TableProCore/Sources/TableProTrinoCore/TrinoTransport.swift +++ b/apple/Packages/TableProCore/Sources/TableProTrinoCore/TrinoTransport.swift @@ -1,5 +1,6 @@ import Foundation import Security +import TableProTLSClientIdentity public struct TrinoHeaderFields: Sendable, Equatable { private let storage: [String: String] @@ -395,6 +396,9 @@ struct TrinoTLSChallengeHandler { guard SecTrustEvaluateWithError(serverTrust, &error) else { return .refuse(Self.failureKind(of: error), message: error.map { CFErrorCopyDescription($0) as String } ?? "") } + if tls.mode == .caOnly, !ServerCertificatePurpose.leafPermitsServerAuthentication(serverTrust) { + return .refuse(.untrustedCertificate, message: "The server's certificate is not issued for a server.") + } return TrinoChallengeAnswer(disposition: .useCredential, credential: URLCredential(trust: serverTrust)) } diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift index 4eb10b12e2..0df55161a6 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift @@ -37,6 +37,19 @@ struct QdrantClientTests { #expect(transport.recorded.count == before + 2) } + @Test("When a re-read boundary moves, the cursors past it are dropped instead of starting pages at the wrong row") + func movedBoundaryDropsLaterCursors() { + let cache = QdrantPagingCache() + cache.storeCursor(NSNumber(value: 101), at: 100, key: "k") + cache.storeCursor(NSNumber(value: 201), at: 200, key: "k") + cache.storeCursor(NSNumber(value: 101), at: 100, key: "k") + #expect(cache.nearestCursor(at: 200, key: "k").offset == 200) + cache.storeCursor(NSNumber(value: 102), at: 100, key: "k") + let nearest = cache.nearestCursor(at: 200, key: "k") + #expect(nearest.offset == 100) + #expect((nearest.cursor as? NSNumber)?.intValue == 102) + } + @Test("An offset past the end answers an empty page") func pastEnd() async throws { let client = makeClient(transport: scrollingTransport(count: 5)) diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift index 38a506c1c1..37e28ee592 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift @@ -5,7 +5,7 @@ import Testing struct QdrantRowWriteTests { private let types: [String: String] = [ "id": "id", "city": "keyword", "count": "integer", "pop": "float", "active": "bool", - "note": "string", "meta": "object", "payload.id": "keyword", + "note": "string", "meta": "object", "payload.id": "keyword", "tags": "json(keyword)", "vector": "vector(2)", "vector.text": "multivector", "vector.sp": "sparse_vector" ] @@ -13,6 +13,7 @@ struct QdrantRowWriteTests { try QdrantRowWriter.writes( collection: "docs", changes: changes, + serverVersion: "1.20.0", typeName: { types[$0] }, newID: { "00000000-0000-0000-0000-000000000001" } ).map(\.request.text) @@ -85,6 +86,31 @@ struct QdrantRowWriteTests { #expect(refusal(idChange)?.rowIndex == 5) } + @Test("An indexed field that holds lists takes a JSON list, or one value of the index's type") + func indexedListValues() throws { + let change = QdrantRowChange(kind: .update, rowIndex: 0, id: "7", cells: [QdrantCell(column: "tags", text: #"["a","b"]"#)]) + #expect(try writes([change]) == [ + "POST /collections/docs/points/payload?wait=true\n" + #"{"payload":{"tags":["a","b"]},"points":[7]}"# + ]) + let single = QdrantRowChange(kind: .update, rowIndex: 0, id: "7", cells: [QdrantCell(column: "tags", text: "42")]) + #expect(try writes([single]) == [ + "POST /collections/docs/points/payload?wait=true\n" + #"{"payload":{"tags":"42"},"points":[7]}"# + ]) + } + + @Test("A server that would overwrite a taken id gets no grid insert") + func insertNeedsConditionalUpsert() { + let change = QdrantRowChange(kind: .insert, rowIndex: 6, id: nil, cells: [QdrantCell(column: "city", text: "Hanoi")]) + for version in ["1.14.1", nil] as [String?] { + #expect(throws: QdrantRowRefusal.self) { + _ = try QdrantRowWriter.writes(collection: "docs", changes: [change], serverVersion: version, typeName: { types[$0] }) + } + } + #expect(QdrantServerVersion.insertsWithoutOverwrite("1.15.0")) + #expect(QdrantServerVersion.insertsWithoutOverwrite("2.0.1-dev")) + #expect(!QdrantServerVersion.insertsWithoutOverwrite("1.9.7")) + } + @Test("An insert carries its vectors, keeps an existing id untouched, and first checks that the id is free") func insert() throws { let change = QdrantRowChange(kind: .insert, rowIndex: 0, id: nil, cells: [ @@ -95,11 +121,11 @@ struct QdrantRowWriteTests { QdrantCell(column: "vector", text: "[1,2]") ]) let write = try #require(try QdrantRowWriter.writes( - collection: "docs", changes: [change], typeName: { types[$0] }, + collection: "docs", changes: [change], serverVersion: "1.20.0", typeName: { types[$0] }, newID: { "00000000-0000-0000-0000-000000000001" } ).first) #expect(write.request.text == "PUT /collections/docs/points?wait=true\n" - + #"{"points":[{"id":"00000000-0000-0000-0000-000000000001","payload":{"active":true,"city":"Hanoi"},"vector":{"":[1,2]}}],"update_mode":"insert_only"}"#) + + #"{"points":[{"id":"00000000-0000-0000-0000-000000000001","payload":{"active":true,"city":"Hanoi"},"vector":{"":[1,2]}}],"update_filter":{"must":[{"has_id":[]}]}}"#) #expect(write.mustFindNothing?.text == "POST /collections/docs/points\n" + #"{"ids":["00000000-0000-0000-0000-000000000001"],"with_payload":false,"with_vector":false}"#) } diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift index f05cb22745..7a905bc107 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift @@ -80,8 +80,8 @@ struct QdrantSchemaTests { ] ) let types = Dictionary(uniqueKeysWithValues: QdrantSchema.columns(for: collection).map { ($0.name, $0.type) }) - #expect(types["tags"] == "array") - #expect(types["mixed"] == "json") + #expect(types["tags"] == "json(keyword)") + #expect(types["mixed"] == "json(keyword)") #expect(types["city"] == "keyword") #expect(collection.fieldType(ofColumn: "tags") == .keyword) let condition = try QdrantFilterBuilder.condition( @@ -90,6 +90,21 @@ struct QdrantSchemaTests { #expect(canonical(condition) == #"{"key":"\"tags\"","match":{"value":"42"}}"#) } + @Test("A page's points widen a sampled type without adding columns the table did not open with") + func pageWidensTypes() throws { + let collection = QdrantCollection.parse( + name: "docs", + info: info(vectors: ["size": 2], payloadSchema: ["tags": ["data_type": "keyword"]]), + sample: [try point(#"{"id":1,"payload":{"tags":"a","note":"x"}}"#)] + ) + let page = [try point(#"{"id":101,"payload":{"tags":["a","b"],"note":["y"],"late":1}}"#)] + let typed = collection.including(page, addingKeys: false) + let types = Dictionary(uniqueKeysWithValues: QdrantSchema.columns(for: typed).map { ($0.name, $0.type) }) + #expect(types["tags"] == "json(keyword)") + #expect(types["note"] == "json") + #expect(types["late"] == nil) + } + @Test("A named vector called size is not mistaken for the unnamed vector's dimension") func vectorNamedSize() { let collection = QdrantCollection.parse( diff --git a/apple/Packages/TableProCore/Tests/TableProTLSClientIdentityTests/ServerCertificatePurposeTests.swift b/apple/Packages/TableProCore/Tests/TableProTLSClientIdentityTests/ServerCertificatePurposeTests.swift new file mode 100644 index 0000000000..6e10a38aa0 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProTLSClientIdentityTests/ServerCertificatePurposeTests.swift @@ -0,0 +1,37 @@ +#if os(macOS) +import Foundation +import Security +import TableProTLSClientIdentity +import TableProTLSTestFixtures +import Testing + +struct ServerCertificatePurposeTests { + private func certificate(_ pem: String) throws -> SecCertificate { + try #require(SecCertificateCreateWithData(nil, TLSTestFixtures.der(pem) as CFData)) + } + + private func trust(_ leafPEM: String) throws -> SecTrust { + var trust: SecTrust? + SecTrustCreateWithCertificates(try certificate(leafPEM), SecPolicyCreateBasicX509(), &trust) + let created = try #require(trust) + SecTrustSetAnchorCertificates(created, [try certificate(TLSTestFixtures.caCertificate)] as CFArray) + SecTrustSetAnchorCertificatesOnly(created, true) + return created + } + + @Test("A client-only certificate chains to the CA under the X.509 policy but is not a server's") + func clientCertificateRefused() throws { + let trust = try trust(TLSTestFixtures.clientCertificate) + #expect(SecTrustEvaluateWithError(trust, nil)) + #expect(!ServerCertificatePurpose.leafPermitsServerAuthentication(trust)) + } + + @Test("A server certificate and a certificate with no key usage list both pass") + func serverCertificatePasses() throws { + let trust = try trust(TLSTestFixtures.serverCertificate) + #expect(SecTrustEvaluateWithError(trust, nil)) + #expect(ServerCertificatePurpose.leafPermitsServerAuthentication(trust)) + #expect(ServerCertificatePurpose.permitsServerAuthentication(try certificate(TLSTestFixtures.caCertificate))) + } +} +#endif diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift index cf57e7d4eb..7b6aa99a60 100644 --- a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift @@ -50,8 +50,12 @@ extension QdrantPluginDriver { offset: request.offset, limit: request.limit, cache: paging ) } - let columns = QdrantSchema.columns(for: collection).map(\.name) - return render(points, columns: columns, collection: collection, started: started) + let typed = collection.including(points, addingKeys: false) + if typed != collection { + remember(typed) + } + let columns = QdrantSchema.columns(for: typed).map(\.name) + return render(points, columns: columns, collection: typed, started: started) } private func executeConsole( diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift index 7cb996c4d1..1a5266da47 100644 --- a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift @@ -142,7 +142,7 @@ extension QdrantPluginDriver { deletedRowIndices: deletedRowIndices, insertedRowIndices: insertedRowIndices) } do { - return try QdrantRowWriter.writes(collection: table, changes: tracked) { column in + return try QdrantRowWriter.writes(collection: table, changes: tracked, serverVersion: serverVersion) { column in typeNames[column] ?? known.flatMap { QdrantSchema.knownTypeName(of: column, in: $0) } }.map { write in PluginRowWrite( diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift index 357ce143bd..fe11918a98 100644 --- a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift @@ -94,10 +94,14 @@ internal final class QdrantPluginDriver: PluginDatabaseDriver, @unchecked Sendab return cached } let collection = try await requireClient().collection(named: name) - lock.withLock { cachedCollections[name] = collection } + remember(collection) return collection } + func remember(_ collection: QdrantCollection) { + lock.withLock { cachedCollections[collection.name] = collection } + } + // A write moves points between pages and may add payload keys. func forgetAfterWrite() { lock.withLock { cachedCollections.removeAll() } diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantURLSessionTransport.swift b/apple/Plugins/QdrantDriverPlugin/QdrantURLSessionTransport.swift index 1e9220f72e..1494bb383a 100644 --- a/apple/Plugins/QdrantDriverPlugin/QdrantURLSessionTransport.swift +++ b/apple/Plugins/QdrantDriverPlugin/QdrantURLSessionTransport.swift @@ -214,9 +214,14 @@ private final class QdrantTaskDelegate: NSObject, URLSessionTaskDelegate, @unche SecTrustSetPolicies(trust, SecPolicyCreateBasicX509()) } var error: CFError? - guard !SecTrustEvaluateWithError(trust, &error) else { return nil } - let detail = error.map { CFErrorCopyDescription($0) as String } ?? "" - return String(format: String(localized: "The server's certificate is not trusted. %@"), detail) + guard SecTrustEvaluateWithError(trust, &error) else { + let detail = error.map { CFErrorCopyDescription($0) as String } ?? "" + return String(format: String(localized: "The server's certificate is not trusted. %@"), detail) + } + guard tls.verification != .caOnly || ServerCertificatePurpose.leafPermitsServerAuthentication(trust) else { + return String(localized: "The server's certificate is not issued for a server, so it cannot prove the server's identity.") + } + return nil } } } diff --git a/apple/scripts/probes/qdrant-probe.swift b/apple/scripts/probes/qdrant-probe.swift index c43850aedf..71300c2f6c 100644 --- a/apple/scripts/probes/qdrant-probe.swift +++ b/apple/scripts/probes/qdrant-probe.swift @@ -130,7 +130,7 @@ let writes = try QdrantRowWriter.writes(collection: name, changes: [ QdrantRowChange(kind: .insert, rowIndex: 2, id: nil, cells: [ QdrantCell(column: "id", text: "999"), QdrantCell(column: "city", text: "new"), QdrantCell(column: "vector", text: "[1, 2]") ]) -], typeName: { types[$0] }) +], serverVersion: client.serverVersion, typeName: { types[$0] }) for write in writes { if let lookup = write.mustFindNothing { check(QdrantPoint.list(fromResponse: try await run(lookup.text).json)?.isEmpty == true, "insert lookup finds a free id") @@ -139,8 +139,11 @@ for write in writes { } let taken = try QdrantRowWriter.writes(collection: name, changes: [ QdrantRowChange(kind: .insert, rowIndex: 0, id: nil, cells: [QdrantCell(column: "id", text: "3")]) -], typeName: { types[$0] }) +], serverVersion: client.serverVersion, typeName: { types[$0] }) check(QdrantPoint.list(fromResponse: try await run(taken[0].mustFindNothing?.text).json)?.count == 1, "insert lookup finds a taken id") +_ = try await run(taken[0].request.text) +let untouched = QdrantPoint.list(fromResponse: try await run(#"POST \#(base)/points {"ids":[3],"with_payload":true,"with_vector":true}"#).json) +check(untouched?.first?.cell(for: "city") == "odd" && untouched?.first?.cell(for: "vector") == "[3,1]", "insert leaves a taken id untouched") let after = try await run(#"POST \#(base)/points {"ids":[1,2,999],"with_payload":true,"with_vector":true}"#) let byID = Dictionary(uniqueKeysWithValues: (QdrantPoint.list(fromResponse: after.json) ?? []).map { ($0.idText ?? "", $0) }) @@ -150,7 +153,7 @@ check(byID["2"] == nil, "delete") check(byID["999"]?.cell(for: "city") == "new" && byID["999"]?.cell(for: "vector") == "[1,2]", "insert with vector") let cleared = try QdrantRowWriter.writes(collection: name, changes: [ QdrantRowChange(kind: .update, rowIndex: 0, id: "1", cells: [QdrantCell(column: "vector.sp", text: nil)]) -], typeName: { types[$0] }) +], serverVersion: client.serverVersion, typeName: { types[$0] }) _ = try await run(cleared[0].request.text) let afterClear = QdrantPoint.list(fromResponse: try await run(#"POST \#(base)/points {"ids":[1],"with_vector":true}"#).json) check(afterClear?.first?.cell(for: "vector.sp") == nil, "NULL removes a vector") From 57df5b97c01a570ecf9febf3d1f4bb8f65ff3342 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 21:36:03 +0700 Subject: [PATCH 07/20] fix(plugin-qdrant): new id for duplicated rows, list element types in filters, snapshot routes by position --- .../TableProQdrantCore/QdrantConsole.swift | 10 ++++++- .../TableProQdrantCore/QdrantRowWrites.swift | 8 ++++-- .../TableProQdrantCore/QdrantSchema.swift | 26 ++++++++++++++++--- .../QdrantConsoleTests.swift | 3 +++ .../QdrantRowWriteTests.swift | 8 ++++++ .../QdrantSchemaTests.swift | 14 ++++++++++ apple/scripts/probes/qdrant-probe.swift | 11 +++++++- 7 files changed, 72 insertions(+), 8 deletions(-) diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConsole.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConsole.swift index 4bf7255aee..6c0be982a8 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConsole.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConsole.swift @@ -71,12 +71,20 @@ public struct QdrantRequestKind: Sendable, Equatable { ["points", "search", "matrix", "offsets"] ] + // `/snapshots`, `/collections/{name}/snapshots` and `/collections/{name}/shards/{id}/snapshots`, by + // position, so a collection that is itself named `snapshots` is not one. + static func touchesSnapshots(_ route: [String]) -> Bool { + if route.first == "snapshots" { return true } + guard route.count >= 3, route[0] == "collections" else { return false } + return route[2] == "snapshots" || (route.count >= 5 && route[2] == "shards" && route[4] == "snapshots") + } + public static func of(method: String, path: String) -> QdrantRequestKind { let verb = method.uppercased() guard let route = QdrantPathEncoding.segments(of: path) else { return QdrantRequestKind(tier: .write, touchesServerFiles: true) } - let touchesServerFiles = route.contains("snapshots") + let touchesServerFiles = touchesSnapshots(route) if verb == "GET" || verb == "HEAD" { return QdrantRequestKind(tier: .read, touchesServerFiles: touchesServerFiles) } diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift index 797c78c5d9..d94f09cb1d 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift @@ -101,9 +101,10 @@ public enum QdrantRowWriter { let id = try pointID(change.id) return [try write("POST", "\(points)/delete?wait=true", ["points": [id]], change)] case .insert: - let idText = change.cells.first { $0.column == QdrantSchema.idColumn }?.text.flatMap { $0.isEmpty ? nil : $0 } + let cells = change.cells.filter { $0.text != nil && $0.text != defaultMarker } + let idText = cells.first { $0.column == QdrantSchema.idColumn }?.text.flatMap { $0.isEmpty ? nil : $0 } let id = try pointID(idText ?? newID()) - let values = try typedValues(change.cells.filter { $0.text != nil }, typeName: typeName) + let values = try typedValues(cells, typeName: typeName) let point: [String: Any] = ["id": id, "vector": values.vectors, "payload": values.payload] let lookup = try request("POST", points, ["ids": [id], "with_payload": false, "with_vector": false]) return [QdrantRowWrite( @@ -133,6 +134,9 @@ public enum QdrantRowWriter { } } + // The host's marker for a cell the server fills; a duplicated row carries it in its key. + static let defaultMarker = "__DEFAULT__" + // An upsert updates only existing points this filter matches, and it matches none. private static var matchesNoPoint: [String: Any] { ["must": [["has_id": [Any]()]]] } diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift index f4ae63e5ba..c585be170b 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift @@ -55,6 +55,11 @@ public enum QdrantFieldType: String, Sendable, Equatable, CaseIterable { } } + static func elementType(of value: Any?) -> QdrantFieldType? { + guard let items = value as? [Any] else { return nil } + return items.compactMap(observed).reduce(nil) { merged, type in merged.map { $0.merged(with: type) } ?? type } + } + func merged(with other: QdrantFieldType) -> QdrantFieldType { if self == other { return self } if Set([self, other]) == [.integer, .float] { return .float } @@ -120,6 +125,8 @@ public struct QdrantPayloadField: Sendable, Equatable { public let hasTypedValue: Bool public let holdsLists: Bool public let holdsScalars: Bool + // The type of a sampled list's elements, so `tags = 42` on `["42"]` matches the string it holds. + public let elementType: QdrantFieldType? public init( name: String, @@ -127,7 +134,8 @@ public struct QdrantPayloadField: Sendable, Equatable { isIndexed: Bool, hasTypedValue: Bool = true, holdsLists: Bool = false, - holdsScalars: Bool = false + holdsScalars: Bool = false, + elementType: QdrantFieldType? = nil ) { self.name = name self.type = type @@ -135,6 +143,11 @@ public struct QdrantPayloadField: Sendable, Equatable { self.hasTypedValue = hasTypedValue self.holdsLists = holdsLists self.holdsScalars = holdsScalars + self.elementType = elementType + } + + public var filterType: QdrantFieldType { + type == .array ? elementType ?? .json : type } // What a cell holds and is written back as. An indexed field that holds lists is `json()`, @@ -180,7 +193,7 @@ public struct QdrantCollection: Sendable, Equatable { } public func fieldType(ofColumn column: String) -> QdrantFieldType? { - field(ofColumn: column)?.type + field(ofColumn: column)?.filterType } public func cellTypeName(ofColumn column: String) -> String? { @@ -200,11 +213,13 @@ public struct QdrantCollection: Sendable, Equatable { for point in points { for key in point.payloadKeys { let observed = point.payload[key].flatMap(QdrantFieldType.observed) + let elements = QdrantFieldType.elementType(of: point.payload[key]) guard let index = position[key] else { guard addingKeys else { continue } position[key] = fields.count fields.append(QdrantPayloadField( - name: key, type: observed ?? .json, isIndexed: false, hasTypedValue: observed != nil + name: key, type: observed ?? .json, isIndexed: false, hasTypedValue: observed != nil, + elementType: elements )) continue } @@ -219,7 +234,10 @@ public struct QdrantCollection: Sendable, Equatable { continue } let type = field.hasTypedValue ? field.type.merged(with: observed) : observed - fields[index] = QdrantPayloadField(name: key, type: type, isIndexed: false) + let elementType = [field.elementType, elements].compactMap { $0 }.reduce(nil) { merged, next in + merged.map { $0.merged(with: next) } ?? next + } + fields[index] = QdrantPayloadField(name: key, type: type, isIndexed: false, elementType: elementType) } } return QdrantCollection( diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantConsoleTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantConsoleTests.swift index 8619ca221e..5779368989 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantConsoleTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantConsoleTests.swift @@ -44,6 +44,9 @@ struct QdrantConsoleTests { let dotted = QdrantRequestKind.of(method: "POST", path: "/collections/docs/points/query/../delete") #expect(dotted == QdrantRequestKind(tier: .write, touchesServerFiles: true)) #expect(QdrantPathEncoding.segments(of: "/collections/%2e%2e/x") == nil) + #expect(!QdrantRequestKind.of(method: "POST", path: "/collections/snapshots/points/query").touchesServerFiles) + #expect(QdrantRequestKind.of(method: "POST", path: "/snapshots").touchesServerFiles) + #expect(QdrantRequestKind.of(method: "PUT", path: "/collections/docs/shards/0/snapshots/recover").touchesServerFiles) } @Test("A request path keeps its query string and never leaves the node") diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift index 37e28ee592..d70bbff3f3 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift @@ -98,6 +98,14 @@ struct QdrantRowWriteTests { ]) } + @Test("A duplicated row's default marker asks for a new id instead of failing as one") + func duplicatedRowGetsNewID() throws { + let change = QdrantRowChange(kind: .insert, rowIndex: 0, id: nil, cells: [ + QdrantCell(column: "id", text: "__DEFAULT__"), QdrantCell(column: "city", text: "Hanoi") + ]) + #expect(try writes([change]).first?.contains(#""id":"00000000-0000-0000-0000-000000000001""#) == true) + } + @Test("A server that would overwrite a taken id gets no grid insert") func insertNeedsConditionalUpsert() { let change = QdrantRowChange(kind: .insert, rowIndex: 6, id: nil, cells: [QdrantCell(column: "city", text: "Hanoi")]) diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift index 7a905bc107..25ec8c338d 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift @@ -105,6 +105,20 @@ struct QdrantSchemaTests { #expect(types["late"] == nil) } + @Test("A sampled list filters by the type of its elements, so 42 matches the string it holds") + func sampledListElements() throws { + let collection = QdrantCollection.parse( + name: "docs", info: info(vectors: ["size": 2]), + sample: [try point(#"{"id":1,"payload":{"tags":["42","a"],"ranks":[1,2]}}"#)] + ) + func condition(_ column: String, _ value: String) throws -> String { + canonical(try QdrantFilterBuilder.condition(for: QdrantFilterSpec(column: column, op: "=", value: value), collection: collection)) + } + #expect(try condition("tags", "42") == #"{"key":"\"tags\"","match":{"value":"42"}}"#) + #expect(try condition("ranks", "2") == #"{"key":"\"ranks\"","match":{"value":2}}"#) + #expect(QdrantSchema.columns(for: collection).first { $0.name == "tags" }?.type == "array") + } + @Test("A named vector called size is not mistaken for the unnamed vector's dimension") func vectorNamedSize() { let collection = QdrantCollection.parse( diff --git a/apple/scripts/probes/qdrant-probe.swift b/apple/scripts/probes/qdrant-probe.swift index 71300c2f6c..e588dbc634 100644 --- a/apple/scripts/probes/qdrant-probe.swift +++ b/apple/scripts/probes/qdrant-probe.swift @@ -50,6 +50,7 @@ var points: [[String: Any]] = [] for i in 1...250 { var payload: [String: Any] = ["city": i % 2 == 0 ? "even" : "odd", "body": "Hello World \(i)", "f": Double(i) + 0.5] if i <= 200 { payload["n"] = i } + if i <= 100 { payload["when"] = String(format: "2024-01-01T%02d:%02d:00Z", i / 60, i % 60) } if i == 7 { payload["note"] = NSNull() } if i == 8 { payload["note"] = "" } payload["meta"] = ["source": "doc\(i).pdf"] @@ -57,7 +58,7 @@ for i in 1...250 { points.append(["id": i, "vector": ["": [Double(i), 1]], "payload": payload]) } _ = try await run("PUT \(base)/points?wait=true\n" + (try QdrantJSON.text(["points": points]))) -for (field, schema) in [("n", "integer"), ("f", "float"), ("city", "keyword"), ("meta.source", "keyword")] { +for (field, schema) in [("n", "integer"), ("f", "float"), ("when", "datetime"), ("city", "keyword"), ("meta.source", "keyword")] { _ = try await run(#"PUT \#(base)/index?wait=true {"field_name":"\#(field)","field_schema":"\#(schema)"}"#) } @@ -105,6 +106,14 @@ func sorted(_ offset: Int, _ limit: Int, ascending: Bool = false) async throws - check(try await sorted(5, 3) == ["195", "194", "193"], "order_by with offset") check(try await sorted(198, 4) == ["2", "1", "201", "202"], "points without the key follow the ordered ones") check(try await sorted(248, 10) == ["249", "250"], "every point reachable when sorted") +let byDate = { (offset: Int, limit: Int) in + try await client.sortedPage( + collection: name, filter: nil, column: "when", key: "\"when\"", ascending: false, + offset: offset, limit: limit, cache: cache + ).map { $0.idText ?? "" } +} +check(try await byDate(0, 2) == ["100", "99"], "datetime sort orders by date") +check(try await byDate(99, 3) == ["1", "101", "102"], "datetime sort puts undated points after") _ = try await run(#"POST \#(base)/points/payload?wait=true {"payload":{"n":[1,2]},"points":[3]}"#) do { _ = try await client.sortedPage(collection: name, filter: nil, column: "n", key: "\"n\"", ascending: true, offset: 0, limit: 5, cache: QdrantPagingCache()) From de7428ceefee5221bb5696eeabe9ef485a2b24df Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 21:47:25 +0700 Subject: [PATCH 08/20] fix(plugin-qdrant): count with the page driver's types, type console results by their points --- .../QdrantPluginDriver+Execution.swift | 8 +- ...luginMetadataRegistry+QdrantDefaults.swift | 1 + .../TablePro/Resources/Localizable.xcstrings | 89 +++++++++++++++++-- 3 files changed, 90 insertions(+), 8 deletions(-) diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift index 7b6aa99a60..8cc6b4594e 100644 --- a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift @@ -71,11 +71,13 @@ extension QdrantPluginDriver { let route = QdrantPathEncoding.segments(of: request.path) ?? [] if let points = QdrantPoint.list(fromResponse: response.json), !points.isEmpty || route.contains("points") { let name = route.count >= 2 && route[0] == "collections" ? Self.collectionName(in: request.path) : nil - let collection = name.flatMap(rememberedCollection) + let known = name.flatMap(rememberedCollection) + // Typed by the points that came back as well, so an export of a fresh query keeps numbers numbers. + let typed = (known ?? QdrantCollection(name: name ?? "", vectors: [], payloadFields: [])).including(points) return render( points, - columns: QdrantPoint.columns(of: points, collection: collection), - collection: collection ?? QdrantCollection(name: name ?? "", vectors: [], payloadFields: []), + columns: QdrantPoint.columns(of: points, collection: known), + collection: typed, started: started ) } diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift index 95c510298b..7592bcc098 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift @@ -34,6 +34,7 @@ extension PluginMetadataRegistry { supportsOpportunisticTLS: false, verifiesServerWithSystemTrust: true, columnsAreSampled: true, + filtersTypedByReadRows: true, supportsPrincipalConnectionLimit: false ), schema: PluginMetadataSnapshot.SchemaInfo( diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index 4b14f977e6..90eabe204b 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -3556,6 +3556,10 @@ }, "%@ has no recorded reads." : { + }, + "%@ holds %@ values, and Qdrant matches words only in text." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "%@ holds a document or an array, and this text is not valid JSON." : { "extractionState" : "manual", @@ -3569,6 +3573,10 @@ } } }, + "%@ holds a list in %d points, and Qdrant sorts such a point once per value. Sort by a field that holds one value per point." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "%@ holds an empty key, which the MongoDB client library will not insert. Set it to NULL, then set it on the saved document." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -3593,7 +3601,23 @@ } } }, - "%@ holds booleans, so it matches only true or false, not %@." : { + "%@ holds booleans, so it takes true or false, not %@." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "%@ holds geo points, written as {\"lon\": 13.4, \"lat\": 52.5}." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "%@ holds integers, and %@ is not one." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "%@ holds JSON arrays, written as [1, 2, 3]." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "%@ holds JSON objects, written as {\"key\": \"value\"}." : { "extractionState" : "manual", "generatesSymbol" : false }, @@ -3705,6 +3729,10 @@ } } }, + "%@ is a multivector, written as arrays of numbers such as [[0.1, 0.2], [0.3, 0.4]]." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "%@ is a numeric field, and %@ is not a number." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -3739,6 +3767,10 @@ } } }, + "%@ is a sparse vector, written as {\"indices\": [1, 5], \"values\": [0.2, 0.7]}." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "%@ is a system collection. Its fields cannot be renamed or removed." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -3763,6 +3795,10 @@ } } }, + "%@ is a vector, written as an array of numbers such as [0.1, 0.2]." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "%@ is a view. Rename or remove the field in the collection it reads." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -4660,10 +4696,6 @@ } } }, - "%@ on %@ needs a full-text payload index on that field. Without one Qdrant matches nothing." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, "%@ on branch %@" : { "localizations" : { "en" : { @@ -16273,6 +16305,10 @@ } } }, + "A point's id cannot change. Insert a new row with the new id instead." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "a query" : { "localizations" : { "fr" : { @@ -16520,6 +16556,9 @@ } } } + }, + "A row with this key already exists, so the insert would not add one." : { + }, "A row with this primary key already exists. Edit that row instead." : { "extractionState" : "manual", @@ -139307,6 +139346,10 @@ } } }, + "Qdrant %@ overwrites a point that already has the id. Insert it from the REST editor, or upgrade Qdrant to 1.15 or later." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Qdrant answered a count without a number." : { "extractionState" : "manual", "generatesSymbol" : false @@ -139315,6 +139358,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "Qdrant cannot filter on %@, because a key holding a double quote has no path it accepts." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Qdrant cannot filter on %@." : { "extractionState" : "manual", "generatesSymbol" : false @@ -139323,6 +139370,14 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "Qdrant cannot match a whole %@ value in %@. Use a raw filter on one of its fields." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Qdrant cannot sort by %@." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Qdrant compares only numbers and dates with %@, and %@ holds neither." : { "extractionState" : "manual", "generatesSymbol" : false @@ -139331,10 +139386,18 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "Qdrant lists points by id in ascending order only." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Qdrant returned HTTP %d." : { "extractionState" : "manual", "generatesSymbol" : false }, + "Qdrant sorts by one field at a time. Remove the sort from the other columns." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Queries" : { "comment" : "A section header for the list of queries in the favorites tab.", "isCommentAutoGenerated" : true, @@ -179839,6 +179902,10 @@ }, "TablePro does not have Local Network access, so it cannot reach the SSH server at %@." : { + }, + "TablePro does not know what %@ holds. Reload the collection, then save again." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "TablePro does not run a pre-connect script on its own." : { @@ -191120,6 +191187,14 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "The server's certificate is not issued for a server, so it cannot prove the server's identity." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "The server's certificate is not trusted. %@" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The server's socket file on this Mac. SELECT @@socket in the mysql client shows it." : { }, @@ -199147,6 +199222,10 @@ } } }, + "This row has no point id, so Qdrant cannot tell which point to change." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This row has no value for the key column %@, so it cannot be matched." : { }, From b37d1521901cae76091d21774ecb0d31a3e8fb07 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 21:59:50 +0700 Subject: [PATCH 09/20] refactor(plugin-qdrant): type filters by payload index only, recheck sort lists on the first page --- .../QdrantFilterBuilder.swift | 33 ++++++++++----- .../TableProQdrantCore/QdrantSchema.swift | 29 +++---------- .../Wire/QdrantSortedPager.swift | 3 +- .../QdrantFilterBuilderTests.swift | 41 ++++++++++++------- .../QdrantSchemaTests.swift | 25 ++++++----- .../QdrantTestSupport.swift | 5 ++- ...luginMetadataRegistry+QdrantDefaults.swift | 1 - apple/scripts/probes/qdrant-probe.swift | 5 +++ 8 files changed, 76 insertions(+), 66 deletions(-) diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift index 660b2191dc..65700c0ec2 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift @@ -93,7 +93,7 @@ public enum QdrantFilterBuilder { guard let key = QdrantKeyPath.quoted(QdrantSchema.payloadKey(forColumn: filter.column)) else { throw QdrantFilterError.keyNotAddressable(column: filter.column) } - let field = Field(column: filter.column, key: key, type: collection.fieldType(ofColumn: filter.column) ?? .json) + let field = Field(column: filter.column, key: key, type: collection.indexType(ofColumn: filter.column)) switch op { case "=": @@ -131,9 +131,12 @@ public enum QdrantFilterBuilder { private struct Field { let column: String let key: String - let type: QdrantFieldType + // The payload index's type. Nil for a field nobody indexed, whose points may hold `"42"`, `42` + // or a list of either, so a value there matches every type its text can spell. + let type: QdrantFieldType? func equality(_ value: String) throws -> [String: Any] { + guard let type else { return anyType(equalTo: value) } switch type { case .keyword, .text, .string, .uuid: return match(value) @@ -150,16 +153,24 @@ public enum QdrantFilterBuilder { case .float, .datetime: return try range(op: "=", bounds: ["gte": value, "lte": value]) case .array, .json: - let trimmed = value.trimmingCharacters(in: .whitespaces) - if let number = QdrantValue.integer(trimmed) { return match(NSNumber(value: number)) } - if trimmed == "true" || trimmed == "false" { return match(trimmed == "true") } - if QdrantValue.number(trimmed) != nil { return try range(op: "=", bounds: ["gte": value, "lte": value]) } - return match(value) + return anyType(equalTo: value) case .geo, .object: throw QdrantFilterError.wholeValueNotMatchable(column: column, type: type.rawValue) } } + private func anyType(equalTo value: String) -> [String: Any] { + let trimmed = value.trimmingCharacters(in: .whitespaces) + var options = [match(value)] + if let number = QdrantValue.number(trimmed) { + options.append(["key": key, "range": ["gte": NSNumber(value: number), "lte": NSNumber(value: number)]]) + } + if let flag = QdrantValue.bool(trimmed) { + options.append(match(flag)) + } + return options.count == 1 ? options[0] : ["should": options] + } + // Qdrant's `match.any` takes strings alone or integers alone; anything else is one `should`. func anyOf(_ values: [String]) throws -> [String: Any] { let conditions = try anyOfConditions(values) @@ -177,7 +188,7 @@ public enum QdrantFilterBuilder { } func range(op: String, bounds: [String: String]) throws -> [String: Any] { - guard type.isOrdered || type == .json || type == .array else { + if let type, !type.isOrdered, type != .json, type != .array { throw QdrantFilterError.comparisonNeedsNumberOrDate(column: column, op: op) } var range: [String: Any] = [:] @@ -187,7 +198,7 @@ public enum QdrantFilterBuilder { range[bound] = trimmed } else if let number = QdrantValue.number(trimmed) { range[bound] = NSNumber(value: number) - } else if type.isOrdered { + } else if type?.isOrdered == true { throw QdrantValueError.notANumber(column: column, value: text) } else { range[bound] = trimmed @@ -198,7 +209,7 @@ public enum QdrantFilterBuilder { // Without a full-text index Qdrant still matches the words, lowercased. func textMatch(_ value: String) throws -> [String: Any] { - guard type.holdsText else { + if let type, !type.holdsText { throw QdrantFilterError.textMatchNeedsText(column: column, type: type.rawValue) } return ["key": key, "match": ["text": value]] @@ -210,7 +221,7 @@ public enum QdrantFilterBuilder { } var isBlank: [String: Any] { - guard type.holdsText else { return isEmpty } + if let type, !type.holdsText { return isEmpty } return ["should": [isEmpty, match("")]] } diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift index c585be170b..16460a1178 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift @@ -55,11 +55,6 @@ public enum QdrantFieldType: String, Sendable, Equatable, CaseIterable { } } - static func elementType(of value: Any?) -> QdrantFieldType? { - guard let items = value as? [Any] else { return nil } - return items.compactMap(observed).reduce(nil) { merged, type in merged.map { $0.merged(with: type) } ?? type } - } - func merged(with other: QdrantFieldType) -> QdrantFieldType { if self == other { return self } if Set([self, other]) == [.integer, .float] { return .float } @@ -125,8 +120,6 @@ public struct QdrantPayloadField: Sendable, Equatable { public let hasTypedValue: Bool public let holdsLists: Bool public let holdsScalars: Bool - // The type of a sampled list's elements, so `tags = 42` on `["42"]` matches the string it holds. - public let elementType: QdrantFieldType? public init( name: String, @@ -134,8 +127,7 @@ public struct QdrantPayloadField: Sendable, Equatable { isIndexed: Bool, hasTypedValue: Bool = true, holdsLists: Bool = false, - holdsScalars: Bool = false, - elementType: QdrantFieldType? = nil + holdsScalars: Bool = false ) { self.name = name self.type = type @@ -143,11 +135,6 @@ public struct QdrantPayloadField: Sendable, Equatable { self.hasTypedValue = hasTypedValue self.holdsLists = holdsLists self.holdsScalars = holdsScalars - self.elementType = elementType - } - - public var filterType: QdrantFieldType { - type == .array ? elementType ?? .json : type } // What a cell holds and is written back as. An indexed field that holds lists is `json()`, @@ -192,8 +179,9 @@ public struct QdrantCollection: Sendable, Equatable { self.info = info } - public func fieldType(ofColumn column: String) -> QdrantFieldType? { - field(ofColumn: column)?.filterType + // Only an index states a field's type; a sample guesses it from 100 points. + public func indexType(ofColumn column: String) -> QdrantFieldType? { + field(ofColumn: column).flatMap { $0.isIndexed ? $0.type : nil } } public func cellTypeName(ofColumn column: String) -> String? { @@ -213,13 +201,11 @@ public struct QdrantCollection: Sendable, Equatable { for point in points { for key in point.payloadKeys { let observed = point.payload[key].flatMap(QdrantFieldType.observed) - let elements = QdrantFieldType.elementType(of: point.payload[key]) guard let index = position[key] else { guard addingKeys else { continue } position[key] = fields.count fields.append(QdrantPayloadField( - name: key, type: observed ?? .json, isIndexed: false, hasTypedValue: observed != nil, - elementType: elements + name: key, type: observed ?? .json, isIndexed: false, hasTypedValue: observed != nil )) continue } @@ -234,10 +220,7 @@ public struct QdrantCollection: Sendable, Equatable { continue } let type = field.hasTypedValue ? field.type.merged(with: observed) : observed - let elementType = [field.elementType, elements].compactMap { $0 }.reduce(nil) { merged, next in - merged.map { $0.merged(with: next) } ?? next - } - fields[index] = QdrantPayloadField(name: key, type: type, isIndexed: false, elementType: elementType) + fields[index] = QdrantPayloadField(name: key, type: type, isIndexed: false) } } return QdrantCollection( diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantSortedPager.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantSortedPager.swift index 2271467404..a8f3302d07 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantSortedPager.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantSortedPager.swift @@ -31,7 +31,8 @@ public extension QdrantClient { } let checkKey = "\(collection)\u{0}\(key)\u{0}\((try? QdrantJSON.text(ordered)) ?? "")" - if !cache.isSortChecked(checkKey) { + // Rechecked on every first page, so a refresh sees a list another client stored since. + if offset == 0 || !cache.isSortChecked(checkKey) { // A field with no range index matches no point, so without asking Qdrant to order by it // every point would land in the remainder and the page would look sorted by id. _ = try await query(collection: collection, body: orderBody(offset: 0, limit: 1, withData: false)) diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift index a4043cedeb..0c7b68c947 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift @@ -10,12 +10,21 @@ struct QdrantFilterBuilderTests { )) } - @Test("Equality matches a keyword as text, an integer as a number and a boolean as true or false") - func equality() throws { + @Test("An indexed field matches as its index's type") + func indexedEquality() throws { #expect(try condition("city", "=", "Berlin") == #"{"key":"\"city\"","match":{"value":"Berlin"}}"#) - #expect(try condition("count", "=", "7") == #"{"key":"\"count\"","match":{"value":7}}"#) - #expect(try condition("active", "=", "1") == #"{"key":"\"active\"","match":{"value":true}}"#) - #expect(throws: QdrantValueError.notAnInteger(column: "count", value: "x")) { try condition("count", "=", "x") } + #expect(try condition("rank", "=", "7") == #"{"key":"\"rank\"","match":{"value":7}}"#) + #expect(try condition("flag", "=", "1") == #"{"key":"\"flag\"","match":{"value":true}}"#) + #expect(throws: QdrantValueError.notAnInteger(column: "rank", value: "x")) { try condition("rank", "=", "x") } + } + + @Test("A field nobody indexed matches every type its text can spell, whatever the sample guessed") + func unindexedEquality() throws { + #expect(try condition("count", "=", "42") + == #"{"should":[{"key":"\"count\"","match":{"value":"42"}},{"key":"\"count\"","range":{"gte":42,"lte":42}}]}"#) + #expect(try condition("note", "=", "1") + == #"{"should":[{"key":"\"note\"","match":{"value":"1"}},{"key":"\"note\"","range":{"gte":1,"lte":1}},{"key":"\"note\"","match":{"value":true}}]}"#) + #expect(try condition("note", "=", "Berlin") == #"{"key":"\"note\"","match":{"value":"Berlin"}}"#) } @Test("A key holding a dot stays one top-level key, and one holding a quote is refused") @@ -45,25 +54,26 @@ struct QdrantFilterBuilderTests { func ranges() throws { #expect(try condition("pop", ">", "2") == #"{"key":"\"pop\"","range":{"gt":2}}"#) #expect(try condition("count", "BETWEEN", "1,5", "5") == #"{"key":"\"count\"","range":{"gte":1,"lte":5}}"#) - #expect(throws: QdrantFilterError.comparisonNeedsNumberOrDate(column: "note", op: ">")) { - try condition("note", ">", "a") + #expect(throws: QdrantFilterError.comparisonNeedsNumberOrDate(column: "city", op: ">")) { + try condition("city", ">", "a") } } @Test("IN is the equality of each value: match any for strings or integers, ranges for floats") func inLists() throws { #expect(try condition("city", "IN", "Berlin, Paris") == #"{"key":"\"city\"","match":{"any":["Berlin","Paris"]}}"#) - #expect(try condition("count", "IN", "1,2") == #"{"key":"\"count\"","match":{"any":[1,2]}}"#) + #expect(try condition("rank", "IN", "1,2") == #"{"key":"\"rank\"","match":{"any":[1,2]}}"#) + #expect(try condition("note", "IN", "a, b") == #"{"key":"\"note\"","match":{"any":["a","b"]}}"#) #expect(try condition("pop", "IN", "1.5, 2.5") == #"{"should":[{"key":"\"pop\"","range":{"gte":1.5,"lte":1.5}},{"key":"\"pop\"","range":{"gte":2.5,"lte":2.5}}]}"#) - #expect(try condition("active", "IN", "true") - == #"{"should":[{"key":"\"active\"","match":{"value":true}}]}"#) + #expect(try condition("flag", "IN", "true") + == #"{"should":[{"key":"\"flag\"","match":{"value":true}}]}"#) } @Test("NOT IN excludes every value the way != excludes one") func notInLists() throws { - #expect(try condition("count", "NOT IN", "1,2") - == #"{"must_not":[{"key":"\"count\"","match":{"value":1}},{"key":"\"count\"","match":{"value":2}}]}"#) + #expect(try condition("rank", "NOT IN", "1,2") + == #"{"must_not":[{"key":"\"rank\"","match":{"value":1}},{"key":"\"rank\"","match":{"value":2}}]}"#) #expect(try condition("pop", "NOT IN", "1.5") == #"{"must_not":[{"key":"\"pop\"","range":{"gte":1.5,"lte":1.5}}]}"#) } @@ -78,9 +88,10 @@ struct QdrantFilterBuilderTests { func contains() throws { #expect(try condition("body", "CONTAINS", "vector") == #"{"key":"\"body\"","match":{"text":"vector"}}"#) #expect(try condition("note", "CONTAINS", "Ber") == #"{"key":"\"note\"","match":{"text":"Ber"}}"#) - #expect(throws: QdrantFilterError.textMatchNeedsText(column: "count", type: "integer")) { - try condition("count", "CONTAINS", "1") + #expect(throws: QdrantFilterError.textMatchNeedsText(column: "rank", type: "integer")) { + try condition("rank", "CONTAINS", "1") } + #expect(try condition("count", "CONTAINS", "1") == #"{"key":"\"count\"","match":{"text":"1"}}"#) } @Test("IS NULL matches a missing key as well as null, and IS EMPTY adds the empty string") @@ -89,7 +100,7 @@ struct QdrantFilterBuilderTests { #expect(try condition("note", "IS NOT NULL") == #"{"must_not":[{"is_empty":{"key":"\"note\""}}]}"#) #expect(try condition("note", "IS EMPTY") == #"{"should":[{"is_empty":{"key":"\"note\""}},{"key":"\"note\"","match":{"value":""}}]}"#) - #expect(try condition("count", "IS EMPTY") == #"{"is_empty":{"key":"\"count\""}}"#) + #expect(try condition("rank", "IS EMPTY") == #"{"is_empty":{"key":"\"rank\""}}"#) } @Test("The id column filters with has_id, keeping integer and UUID ids apart") diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift index 25ec8c338d..6392a2ee5b 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift @@ -45,7 +45,7 @@ struct QdrantSchemaTests { ) #expect(QdrantSchema.columns(for: collection).map(\.name) == ["id", "a.b", "rank", "metadata", "tags", "vector"]) #expect(collection.indexes.map(\.path) == ["\"a.b\"", "metadata.source", "rank", "tags[]"]) - #expect(collection.fieldType(ofColumn: "metadata") == .object) + #expect(QdrantSchema.typeName(of: "metadata", in: collection) == "object") } @Test("A payload key named like a point column shows as payload. and maps back") @@ -83,7 +83,7 @@ struct QdrantSchemaTests { #expect(types["tags"] == "json(keyword)") #expect(types["mixed"] == "json(keyword)") #expect(types["city"] == "keyword") - #expect(collection.fieldType(ofColumn: "tags") == .keyword) + #expect(collection.indexType(ofColumn: "tags") == .keyword) let condition = try QdrantFilterBuilder.condition( for: QdrantFilterSpec(column: "tags", op: "=", value: "42"), collection: collection ) @@ -105,17 +105,16 @@ struct QdrantSchemaTests { #expect(types["late"] == nil) } - @Test("A sampled list filters by the type of its elements, so 42 matches the string it holds") - func sampledListElements() throws { + @Test("Only an index types a filter; a sampled field's type never decides what a value matches") + func filtersIgnoreSampledTypes() throws { let collection = QdrantCollection.parse( - name: "docs", info: info(vectors: ["size": 2]), - sample: [try point(#"{"id":1,"payload":{"tags":["42","a"],"ranks":[1,2]}}"#)] + name: "docs", info: info(vectors: ["size": 2], payloadSchema: ["rank": ["data_type": "integer"]]), + sample: [try point(#"{"id":1,"payload":{"tags":["42","a"],"mixed":"42","rank":1}}"#), + try point(#"{"id":2,"payload":{"mixed":["42"]}}"#)] ) - func condition(_ column: String, _ value: String) throws -> String { - canonical(try QdrantFilterBuilder.condition(for: QdrantFilterSpec(column: column, op: "=", value: value), collection: collection)) - } - #expect(try condition("tags", "42") == #"{"key":"\"tags\"","match":{"value":"42"}}"#) - #expect(try condition("ranks", "2") == #"{"key":"\"ranks\"","match":{"value":2}}"#) + #expect(collection.indexType(ofColumn: "rank") == .integer) + #expect(collection.indexType(ofColumn: "tags") == nil) + #expect(collection.indexType(ofColumn: "mixed") == nil) #expect(QdrantSchema.columns(for: collection).first { $0.name == "tags" }?.type == "array") } @@ -134,8 +133,8 @@ struct QdrantSchemaTests { let collection = QdrantCollection.parse(name: "docs", info: info(vectors: ["size": 2]), sample: sample) let widened = collection.including([try point(#"{"id":2,"payload":{"late":true,"rank":3}}"#)]) #expect(widened.payloadFields.map(\.name) == ["city", "rank", "late"]) - #expect(widened.fieldType(ofColumn: "rank") == .integer) - #expect(widened.fieldType(ofColumn: "late") == .bool) + #expect(QdrantSchema.typeName(of: "rank", in: widened) == "integer") + #expect(QdrantSchema.typeName(of: "late", in: widened) == "bool") } @Test("Named, multi and sparse vectors each get a column typed by their kind") diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantTestSupport.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantTestSupport.swift index 3ac74609d2..baf4c4f4e2 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantTestSupport.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantTestSupport.swift @@ -61,11 +61,12 @@ let sampleCollection = QdrantCollection( QdrantPayloadField(name: "body", type: .text, isIndexed: true), QdrantPayloadField(name: "pop", type: .float, isIndexed: true), QdrantPayloadField(name: "seen", type: .datetime, isIndexed: true), + QdrantPayloadField(name: "rank", type: .integer, isIndexed: true), + QdrantPayloadField(name: "flag", type: .bool, isIndexed: true), + QdrantPayloadField(name: "home", type: .geo, isIndexed: true), QdrantPayloadField(name: "count", type: .integer, isIndexed: false), - QdrantPayloadField(name: "active", type: .bool, isIndexed: false), QdrantPayloadField(name: "note", type: .string, isIndexed: false), QdrantPayloadField(name: "tags", type: .array, isIndexed: false), - QdrantPayloadField(name: "home", type: .geo, isIndexed: false), QdrantPayloadField(name: "a.b", type: .string, isIndexed: false) ] ) diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift index 7592bcc098..95c510298b 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift @@ -34,7 +34,6 @@ extension PluginMetadataRegistry { supportsOpportunisticTLS: false, verifiesServerWithSystemTrust: true, columnsAreSampled: true, - filtersTypedByReadRows: true, supportsPrincipalConnectionLimit: false ), schema: PluginMetadataSnapshot.SchemaInfo( diff --git a/apple/scripts/probes/qdrant-probe.swift b/apple/scripts/probes/qdrant-probe.swift index e588dbc634..8d9921c363 100644 --- a/apple/scripts/probes/qdrant-probe.swift +++ b/apple/scripts/probes/qdrant-probe.swift @@ -53,6 +53,9 @@ for i in 1...250 { if i <= 100 { payload["when"] = String(format: "2024-01-01T%02d:%02d:00Z", i / 60, i % 60) } if i == 7 { payload["note"] = NSNull() } if i == 8 { payload["note"] = "" } + if i == 10 { payload["mixed"] = "42" } + if i == 11 { payload["mixed"] = 42 } + if i == 12 { payload["mixed"] = ["42"] } payload["meta"] = ["source": "doc\(i).pdf"] payload["a.b"] = "dotted" points.append(["id": i, "vector": ["": [Double(i), 1]], "payload": payload]) @@ -96,6 +99,8 @@ check(try await filtered([QdrantFilterSpec(column: "n", op: "IS NULL", value: "" check(try await filtered([QdrantFilterSpec(column: "note", op: "IS NULL", value: "")]) == 249, "IS NULL counts null and missing") check(try await filtered([QdrantFilterSpec(column: "note", op: "IS EMPTY", value: "")]) == 250, "IS EMPTY adds the empty string") check(try await filtered([QdrantFilterSpec(column: "a.b", op: "=", value: "dotted")]) == 250, "dotted top-level key") +check(try await filtered([QdrantFilterSpec(column: "mixed", op: "=", value: "42")]) == 3, "an unindexed value matches as text, number and list") +check(try await filtered([QdrantFilterSpec(column: "mixed", op: "NOT IN", value: "42, 7")]) == 247, "NOT IN excludes every spelling") func sorted(_ offset: Int, _ limit: Int, ascending: Bool = false) async throws -> [String] { try await client.sortedPage( From a889fcbc5f7c93fa067a398a64029f84652ed695 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 22:17:50 +0700 Subject: [PATCH 10/20] fix(plugin-qdrant): quote text in json cells, exact large integers, id search, drop cursors after outside changes --- .../QdrantFilterBuilder.swift | 14 ++++++++++- .../TableProQdrantCore/QdrantPoint.swift | 14 +++++++++++ .../Wire/QdrantScrollPager.swift | 19 +++++++++++++++ .../QdrantClientTests.swift | 23 ++++++++++++++----- .../QdrantFilterBuilderTests.swift | 4 ++++ .../QdrantSchemaTests.swift | 10 ++++++++ .../QdrantTestSupport.swift | 5 ++++ .../QdrantPluginDriver+Execution.swift | 19 +++++++++------ apple/scripts/probes/qdrant-probe.swift | 9 ++++++++ 9 files changed, 103 insertions(+), 14 deletions(-) diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift index 65700c0ec2..8c8a99650c 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift @@ -88,6 +88,9 @@ public enum QdrantFilterBuilder { } let op = filter.op.uppercased() if filter.column == QdrantSchema.idColumn { + if op == "CONTAINS" || op == "NOT CONTAINS" { + return idContains(op: op, value: filter.value) + } return try idCondition(op: op, value: filter.value) } guard let key = QdrantKeyPath.quoted(QdrantSchema.payloadKey(forColumn: filter.column)) else { @@ -162,7 +165,10 @@ public enum QdrantFilterBuilder { private func anyType(equalTo value: String) -> [String: Any] { let trimmed = value.trimmingCharacters(in: .whitespaces) var options = [match(value)] - if let number = QdrantValue.number(trimmed) { + // A range also finds 42.0, but a double rounds an integer past 2^53, so a larger one matches exactly. + if let integer = QdrantValue.integer(trimmed), integer.magnitude > 1 << 53 { + options.append(match(NSNumber(value: integer))) + } else if let number = QdrantValue.number(trimmed) { options.append(["key": key, "range": ["gte": NSNumber(value: number), "lte": NSNumber(value: number)]]) } if let flag = QdrantValue.bool(trimmed) { @@ -259,6 +265,12 @@ public enum QdrantFilterBuilder { } } + // Ids have no substring search, so only a whole id is found; Search All Rows still runs on the other columns. + private static func idContains(op: String, value: String) -> [String: Any] { + let found: [Any] = QdrantPointID.json(from: value).map { [$0] } ?? [] + return op == "NOT CONTAINS" ? negated([["has_id": found]]) : ["has_id": found] + } + private static func list(_ value: String, column: String) throws -> [String] { let parts = value.split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty } guard !parts.isEmpty else { throw QdrantFilterError.emptyList(column: column) } diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantPoint.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantPoint.swift index b22222e959..e544823b95 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantPoint.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantPoint.swift @@ -63,6 +63,16 @@ public struct QdrantPoint: @unchecked Sendable { return (vector as? [String: Any])?[name] } + // A column holding `"42"` in one point and `42` in another is typed `json`, so its strings keep their + // quotes: without them both cells read 42 and a save would turn the string into a number. + public func cell(for column: String, quotesText: Bool) -> String? { + guard quotesText, let text = payload[QdrantSchema.payloadKey(forColumn: column)] as? String else { + return cell(for: column) + } + return (try? JSONSerialization.data(withJSONObject: text, options: [.fragmentsAllowed, .withoutEscapingSlashes])) + .flatMap { String(data: $0, encoding: .utf8) } + } + public func cell(for column: String) -> String? { if column == QdrantSchema.idColumn { return idText @@ -80,6 +90,10 @@ public struct QdrantPoint: @unchecked Sendable { columns.map(cell(for:)) } + public func row(columns: [String], quotingTextIn jsonColumns: Set) -> [String?] { + columns.map { cell(for: $0, quotesText: jsonColumns.contains($0)) } + } + public static func columns(of points: [QdrantPoint], collection: QdrantCollection?) -> [String] { var columns = [QdrantSchema.idColumn] if points.contains(where: { $0.score != nil }) { diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift index 9fa582eccc..8ba405f338 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift @@ -9,6 +9,7 @@ public final class QdrantPagingCache: @unchecked Sendable { private var cursors: [String: [Int: Any]] = [:] private var order: [String] = [] private var checkedSorts: Set = [] + private var pointCounts: [String: Int] = [:] public init() {} @@ -38,6 +39,20 @@ public final class QdrantPagingCache: @unchecked Sendable { } } + // Another client's insert or delete shifts every later row, so cursors cached before it start pages + // at the wrong row. The point count is the one cheap signal of that the driver can read. + public func revalidate(collection: String, pointCount: Int) { + lock.withLock { + defer { pointCounts[collection] = pointCount } + guard let known = pointCounts[collection], known != pointCount else { return } + let prefix = "\(collection)\u{0}" + for key in cursors.keys where key.hasPrefix(prefix) { + cursors[key] = nil + } + order.removeAll { $0.hasPrefix(prefix) } + } + } + public func isSortChecked(_ key: String) -> Bool { lock.withLock { checkedSorts.contains(key) } } @@ -51,6 +66,7 @@ public final class QdrantPagingCache: @unchecked Sendable { cursors.removeAll() order.removeAll() checkedSorts.removeAll() + pointCounts.removeAll() } } } @@ -66,6 +82,9 @@ public extension QdrantClient { cache: QdrantPagingCache ) async throws -> [QdrantPoint] { let key = "\(collection)\u{0}\(filter.flatMap { try? QdrantJSON.text($0) } ?? "")" + if offset > 0 { + cache.revalidate(collection: collection, pointCount: try await count(collection: collection, filter: nil)) + } var (position, cursor) = cache.nearestCursor(at: offset, key: key) while position < offset { try Task.checkCancellation() diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift index 0df55161a6..dbb6116162 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift @@ -14,8 +14,8 @@ struct QdrantClientTests { #expect(first.compactMap(\.idText) == (1...10).map(String.init)) #expect(second.compactMap(\.idText) == (11...20).map(String.init)) - #expect(transport.recorded.count == 2) - #expect(bodyObject(transport.recorded[1])["offset"] as? Int == 11) + #expect(transport.scrolls.count == 2) + #expect(bodyObject(transport.scrolls[1])["offset"] as? Int == 11) } @Test("A jump walks ids only and keeps every cursor it passes, so a nearby jump starts from one") @@ -26,15 +26,15 @@ struct QdrantClientTests { let page = try await client.scrollPage(collection: "docs", filter: nil, offset: 20_000, limit: 10, cache: cache) #expect(page.compactMap(\.idText) == (20_001...20_010).map(String.init)) - let walks = transport.recorded.dropLast().map(bodyObject) + let walks = transport.scrolls.dropLast().map(bodyObject) #expect(walks.map { $0["limit"] as? Int } == [10_000, 10_000]) #expect(walks.allSatisfy { $0["with_payload"] as? Bool == false && $0["with_vector"] as? Bool == false }) - let before = transport.recorded.count + let before = transport.scrolls.count let nearby = try await client.scrollPage(collection: "docs", filter: nil, offset: 15_000, limit: 10, cache: cache) #expect(nearby.first?.idText == "15001") - #expect(bodyObject(transport.recorded[before])["offset"] as? Int == 10_001) - #expect(transport.recorded.count == before + 2) + #expect(bodyObject(transport.scrolls[before])["offset"] as? Int == 10_001) + #expect(transport.scrolls.count == before + 2) } @Test("When a re-read boundary moves, the cursors past it are dropped instead of starting pages at the wrong row") @@ -50,6 +50,17 @@ struct QdrantClientTests { #expect((nearest.cursor as? NSNumber)?.intValue == 102) } + @Test("A changed point count drops the collection's cursors, so a page after another client's delete starts at the right row") + func pointCountDropsCursors() { + let cache = QdrantPagingCache() + cache.revalidate(collection: "docs", pointCount: 400) + cache.storeCursor(NSNumber(value: 101), at: 100, key: "docs\u{0}") + cache.revalidate(collection: "docs", pointCount: 400) + #expect(cache.nearestCursor(at: 100, key: "docs\u{0}").offset == 100) + cache.revalidate(collection: "docs", pointCount: 399) + #expect(cache.nearestCursor(at: 100, key: "docs\u{0}").offset == 0) + } + @Test("An offset past the end answers an empty page") func pastEnd() async throws { let client = makeClient(transport: scrollingTransport(count: 5)) diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift index 0c7b68c947..f634a27413 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift @@ -25,6 +25,8 @@ struct QdrantFilterBuilderTests { #expect(try condition("note", "=", "1") == #"{"should":[{"key":"\"note\"","match":{"value":"1"}},{"key":"\"note\"","range":{"gte":1,"lte":1}},{"key":"\"note\"","match":{"value":true}}]}"#) #expect(try condition("note", "=", "Berlin") == #"{"key":"\"note\"","match":{"value":"Berlin"}}"#) + #expect(try condition("count", "=", "9007199254740993") + == #"{"should":[{"key":"\"count\"","match":{"value":"9007199254740993"}},{"key":"\"count\"","match":{"value":9007199254740993}}]}"#) } @Test("A key holding a dot stays one top-level key, and one holding a quote is refused") @@ -109,6 +111,8 @@ struct QdrantFilterBuilderTests { #expect(try condition("id", "IN", "1, \(uuid)") == #"{"has_id":[1,"\#(uuid)"]}"#) #expect(try condition("id", "!=", "1") == #"{"must_not":[{"has_id":[1]}]}"#) #expect(throws: QdrantFilterError.notAPointID(value: "abc")) { try condition("id", "=", "abc") } + #expect(try condition("id", "CONTAINS", "7") == #"{"has_id":[7]}"#) + #expect(try condition("id", "CONTAINS", "Berlin") == #"{"has_id":[]}"#) } @Test("Vectors cannot be filtered and unsupported operators say so") diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift index 6392a2ee5b..79322af4a0 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift @@ -118,6 +118,16 @@ struct QdrantSchemaTests { #expect(QdrantSchema.columns(for: collection).first { $0.name == "tags" }?.type == "array") } + @Test("A json column keeps a string's quotes, so 42 and \"42\" read apart") + func jsonCellsQuoteText() throws { + let text = try point(#"{"id":1,"payload":{"mixed":"42"}}"#) + let number = try point(#"{"id":2,"payload":{"mixed":42}}"#) + #expect(text.row(columns: ["mixed"], quotingTextIn: ["mixed"]) == [#""42""#]) + #expect(number.row(columns: ["mixed"], quotingTextIn: ["mixed"]) == ["42"]) + #expect(text.row(columns: ["mixed"], quotingTextIn: []) == ["42"]) + #expect(try QdrantValue.payload(#""42""#, type: .json, column: "mixed") as? String == "42") + } + @Test("A named vector called size is not mistaken for the unnamed vector's dimension") func vectorNamedSize() { let collection = QdrantCollection.parse( diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantTestSupport.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantTestSupport.swift index baf4c4f4e2..7eed1272c5 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantTestSupport.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantTestSupport.swift @@ -18,6 +18,8 @@ final class RecordingTransport: QdrantTransport, @unchecked Sendable { func cancelAll() {} var recorded: [QdrantHTTPRequest] { lock.withLock { requests } } + + var scrolls: [QdrantHTTPRequest] { recorded.filter { $0.url.path.hasSuffix("/points/scroll") } } } func jsonResponse(_ object: Any, status: Int = 200) -> QdrantHTTPResponse { @@ -45,6 +47,9 @@ func scrollResponse(_ ids: [Int], next: Int?) -> QdrantHTTPResponse { // Integer ids 1...count, answering `scroll` from `offset` (an id) the way Qdrant does. func scrollingTransport(count: Int) -> RecordingTransport { RecordingTransport { request in + if request.url.path.hasSuffix("/points/count") { + return jsonResponse(["result": ["count": count], "status": "ok"]) + } let body = bodyObject(request) let limit = body["limit"] as? Int ?? 10 let start = body["offset"] as? Int ?? 1 diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift index 8cc6b4594e..a8dbd15218 100644 --- a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift @@ -96,17 +96,21 @@ extension QdrantPluginDriver { collection: QdrantCollection, started: Date ) -> PluginQueryResult { - PluginQueryResult( + let typeNames = columns.map { QdrantSchema.typeName(of: $0, in: collection) } + return PluginQueryResult( columns: columns, - columnTypeNames: columns.map { QdrantSchema.typeName(of: $0, in: collection) }, - rows: points.map(cells(columns: columns)), + columnTypeNames: typeNames, + rows: points.map(cells(columns: columns, typeNames: typeNames)), rowsAffected: 0, executionTime: Date().timeIntervalSince(started) ) } - private func cells(columns: [String]) -> (QdrantPoint) -> [PluginCellValue] { - { point in point.row(columns: columns).map { $0.map(PluginCellValue.text) ?? .null } } + private func cells(columns: [String], typeNames: [String]) -> (QdrantPoint) -> [PluginCellValue] { + let jsonColumns = Set(zip(columns, typeNames).filter { $0.1 == QdrantFieldType.json.rawValue }.map(\.0)) + return { point in + point.row(columns: columns, quotingTextIn: jsonColumns).map { $0.map(PluginCellValue.text) ?? .null } + } } private func renderJSON(_ response: QdrantHTTPResponse, started: Date) -> PluginQueryResult { @@ -165,13 +169,14 @@ extension QdrantPluginDriver { collection = collection.including(points) } let columns = QdrantSchema.columns(for: collection).map(\.name) + let typeNames = columns.map { QdrantSchema.typeName(of: $0, in: collection) } continuation.yield(.header(PluginStreamHeader( columns: columns, - columnTypeNames: columns.map { QdrantSchema.typeName(of: $0, in: collection) }, + columnTypeNames: typeNames, estimatedRowCount: collection.pointsCount ))) for try await points in client.pointPages(collection: name, pageSize: Self.exportPageSize) { - continuation.yield(.rows(points.map(cells(columns: columns)))) + continuation.yield(.rows(points.map(cells(columns: columns, typeNames: typeNames)))) } } } diff --git a/apple/scripts/probes/qdrant-probe.swift b/apple/scripts/probes/qdrant-probe.swift index 8d9921c363..6d4ffbdee6 100644 --- a/apple/scripts/probes/qdrant-probe.swift +++ b/apple/scripts/probes/qdrant-probe.swift @@ -81,6 +81,10 @@ check(p0.first?.idText == "1" && p1.first?.idText == "101" && p2.count == 50 && let jump = try await client.scrollPage(collection: name, filter: nil, offset: 170, limit: 10, cache: QdrantPagingCache()) check(jump.map { $0.idText ?? "" } == (171...180).map(String.init), "jump walks to offset") check(try await client.scrollPage(collection: name, filter: nil, offset: 400, limit: 10, cache: cache).isEmpty, "past end") +_ = try await run(#"POST \#(base)/points/delete?wait=true {"points":[50]}"#) +let shifted = try await client.scrollPage(collection: name, filter: nil, offset: 100, limit: 2, cache: cache) +check(shifted.map { $0.idText ?? "" } == ["102", "103"], "a page after another client's delete starts at the right row") +_ = try await run(#"PUT \#(base)/points?wait=true {"points":[{"id":50,"vector":[50,1],"payload":{"city":"even","body":"Hello World 50","f":50.5,"n":50,"meta":{"source":"doc50.pdf"},"a.b":"dotted","when":"2024-01-01T00:50:00Z"}}]}"#) func filtered(_ specs: [QdrantFilterSpec], _ mode: String = "AND") async throws -> Int { let filter = try QdrantFilterBuilder.filter(specs, logicMode: mode, collection: collection) @@ -92,6 +96,11 @@ check(try await filtered([QdrantFilterSpec(column: "n", op: ">", value: "190"), check(try await filtered([QdrantFilterSpec(column: "n", op: "<", value: "3"), QdrantFilterSpec(column: "n", op: ">", value: "198")], "OR") == 4, "or") check(try await filtered([QdrantFilterSpec(column: "body", op: "CONTAINS", value: "world")]) == 250, "contains without a text index matches words") check(try await filtered([QdrantFilterSpec(column: "id", op: "IN", value: "1,2,3")]) == 3, "has_id") +let searchAll = [ + QdrantFilterSpec(column: "id", op: "CONTAINS", value: "Berlin"), + QdrantFilterSpec(column: "city", op: "CONTAINS", value: "even") +] +check(try await filtered(searchAll, "OR") == 125, "search all rows skips the id column") check(try await filtered([QdrantFilterSpec(column: "n", op: "IN", value: "1,2,300")]) == 2, "match any") check(try await filtered([QdrantFilterSpec(column: "f", op: "IN", value: "1.5, 2.5")]) == 2, "IN on floats") check(try await filtered([QdrantFilterSpec(column: "f", op: "NOT IN", value: "1.5")]) == 249, "NOT IN on floats") From a17978307db8d48a9f54adaa6fd4737fc738317c Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 22:28:45 +0700 Subject: [PATCH 11/20] refactor(plugin-qdrant): type cells by stored values, not the payload index --- .../TableProQdrantCore/QdrantRowWrites.swift | 4 +- .../TableProQdrantCore/QdrantSchema.swift | 61 +++++++------------ .../TableProQdrantCore/QdrantValue.swift | 7 --- .../Wire/QdrantScrollPager.swift | 4 +- .../QdrantRowWriteTests.swift | 22 +++---- .../QdrantSchemaTests.swift | 18 +++--- 6 files changed, 47 insertions(+), 69 deletions(-) diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift index d94f09cb1d..712a645360 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift @@ -159,9 +159,9 @@ public enum QdrantRowWriter { } continue } - guard let cellType = QdrantCellType(name: type) else { throw unknownType(cell.column) } + guard let fieldType = QdrantFieldType(rawValue: type.lowercased()) else { throw unknownType(cell.column) } let key = QdrantSchema.payloadKey(forColumn: cell.column) - values.payload[key] = try cell.text.map { try QdrantValue.payload($0, cellType: cellType, column: cell.column) } ?? NSNull() + values.payload[key] = try cell.text.map { try QdrantValue.payload($0, type: fieldType, column: cell.column) } ?? NSNull() } return values } diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift index 16460a1178..c639762857 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift @@ -55,6 +55,17 @@ public enum QdrantFieldType: String, Sendable, Equatable, CaseIterable { } } + func stores(_ stored: QdrantFieldType) -> Bool { + switch self { + case .keyword, .text, .uuid, .datetime, .string: return stored == .string + case .integer: return stored == .integer + case .float: return stored == .float || stored == .integer + case .bool: return stored == .bool + case .geo: return stored == .object + case .object, .array, .json: return stored == self + } + } + func merged(with other: QdrantFieldType) -> QdrantFieldType { if self == other { return self } if Set([self, other]) == [.integer, .float] { return .float } @@ -113,35 +124,34 @@ public struct QdrantVectorSpec: Sendable, Equatable { public struct QdrantPayloadField: Sendable, Equatable { public let name: String - // What a filter compares: an index types each element of a list, so this is the element's type. + // The index's type for an indexed field, which filters use; the sampled type otherwise. public let type: QdrantFieldType public let isIndexed: Bool // False while every sampled value was null, so the first real value sets the type outright. public let hasTypedValue: Bool - public let holdsLists: Bool - public let holdsScalars: Bool + // What the sampled points actually store. An index does not constrain it: a keyword index + // still lets a point hold 42 or a list. + public let storedType: QdrantFieldType? public init( name: String, type: QdrantFieldType, isIndexed: Bool, hasTypedValue: Bool = true, - holdsLists: Bool = false, - holdsScalars: Bool = false + storedType: QdrantFieldType? = nil ) { self.name = name self.type = type self.isIndexed = isIndexed self.hasTypedValue = hasTypedValue - self.holdsLists = holdsLists - self.holdsScalars = holdsScalars + self.storedType = storedType } - // What a cell holds and is written back as. An indexed field that holds lists is `json()`, - // which the grid edits as JSON and a save writes as a list, or as one value of the index's type. + // What a cell holds and is written back as, so a value saves as the type it was stored with. + // The index's name stands in only when every value seen fits it. public var cellTypeName: String { - guard isIndexed, holdsLists else { return type.rawValue } - return QdrantCellType(element: type, allowsList: true).name + guard isIndexed, let storedType else { return type.rawValue } + return type.stores(storedType) ? type.rawValue : storedType.rawValue } } @@ -214,8 +224,7 @@ public struct QdrantCollection: Sendable, Equatable { if field.isIndexed { fields[index] = QdrantPayloadField( name: key, type: field.type, isIndexed: true, - holdsLists: field.holdsLists || observed == .array, - holdsScalars: field.holdsScalars || observed != .array + storedType: field.storedType.map { $0.merged(with: observed) } ?? observed ) continue } @@ -354,29 +363,3 @@ public enum QdrantKeyPath { return path.contains(where: { ".[]\"".contains($0) }) ? nil : path } } - -public struct QdrantCellType: Sendable, Equatable { - public let element: QdrantFieldType - public let allowsList: Bool - - public init(element: QdrantFieldType, allowsList: Bool = false) { - self.element = element - self.allowsList = allowsList - } - - public init?(name: String) { - let lower = name.lowercased().trimmingCharacters(in: .whitespaces) - if lower.hasPrefix("json("), lower.hasSuffix(")"), - let element = QdrantFieldType(rawValue: String(lower.dropFirst("json(".count).dropLast())) { - self.init(element: element, allowsList: true) - } else if let element = QdrantFieldType(rawValue: lower) { - self.init(element: element) - } else { - return nil - } - } - - public var name: String { - allowsList ? "json(\(element.rawValue))" : element.rawValue - } -} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantValue.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantValue.swift index 0564a0f46a..9b180236db 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantValue.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantValue.swift @@ -57,13 +57,6 @@ public enum QdrantValue { return value } - public static func payload(_ text: String, cellType: QdrantCellType, column: String) throws -> Any { - if cellType.allowsList, let list = QdrantJSON.parse(text) as? [Any] { - return list - } - return try payload(text, type: cellType.element, column: column) - } - public static func payload(_ text: String, type: QdrantFieldType, column: String) throws -> Any { switch type { case .keyword, .text, .string, .uuid, .datetime: diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift index 8ba405f338..1881146e72 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift @@ -82,9 +82,7 @@ public extension QdrantClient { cache: QdrantPagingCache ) async throws -> [QdrantPoint] { let key = "\(collection)\u{0}\(filter.flatMap { try? QdrantJSON.text($0) } ?? "")" - if offset > 0 { - cache.revalidate(collection: collection, pointCount: try await count(collection: collection, filter: nil)) - } + cache.revalidate(collection: collection, pointCount: try await count(collection: collection, filter: nil)) var (position, cursor) = cache.nearestCursor(at: offset, key: key) while position < offset { try Task.checkCancellation() diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift index d70bbff3f3..70d11e5f56 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift @@ -5,7 +5,7 @@ import Testing struct QdrantRowWriteTests { private let types: [String: String] = [ "id": "id", "city": "keyword", "count": "integer", "pop": "float", "active": "bool", - "note": "string", "meta": "object", "payload.id": "keyword", "tags": "json(keyword)", + "note": "string", "meta": "object", "payload.id": "keyword", "mixed": "json", "vector": "vector(2)", "vector.text": "multivector", "vector.sp": "sparse_vector" ] @@ -86,16 +86,16 @@ struct QdrantRowWriteTests { #expect(refusal(idChange)?.rowIndex == 5) } - @Test("An indexed field that holds lists takes a JSON list, or one value of the index's type") - func indexedListValues() throws { - let change = QdrantRowChange(kind: .update, rowIndex: 0, id: "7", cells: [QdrantCell(column: "tags", text: #"["a","b"]"#)]) - #expect(try writes([change]) == [ - "POST /collections/docs/points/payload?wait=true\n" + #"{"payload":{"tags":["a","b"]},"points":[7]}"# - ]) - let single = QdrantRowChange(kind: .update, rowIndex: 0, id: "7", cells: [QdrantCell(column: "tags", text: "42")]) - #expect(try writes([single]) == [ - "POST /collections/docs/points/payload?wait=true\n" + #"{"payload":{"tags":"42"},"points":[7]}"# - ]) + @Test("A json column saves the JSON its cell spells, so a quoted string stays a string") + func jsonColumnValues() throws { + func written(_ text: String) throws -> String? { + try writes([QdrantRowChange(kind: .update, rowIndex: 0, id: "7", cells: [QdrantCell(column: "mixed", text: text)])]).first + } + let prefix = "POST /collections/docs/points/payload?wait=true\n" + #expect(try written(#""42""#) == prefix + #"{"payload":{"mixed":"42"},"points":[7]}"#) + #expect(try written("42") == prefix + #"{"payload":{"mixed":42},"points":[7]}"#) + #expect(try written(#""[]""#) == prefix + #"{"payload":{"mixed":"[]"},"points":[7]}"#) + #expect(try written(#"["a"]"#) == prefix + #"{"payload":{"mixed":["a"]},"points":[7]}"#) } @Test("A duplicated row's default marker asks for a new id instead of failing as one") diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift index 79322af4a0..162e1c1aa8 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift @@ -67,22 +67,26 @@ struct QdrantSchemaTests { #expect(QdrantSchema.column(forPayloadKey: "payload.id") == "payload.payload.id") } - @Test("An indexed field holding lists stays a list in its cell, while filters compare its elements") + @Test("A cell is typed by what points store; the index names it only when every value fits") func indexedLists() throws { let collection = QdrantCollection.parse( name: "docs", info: info(vectors: ["size": 2], payloadSchema: [ - "tags": ["data_type": "keyword"], "mixed": ["data_type": "keyword"], "city": ["data_type": "keyword"] + "tags": ["data_type": "keyword"], "mixed": ["data_type": "keyword"], "city": ["data_type": "keyword"], + "code": ["data_type": "keyword"], "price": ["data_type": "float"], "fresh": ["data_type": "integer"] ]), sample: [ - try point(#"{"id":1,"payload":{"tags":["a","b"],"mixed":["a"],"city":"Berlin"}}"#), - try point(#"{"id":2,"payload":{"tags":["c"],"mixed":"b"}}"#) + try point(#"{"id":1,"payload":{"tags":["a","b"],"mixed":["a"],"city":"Berlin","code":42,"price":3}}"#), + try point(#"{"id":2,"payload":{"tags":["c"],"mixed":"b","price":2.5}}"#) ] ) let types = Dictionary(uniqueKeysWithValues: QdrantSchema.columns(for: collection).map { ($0.name, $0.type) }) - #expect(types["tags"] == "json(keyword)") - #expect(types["mixed"] == "json(keyword)") + #expect(types["tags"] == "array") + #expect(types["mixed"] == "json") #expect(types["city"] == "keyword") + #expect(types["code"] == "integer") + #expect(types["price"] == "float") + #expect(types["fresh"] == "integer") #expect(collection.indexType(ofColumn: "tags") == .keyword) let condition = try QdrantFilterBuilder.condition( for: QdrantFilterSpec(column: "tags", op: "=", value: "42"), collection: collection @@ -100,7 +104,7 @@ struct QdrantSchemaTests { let page = [try point(#"{"id":101,"payload":{"tags":["a","b"],"note":["y"],"late":1}}"#)] let typed = collection.including(page, addingKeys: false) let types = Dictionary(uniqueKeysWithValues: QdrantSchema.columns(for: typed).map { ($0.name, $0.type) }) - #expect(types["tags"] == "json(keyword)") + #expect(types["tags"] == "json") #expect(types["note"] == "json") #expect(types["late"] == nil) } From f61f63d156dc36a4cae343ed3f422c863ffe2784 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 22:38:11 +0700 Subject: [PATCH 12/20] fix(plugin-qdrant): revalidate cursors per filtered list, let contains match nothing on non-text columns --- .../QdrantFilterBuilder.swift | 22 ++++++++---------- .../Wire/QdrantScrollPager.swift | 23 ++++++++----------- .../QdrantClientTests.swift | 18 ++++++++------- .../QdrantFilterBuilderTests.swift | 9 ++++---- .../TablePro/Resources/Localizable.xcstrings | 4 ---- apple/scripts/probes/qdrant-probe.swift | 13 +++++++++++ 6 files changed, 47 insertions(+), 42 deletions(-) diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift index 8c8a99650c..ad7cb6a70b 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift @@ -20,7 +20,6 @@ public enum QdrantFilterError: Error, LocalizedError, Equatable, Sendable { case notAPointID(value: String) case emptyList(column: String) case comparisonNeedsNumberOrDate(column: String, op: String) - case textMatchNeedsText(column: String, type: String) case wholeValueNotMatchable(column: String, type: String) case vectorNotFilterable(column: String) case keyNotAddressable(column: String) @@ -44,8 +43,6 @@ public enum QdrantFilterError: Error, LocalizedError, Equatable, Sendable { format: String(localized: "Qdrant compares only numbers and dates with %@, and %@ holds neither."), op, column ) - case .textMatchNeedsText(let column, let type): - return String(format: String(localized: "%@ holds %@ values, and Qdrant matches words only in text."), column, type) case .wholeValueNotMatchable(let column, let type): return String( format: String(localized: "Qdrant cannot match a whole %@ value in %@. Use a raw filter on one of its fields."), @@ -83,10 +80,13 @@ public enum QdrantFilterBuilder { if filter.column == rawFilterColumn { return try raw(filter.value) } + let op = filter.op.uppercased() guard !QdrantSchema.isVectorColumn(filter.column) else { + // Search All Rows asks every column; a vector holds no text, so it finds nothing there. + if op == "CONTAINS" { return ["has_id": [Any]()] } + if op == "NOT CONTAINS" { return negated([["has_id": [Any]()]]) } throw QdrantFilterError.vectorNotFilterable(column: filter.column) } - let op = filter.op.uppercased() if filter.column == QdrantSchema.idColumn { if op == "CONTAINS" || op == "NOT CONTAINS" { return idContains(op: op, value: filter.value) @@ -115,9 +115,9 @@ public enum QdrantFilterBuilder { case "NOT IN": return negated(try field.anyOfConditions(list(filter.value, column: filter.column))) case "CONTAINS": - return try field.textMatch(filter.value) + return field.textMatch(filter.value) case "NOT CONTAINS": - return negated([try field.textMatch(filter.value)]) + return negated([field.textMatch(filter.value)]) case "IS NULL": return field.isEmpty case "IS NOT NULL": @@ -213,12 +213,10 @@ public enum QdrantFilterBuilder { return ["key": key, "range": range] } - // Without a full-text index Qdrant still matches the words, lowercased. - func textMatch(_ value: String) throws -> [String: Any] { - if let type, !type.holdsText { - throw QdrantFilterError.textMatchNeedsText(column: column, type: type.rawValue) - } - return ["key": key, "match": ["text": value]] + // Without a full-text index Qdrant still matches the words, lowercased, and a value that is not + // text matches nothing rather than failing, so Search All Rows can ask every column. + func textMatch(_ value: String) -> [String: Any] { + ["key": key, "match": ["text": value]] } // `is_empty` covers a missing key, null and []; `is_null` would miss the missing key the grid shows as NULL. diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift index 1881146e72..9a84aae00a 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift @@ -9,7 +9,7 @@ public final class QdrantPagingCache: @unchecked Sendable { private var cursors: [String: [Int: Any]] = [:] private var order: [String] = [] private var checkedSorts: Set = [] - private var pointCounts: [String: Int] = [:] + private var counts: [String: Int] = [:] public init() {} @@ -39,17 +39,14 @@ public final class QdrantPagingCache: @unchecked Sendable { } } - // Another client's insert or delete shifts every later row, so cursors cached before it start pages - // at the wrong row. The point count is the one cheap signal of that the driver can read. - public func revalidate(collection: String, pointCount: Int) { + // Another client's insert, delete or payload change shifts every later row of a filtered list, so + // cursors cached before it start pages at the wrong row. The list's count is the cheap signal of it. + public func revalidate(key: String, count: Int) { lock.withLock { - defer { pointCounts[collection] = pointCount } - guard let known = pointCounts[collection], known != pointCount else { return } - let prefix = "\(collection)\u{0}" - for key in cursors.keys where key.hasPrefix(prefix) { - cursors[key] = nil - } - order.removeAll { $0.hasPrefix(prefix) } + defer { counts[key] = count } + guard let known = counts[key], known != count else { return } + cursors[key] = nil + order.removeAll { $0 == key } } } @@ -66,7 +63,7 @@ public final class QdrantPagingCache: @unchecked Sendable { cursors.removeAll() order.removeAll() checkedSorts.removeAll() - pointCounts.removeAll() + counts.removeAll() } } } @@ -82,7 +79,7 @@ public extension QdrantClient { cache: QdrantPagingCache ) async throws -> [QdrantPoint] { let key = "\(collection)\u{0}\(filter.flatMap { try? QdrantJSON.text($0) } ?? "")" - cache.revalidate(collection: collection, pointCount: try await count(collection: collection, filter: nil)) + cache.revalidate(key: key, count: try await count(collection: collection, filter: filter)) var (position, cursor) = cache.nearestCursor(at: offset, key: key) while position < offset { try Task.checkCancellation() diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift index dbb6116162..a91333c91e 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift @@ -50,15 +50,17 @@ struct QdrantClientTests { #expect((nearest.cursor as? NSNumber)?.intValue == 102) } - @Test("A changed point count drops the collection's cursors, so a page after another client's delete starts at the right row") - func pointCountDropsCursors() { + @Test("A changed count of the filtered list drops its cursors, so a page after another client's change starts at the right row") + func countDropsCursors() { let cache = QdrantPagingCache() - cache.revalidate(collection: "docs", pointCount: 400) - cache.storeCursor(NSNumber(value: 101), at: 100, key: "docs\u{0}") - cache.revalidate(collection: "docs", pointCount: 400) - #expect(cache.nearestCursor(at: 100, key: "docs\u{0}").offset == 100) - cache.revalidate(collection: "docs", pointCount: 399) - #expect(cache.nearestCursor(at: 100, key: "docs\u{0}").offset == 0) + cache.revalidate(key: "docs\u{0}f", count: 400) + cache.storeCursor(NSNumber(value: 101), at: 100, key: "docs\u{0}f") + cache.storeCursor(NSNumber(value: 7), at: 100, key: "docs\u{0}g") + cache.revalidate(key: "docs\u{0}f", count: 400) + #expect(cache.nearestCursor(at: 100, key: "docs\u{0}f").offset == 100) + cache.revalidate(key: "docs\u{0}f", count: 399) + #expect(cache.nearestCursor(at: 100, key: "docs\u{0}f").offset == 0) + #expect(cache.nearestCursor(at: 100, key: "docs\u{0}g").offset == 100) } @Test("An offset past the end answers an empty page") diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift index f634a27413..6a5b49dbb2 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift @@ -86,13 +86,11 @@ struct QdrantFilterBuilderTests { } } - @Test("CONTAINS matches words in any text field, indexed or not, and refuses numbers") + @Test("CONTAINS matches words in any field, indexed or not, and a value that is not text matches nothing") func contains() throws { #expect(try condition("body", "CONTAINS", "vector") == #"{"key":"\"body\"","match":{"text":"vector"}}"#) #expect(try condition("note", "CONTAINS", "Ber") == #"{"key":"\"note\"","match":{"text":"Ber"}}"#) - #expect(throws: QdrantFilterError.textMatchNeedsText(column: "rank", type: "integer")) { - try condition("rank", "CONTAINS", "1") - } + #expect(try condition("rank", "CONTAINS", "1") == #"{"key":"\"rank\"","match":{"text":"1"}}"#) #expect(try condition("count", "CONTAINS", "1") == #"{"key":"\"count\"","match":{"text":"1"}}"#) } @@ -116,8 +114,9 @@ struct QdrantFilterBuilderTests { } @Test("Vectors cannot be filtered and unsupported operators say so") - func refusals() { + func refusals() throws { #expect(throws: QdrantFilterError.vectorNotFilterable(column: "vector")) { try condition("vector", "=", "1") } + #expect(try condition("vector.sp", "CONTAINS", "a") == #"{"has_id":[]}"#) #expect(throws: QdrantFilterError.unsupportedOperator("STARTS WITH")) { try condition("city", "STARTS WITH", "B") } } diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index 90eabe204b..fdac408d4b 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -3556,10 +3556,6 @@ }, "%@ has no recorded reads." : { - }, - "%@ holds %@ values, and Qdrant matches words only in text." : { - "extractionState" : "manual", - "generatesSymbol" : false }, "%@ holds a document or an array, and this text is not valid JSON." : { "extractionState" : "manual", diff --git a/apple/scripts/probes/qdrant-probe.swift b/apple/scripts/probes/qdrant-probe.swift index 6d4ffbdee6..8f4b6b97bc 100644 --- a/apple/scripts/probes/qdrant-probe.swift +++ b/apple/scripts/probes/qdrant-probe.swift @@ -101,6 +101,19 @@ let searchAll = [ QdrantFilterSpec(column: "city", op: "CONTAINS", value: "even") ] check(try await filtered(searchAll, "OR") == 125, "search all rows skips the id column") +let searchNumbers = [ + QdrantFilterSpec(column: "n", op: "CONTAINS", value: "5"), + QdrantFilterSpec(column: "vector.sp", op: "CONTAINS", value: "5"), + QdrantFilterSpec(column: "city", op: "CONTAINS", value: "even") +] +check(try await filtered(searchNumbers, "OR") == 125, "search all rows runs on number and vector columns") +let evenFilter = try QdrantFilterBuilder.filter([QdrantFilterSpec(column: "city", op: "=", value: "even")], logicMode: "AND", collection: collection) +let evenCache = QdrantPagingCache() +let evenBefore = try await client.scrollPage(collection: name, filter: evenFilter, offset: 10, limit: 2, cache: evenCache) +_ = try await run(#"POST \#(base)/points/payload?wait=true {"payload":{"city":"odd"},"points":[2]}"#) +let evenAfter = try await client.scrollPage(collection: name, filter: evenFilter, offset: 10, limit: 2, cache: evenCache) +_ = try await run(#"POST \#(base)/points/payload?wait=true {"payload":{"city":"even"},"points":[2]}"#) +check(evenBefore.map { $0.idText ?? "" } == ["22", "24"] && evenAfter.map { $0.idText ?? "" } == ["24", "26"], "a filtered page follows another client's payload change") check(try await filtered([QdrantFilterSpec(column: "n", op: "IN", value: "1,2,300")]) == 2, "match any") check(try await filtered([QdrantFilterSpec(column: "f", op: "IN", value: "1.5, 2.5")]) == 2, "IN on floats") check(try await filtered([QdrantFilterSpec(column: "f", op: "NOT IN", value: "1.5")]) == 249, "NOT IN on floats") From d695491342c363435b927a68110903c4c372055c Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 22:46:38 +0700 Subject: [PATCH 13/20] fix(plugin-qdrant): keep id, score and vectors out of payload field lookups --- .../Sources/TableProQdrantCore/QdrantPoint.swift | 4 +++- .../Sources/TableProQdrantCore/QdrantSchema.swift | 8 +++++++- .../TableProQdrantCoreTests/QdrantSchemaTests.swift | 13 +++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantPoint.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantPoint.swift index e544823b95..b725bd810c 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantPoint.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantPoint.swift @@ -66,7 +66,9 @@ public struct QdrantPoint: @unchecked Sendable { // A column holding `"42"` in one point and `42` in another is typed `json`, so its strings keep their // quotes: without them both cells read 42 and a save would turn the string into a number. public func cell(for column: String, quotesText: Bool) -> String? { - guard quotesText, let text = payload[QdrantSchema.payloadKey(forColumn: column)] as? String else { + guard quotesText, !QdrantSchema.isPointColumn(column), + let text = payload[QdrantSchema.payloadKey(forColumn: column)] as? String + else { return cell(for: column) } return (try? JSONSerialization.data(withJSONObject: text, options: [.fragmentsAllowed, .withoutEscapingSlashes])) diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift index c639762857..d21cdfcb2c 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift @@ -198,7 +198,9 @@ public struct QdrantCollection: Sendable, Equatable { field(ofColumn: column)?.cellTypeName } + // `id`, `score` and the vectors are the point's own; a payload key of that name is `payload.`. private func field(ofColumn column: String) -> QdrantPayloadField? { + guard !QdrantSchema.isPointColumn(column) else { return nil } let key = QdrantSchema.payloadKey(forColumn: column) return payloadFields.first { $0.name == key } } @@ -298,6 +300,10 @@ public enum QdrantSchema { return collections.compactMap { $0["name"] as? String }.sorted() } + public static func isPointColumn(_ column: String) -> Bool { + column == idColumn || column == scoreColumn || isVectorColumn(column) + } + public static func isVectorColumn(_ column: String) -> Bool { column == vectorColumn || column.hasPrefix("\(vectorColumn).") } @@ -333,8 +339,8 @@ public enum QdrantSchema { } public static func typeName(of column: String, in collection: QdrantCollection) -> String { - if let known = knownTypeName(of: column, in: collection) { return known } if column == scoreColumn { return QdrantFieldType.float.rawValue } + if let known = knownTypeName(of: column, in: collection) { return known } return isVectorColumn(column) ? "vector" : QdrantFieldType.json.rawValue } diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift index 162e1c1aa8..988348532e 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift @@ -122,6 +122,19 @@ struct QdrantSchemaTests { #expect(QdrantSchema.columns(for: collection).first { $0.name == "tags" }?.type == "array") } + @Test("A payload key named score never stands in for the similarity score column") + func scoreColumnIsThePoints() throws { + let hit = try point(#"{"id":1,"score":0.75,"payload":{"score":"high"}}"#) + let collection = QdrantCollection(name: "docs", vectors: [], payloadFields: [ + QdrantPayloadField(name: "score", type: .json, isIndexed: false) + ]) + let columns = QdrantPoint.columns(of: [hit], collection: collection) + #expect(columns == ["id", "score", "payload.score"]) + #expect(QdrantSchema.typeName(of: "score", in: collection) == "float") + #expect(QdrantSchema.typeName(of: "payload.score", in: collection) == "json") + #expect(hit.row(columns: columns, quotingTextIn: ["score", "payload.score"]) == ["1", "0.75", #""high""#]) + } + @Test("A json column keeps a string's quotes, so 42 and \"42\" read apart") func jsonCellsQuoteText() throws { let text = try point(#"{"id":1,"payload":{"mixed":"42"}}"#) From 09824a9e00f4e278a024e3858e0692fe64dd05d4 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 22:55:13 +0700 Subject: [PATCH 14/20] fix(plugin-qdrant): write integers in float columns exactly --- .../TableProCore/Sources/TableProQdrantCore/QdrantValue.swift | 2 ++ .../Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift | 2 ++ 2 files changed, 4 insertions(+) diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantValue.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantValue.swift index 9b180236db..3945423068 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantValue.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantValue.swift @@ -65,6 +65,8 @@ public enum QdrantValue { guard let value = integer(text) else { throw QdrantValueError.notAnInteger(column: column, value: text) } return NSNumber(value: value) case .float: + // A float column also holds integers, and a double rounds one past 2^53. + if let value = integer(text) { return NSNumber(value: value) } guard let value = number(text) else { throw QdrantValueError.notANumber(column: column, value: text) } return NSNumber(value: value) case .bool: diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift index 70d11e5f56..f9a5b962cb 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift @@ -142,6 +142,8 @@ struct QdrantRowWriteTests { func payloadValues() throws { #expect(try QdrantValue.payload("42", type: .keyword, column: "c") as? String == "42") #expect(try QdrantValue.payload("0", type: .bool, column: "c") as? Bool == false) + #expect((try QdrantValue.payload("9007199254740993", type: .float, column: "c") as? NSNumber)?.int64Value == 9_007_199_254_740_993) + #expect((try QdrantValue.payload("2.5", type: .float, column: "c") as? NSNumber)?.doubleValue == 2.5) #expect(try QdrantValue.payload(#"["a"]"#, type: .array, column: "c") as? [String] == ["a"]) #expect(try QdrantValue.payload("42", type: .json, column: "c") as? Int == 42) #expect(try QdrantValue.payload("plain", type: .json, column: "c") as? String == "plain") From dda9941c968690f19b9e77a23a3f22d28ced93db Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 23:08:21 +0700 Subject: [PATCH 15/20] refactor(plugin-qdrant): page with a server-side offset instead of cached cursors, suppress the default sort --- .../TableProQdrantCore/QdrantRowWrites.swift | 10 ++- .../Wire/QdrantScrollPager.swift | 85 ++++--------------- .../Wire/QdrantSortedPager.swift | 5 +- .../QdrantClientTests.swift | 72 +++++----------- .../QdrantDriverPlugin/QdrantPlugin.swift | 7 +- .../QdrantPluginDriver+Execution.swift | 4 +- apple/scripts/probes/qdrant-probe.swift | 17 ++-- 7 files changed, 67 insertions(+), 133 deletions(-) diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift index 712a645360..3998db5a77 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift @@ -40,13 +40,21 @@ public struct QdrantRowWrite: Sendable, Equatable { } public enum QdrantServerVersion { + public static func hasQueryAPI(_ version: String?) -> Bool { + isAtLeast(version, major: 1, minor: 10) + } + // Measured: 1.15.5 and later honor `update_filter`; 1.13.6 and 1.14.1 ignore it and overwrite the // point. `update_mode: insert_only` is ignored the same way on 1.16.3. public static func insertsWithoutOverwrite(_ version: String?) -> Bool { + isAtLeast(version, major: 1, minor: 15) + } + + static func isAtLeast(_ version: String?, major: Int, minor: Int) -> Bool { guard let version else { return false } let parts = version.split(separator: ".").prefix(2).map { Int($0.prefix { $0.isNumber }) ?? 0 } guard parts.count == 2 else { return false } - return parts[0] > 1 || (parts[0] == 1 && parts[1] >= 15) + return parts[0] > major || (parts[0] == major && parts[1] >= minor) } } diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift index 9a84aae00a..28c0f493e8 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift @@ -1,55 +1,12 @@ import Foundation -// `scroll` pages by point id, so a row offset is reached by walking ids from the nearest cursor -// already known. Every cursor a walk passes is kept, so paging on or jumping nearby costs one request. +// The sort checks a sorted browse has already passed, so only its first page pays for them. public final class QdrantPagingCache: @unchecked Sendable { - private static let maxQueries = 32 - private let lock = NSLock() - private var cursors: [String: [Int: Any]] = [:] - private var order: [String] = [] private var checkedSorts: Set = [] - private var counts: [String: Int] = [:] public init() {} - public func nearestCursor(at offset: Int, key: String) -> (offset: Int, cursor: Any?) { - lock.withLock { - guard let known = cursors[key], - let best = known.keys.filter({ $0 <= offset }).max() - else { return (0, nil) } - return (best, known[best]) - } - } - - // A boundary that moved means points were added or removed before it, so every cursor past it - // now starts its page at the wrong row. - public func storeCursor(_ cursor: Any, at offset: Int, key: String) { - lock.withLock { - if cursors[key] == nil { - order.append(key) - if order.count > Self.maxQueries { - cursors[order.removeFirst()] = nil - } - } - if let known = cursors[key]?[offset], QdrantJSON.displayText(known) != QdrantJSON.displayText(cursor) { - cursors[key] = cursors[key]?.filter { $0.key < offset } - } - cursors[key, default: [:]][offset] = cursor - } - } - - // Another client's insert, delete or payload change shifts every later row of a filtered list, so - // cursors cached before it start pages at the wrong row. The list's count is the cheap signal of it. - public func revalidate(key: String, count: Int) { - lock.withLock { - defer { counts[key] = count } - guard let known = counts[key], known != count else { return } - cursors[key] = nil - order.removeAll { $0 == key } - } - } - public func isSortChecked(_ key: String) -> Bool { lock.withLock { checkedSorts.contains(key) } } @@ -59,28 +16,27 @@ public final class QdrantPagingCache: @unchecked Sendable { } public func removeAll() { - lock.withLock { - cursors.removeAll() - order.removeAll() - checkedSorts.removeAll() - counts.removeAll() - } + lock.withLock { checkedSorts.removeAll() } } } public extension QdrantClient { static let walkBatchSize = 10_000 - func scrollPage( - collection: String, - filter: [String: Any]?, - offset: Int, - limit: Int, - cache: QdrantPagingCache - ) async throws -> [QdrantPoint] { - let key = "\(collection)\u{0}\(filter.flatMap { try? QdrantJSON.text($0) } ?? "")" - cache.revalidate(key: key, count: try await count(collection: collection, filter: filter)) - var (position, cursor) = cache.nearestCursor(at: offset, key: key) + // A page in id order. The Query API with no query pages by id and applies the offset on the server, + // so a page is always the rows at that offset now; a cursor cached from an earlier read would start + // at the wrong row once another client added or removed a point before it. A node older than 1.10 + // has no Query API, so its pages walk the ids from the start. + func pointPage(collection: String, filter: [String: Any]?, offset: Int, limit: Int) async throws -> [QdrantPoint] { + if QdrantServerVersion.hasQueryAPI(serverVersion) { + var body: [String: Any] = ["limit": limit, "offset": offset, "with_payload": true, "with_vector": true] + if let filter { + body["filter"] = filter + } + return try await query(collection: collection, body: body) + } + var position = 0 + var cursor: Any? while position < offset { try Task.checkCancellation() let page = try await scroll( @@ -90,16 +46,11 @@ public extension QdrantClient { guard let next = page.nextCursor else { return [] } position += page.points.count cursor = next - cache.storeCursor(next, at: position, key: key) } - let page = try await scroll( + return try await scroll( collection: collection, body: Self.scrollBody(filter: filter, limit: limit, cursor: cursor, withData: true) - ) - if let next = page.nextCursor { - cache.storeCursor(next, at: offset + page.points.count, key: key) - } - return page.points + ).points } // Without vectors this reads payloads only, which is how an export learns every key before its header. diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantSortedPager.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantSortedPager.swift index a8f3302d07..302080324e 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantSortedPager.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantSortedPager.swift @@ -59,12 +59,11 @@ public extension QdrantClient { } guard points.count < limit else { return points } let withoutValue = QdrantFilterBuilder.allOf(filter, QdrantFilterBuilder.negated([hasValue])) - return points + (try await scrollPage( + return points + (try await pointPage( collection: collection, filter: withoutValue, offset: max(offset - orderedCount, 0), - limit: limit - points.count, - cache: cache + limit: limit - points.count )) } } diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift index a91333c91e..c927c42468 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift @@ -3,70 +3,42 @@ import Foundation import Testing struct QdrantClientTests { - @Test("Paging forward reuses the cursor the previous page returned") - func pagingForward() async throws { - let transport = scrollingTransport(count: 25) + @Test("On Qdrant 1.10 or later a page is one query with the offset applied on the server") + func pageByServerOffset() async throws { + let transport = RecordingTransport { request in + switch request.url.path { + case "/": return jsonResponse(["version": "1.20.0"]) + case "/collections": return jsonResponse(["result": ["collections": [Any]()]]) + default: return jsonResponse(["result": ["points": [["id": 101, "payload": [String: Any]()]]]]) + } + } let client = makeClient(transport: transport) - let cache = QdrantPagingCache() - - let first = try await client.scrollPage(collection: "docs", filter: nil, offset: 0, limit: 10, cache: cache) - let second = try await client.scrollPage(collection: "docs", filter: nil, offset: 10, limit: 10, cache: cache) - - #expect(first.compactMap(\.idText) == (1...10).map(String.init)) - #expect(second.compactMap(\.idText) == (11...20).map(String.init)) - #expect(transport.scrolls.count == 2) - #expect(bodyObject(transport.scrolls[1])["offset"] as? Int == 11) + try await client.connect(requestTimeout: { 5 }) + let filter: [String: Any] = ["must": [["has_id": [101]]]] + let page = try await client.pointPage(collection: "docs", filter: filter, offset: 100, limit: 10) + #expect(page.compactMap(\.idText) == ["101"]) + let request = try #require(transport.recorded.last) + #expect(request.url.path == "/collections/docs/points/query") + #expect(canonical(bodyObject(request)) == #"{"filter":{"must":[{"has_id":[101]}]},"limit":10,"offset":100,"with_payload":true,"with_vector":true}"#) } - @Test("A jump walks ids only and keeps every cursor it passes, so a nearby jump starts from one") - func jump() async throws { + @Test("An older node walks ids from the start every time, so no cached position can go stale") + func olderNodeWalksIDs() async throws { let transport = scrollingTransport(count: 25_000) let client = makeClient(transport: transport) - let cache = QdrantPagingCache() - - let page = try await client.scrollPage(collection: "docs", filter: nil, offset: 20_000, limit: 10, cache: cache) + let page = try await client.pointPage(collection: "docs", filter: nil, offset: 20_000, limit: 10) #expect(page.compactMap(\.idText) == (20_001...20_010).map(String.init)) let walks = transport.scrolls.dropLast().map(bodyObject) #expect(walks.map { $0["limit"] as? Int } == [10_000, 10_000]) #expect(walks.allSatisfy { $0["with_payload"] as? Bool == false && $0["with_vector"] as? Bool == false }) - - let before = transport.scrolls.count - let nearby = try await client.scrollPage(collection: "docs", filter: nil, offset: 15_000, limit: 10, cache: cache) - #expect(nearby.first?.idText == "15001") - #expect(bodyObject(transport.scrolls[before])["offset"] as? Int == 10_001) - #expect(transport.scrolls.count == before + 2) - } - - @Test("When a re-read boundary moves, the cursors past it are dropped instead of starting pages at the wrong row") - func movedBoundaryDropsLaterCursors() { - let cache = QdrantPagingCache() - cache.storeCursor(NSNumber(value: 101), at: 100, key: "k") - cache.storeCursor(NSNumber(value: 201), at: 200, key: "k") - cache.storeCursor(NSNumber(value: 101), at: 100, key: "k") - #expect(cache.nearestCursor(at: 200, key: "k").offset == 200) - cache.storeCursor(NSNumber(value: 102), at: 100, key: "k") - let nearest = cache.nearestCursor(at: 200, key: "k") - #expect(nearest.offset == 100) - #expect((nearest.cursor as? NSNumber)?.intValue == 102) - } - - @Test("A changed count of the filtered list drops its cursors, so a page after another client's change starts at the right row") - func countDropsCursors() { - let cache = QdrantPagingCache() - cache.revalidate(key: "docs\u{0}f", count: 400) - cache.storeCursor(NSNumber(value: 101), at: 100, key: "docs\u{0}f") - cache.storeCursor(NSNumber(value: 7), at: 100, key: "docs\u{0}g") - cache.revalidate(key: "docs\u{0}f", count: 400) - #expect(cache.nearestCursor(at: 100, key: "docs\u{0}f").offset == 100) - cache.revalidate(key: "docs\u{0}f", count: 399) - #expect(cache.nearestCursor(at: 100, key: "docs\u{0}f").offset == 0) - #expect(cache.nearestCursor(at: 100, key: "docs\u{0}g").offset == 100) + _ = try await client.pointPage(collection: "docs", filter: nil, offset: 20_000, limit: 10) + #expect(transport.scrolls.count == 6) } @Test("An offset past the end answers an empty page") func pastEnd() async throws { let client = makeClient(transport: scrollingTransport(count: 5)) - let page = try await client.scrollPage(collection: "docs", filter: nil, offset: 10, limit: 10, cache: QdrantPagingCache()) + let page = try await client.pointPage(collection: "docs", filter: nil, offset: 10, limit: 10) #expect(page.isEmpty) } diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPlugin.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPlugin.swift index 600a19f254..7b6a90473d 100644 --- a/apple/Plugins/QdrantDriverPlugin/QdrantPlugin.swift +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPlugin.swift @@ -2,7 +2,7 @@ import Foundation import TableProPluginKit import TableProQdrantCore -final class QdrantPlugin: NSObject, TableProPlugin, DriverPlugin { +final class QdrantPlugin: NSObject, TableProPlugin, DriverPlugin, PluginDefaultSortProvider { static let pluginName = "Qdrant Driver" static let pluginVersion = "1.0.0" static let pluginDescription = "Qdrant support over the REST API" @@ -50,6 +50,11 @@ final class QdrantPlugin: NSObject, TableProPlugin, DriverPlugin { static var statementCompletions: [CompletionEntry] { qdrantCompletions } + // Points already come in id order, and Qdrant cannot list them by id descending. + func defaultSortHint(forTable table: String) -> DefaultSortHint { + .suppress + } + func createDriver(config: DriverConnectionConfig) -> any PluginDatabaseDriver { QdrantPluginDriver(config: config) } diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift index a8dbd15218..e8dfe492bd 100644 --- a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift @@ -41,8 +41,8 @@ extension QdrantPluginDriver { let points: [QdrantPoint] switch try QdrantSortPlan.resolve(request.sorts) { case .byID: - points = try await client.scrollPage( - collection: request.collection, filter: filter, offset: request.offset, limit: request.limit, cache: paging + points = try await client.pointPage( + collection: request.collection, filter: filter, offset: request.offset, limit: request.limit ) case .byPayload(let column, let key, let ascending): points = try await client.sortedPage( diff --git a/apple/scripts/probes/qdrant-probe.swift b/apple/scripts/probes/qdrant-probe.swift index 8f4b6b97bc..daebc2be68 100644 --- a/apple/scripts/probes/qdrant-probe.swift +++ b/apple/scripts/probes/qdrant-probe.swift @@ -74,15 +74,15 @@ check(!columns.contains { $0.name == "meta.source" } && collection.indexes.conta check(columns.contains { $0.name == "vector" && $0.type == "vector(2)" } && columns.contains { $0.name == "vector.sp" }, "vector columns typed") let cache = QdrantPagingCache() -let p0 = try await client.scrollPage(collection: name, filter: nil, offset: 0, limit: 100, cache: cache) -let p1 = try await client.scrollPage(collection: name, filter: nil, offset: 100, limit: 100, cache: cache) -let p2 = try await client.scrollPage(collection: name, filter: nil, offset: 200, limit: 100, cache: cache) +let p0 = try await client.pointPage(collection: name, filter: nil, offset: 0, limit: 100) +let p1 = try await client.pointPage(collection: name, filter: nil, offset: 100, limit: 100) +let p2 = try await client.pointPage(collection: name, filter: nil, offset: 200, limit: 100) check(p0.first?.idText == "1" && p1.first?.idText == "101" && p2.count == 50 && p2.last?.idText == "250", "paging forward") -let jump = try await client.scrollPage(collection: name, filter: nil, offset: 170, limit: 10, cache: QdrantPagingCache()) +let jump = try await client.pointPage(collection: name, filter: nil, offset: 170, limit: 10) check(jump.map { $0.idText ?? "" } == (171...180).map(String.init), "jump walks to offset") -check(try await client.scrollPage(collection: name, filter: nil, offset: 400, limit: 10, cache: cache).isEmpty, "past end") +check(try await client.pointPage(collection: name, filter: nil, offset: 400, limit: 10).isEmpty, "past end") _ = try await run(#"POST \#(base)/points/delete?wait=true {"points":[50]}"#) -let shifted = try await client.scrollPage(collection: name, filter: nil, offset: 100, limit: 2, cache: cache) +let shifted = try await client.pointPage(collection: name, filter: nil, offset: 100, limit: 2) check(shifted.map { $0.idText ?? "" } == ["102", "103"], "a page after another client's delete starts at the right row") _ = try await run(#"PUT \#(base)/points?wait=true {"points":[{"id":50,"vector":[50,1],"payload":{"city":"even","body":"Hello World 50","f":50.5,"n":50,"meta":{"source":"doc50.pdf"},"a.b":"dotted","when":"2024-01-01T00:50:00Z"}}]}"#) @@ -108,10 +108,9 @@ let searchNumbers = [ ] check(try await filtered(searchNumbers, "OR") == 125, "search all rows runs on number and vector columns") let evenFilter = try QdrantFilterBuilder.filter([QdrantFilterSpec(column: "city", op: "=", value: "even")], logicMode: "AND", collection: collection) -let evenCache = QdrantPagingCache() -let evenBefore = try await client.scrollPage(collection: name, filter: evenFilter, offset: 10, limit: 2, cache: evenCache) +let evenBefore = try await client.pointPage(collection: name, filter: evenFilter, offset: 10, limit: 2) _ = try await run(#"POST \#(base)/points/payload?wait=true {"payload":{"city":"odd"},"points":[2]}"#) -let evenAfter = try await client.scrollPage(collection: name, filter: evenFilter, offset: 10, limit: 2, cache: evenCache) +let evenAfter = try await client.pointPage(collection: name, filter: evenFilter, offset: 10, limit: 2) _ = try await run(#"POST \#(base)/points/payload?wait=true {"payload":{"city":"even"},"points":[2]}"#) check(evenBefore.map { $0.idText ?? "" } == ["22", "24"] && evenAfter.map { $0.idText ?? "" } == ["24", "26"], "a filtered page follows another client's payload change") check(try await filtered([QdrantFilterSpec(column: "n", op: "IN", value: "1,2,300")]) == 2, "match any") From 882a48ac82af9fdf632b574bfea2007290b5e355 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 23:23:01 +0700 Subject: [PATCH 16/20] fix(plugin-qdrant): check an insert landed, filter off-type values under an index --- .../TableProQdrantCore/QdrantConsole.swift | 2 +- .../TableProQdrantCore/QdrantRowWrites.swift | 35 +++++++++++++-- .../TableProQdrantCore/QdrantSchema.swift | 45 ++++++++++++++++--- .../QdrantRowWriteTests.swift | 14 ++++++ .../QdrantSchemaTests.swift | 2 + .../QdrantPluginDriver+Execution.swift | 16 +++++++ .../QdrantPluginDriver+Metadata.swift | 2 +- .../Execution/ExternalStatementGate.swift | 2 +- ...rnalStatementGatePrivateRequestTests.swift | 1 + apple/scripts/probes/qdrant-probe.swift | 12 +++++ 10 files changed, 118 insertions(+), 13 deletions(-) diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConsole.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConsole.swift index 6c0be982a8..9b2880f24a 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConsole.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConsole.swift @@ -165,6 +165,6 @@ public enum QdrantOperations { } public static func isTagged(_ query: String) -> Bool { - query.hasPrefix(QdrantBrowseQuery.tag) || query.hasPrefix(exportTag) + query.hasPrefix(QdrantBrowseQuery.tag) || query.hasPrefix(exportTag) || query.hasPrefix(QdrantInsertCheck.tag) } } diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift index 3998db5a77..a331933e59 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift @@ -37,6 +37,33 @@ public struct QdrantRowWrite: Sendable, Equatable { // A read that must find no point before the write runs: an insert that leaves a taken id alone // still answers success, so without it the save would report a row it never added. public let mustFindNothing: QdrantConsoleRequest? + // Run right after the write: a row back means the point under the id is not the one written, which + // happens when another client takes the id between the lookup and the write. + public let landedCheck: String? +} + +public enum QdrantInsertCheck { + public static let tag = "QDRANT_INSERT_CHECK:" + + public static func encode(collection: String, id: Any, payload: [String: Any]) -> String? { + guard let data = try? QdrantJSON.data(["collection": collection, "id": id, "payload": payload]) else { return nil } + return tag + data.base64EncodedString() + } + + public static func decode(_ text: String) -> (collection: String, id: Any, payload: [String: Any])? { + guard text.hasPrefix(tag), + let data = Data(base64Encoded: String(text.dropFirst(tag.count))), + let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let collection = object["collection"] as? String, let id = object["id"], + let payload = object["payload"] as? [String: Any] + else { return nil } + return (collection, id, payload) + } + + public static func landed(_ point: QdrantPoint?, payload: [String: Any]) -> Bool { + guard let point else { return false } + return (try? QdrantJSON.text(point.payload)) == (try? QdrantJSON.text(payload)) + } } public enum QdrantServerVersion { @@ -84,7 +111,7 @@ public enum QdrantRowWriter { serverVersion ?? "?" )) } - writes += try self.writes(for: change, points: points, typeName: typeName, newID: newID) + writes += try self.writes(for: change, collection: collection, points: points, typeName: typeName, newID: newID) } catch let error as QdrantValueError { throw QdrantRowRefusal(rowIndex: change.rowIndex, reason: error.errorDescription ?? "") } catch let reason as Reason { @@ -100,6 +127,7 @@ public enum QdrantRowWriter { private static func writes( for change: QdrantRowChange, + collection: String, points: String, typeName: (String) -> String?, newID: () -> String @@ -118,7 +146,8 @@ public enum QdrantRowWriter { return [QdrantRowWrite( request: try request("PUT", "\(points)?wait=true", ["points": [point], "update_filter": Self.matchesNoPoint]), rowIndex: change.rowIndex, - mustFindNothing: lookup + mustFindNothing: lookup, + landedCheck: QdrantInsertCheck.encode(collection: collection, id: id, payload: values.payload) )] case .update: let id = try pointID(change.id) @@ -189,7 +218,7 @@ public enum QdrantRowWriter { } private static func write(_ method: String, _ path: String, _ body: [String: Any], _ change: QdrantRowChange) throws -> QdrantRowWrite { - QdrantRowWrite(request: try request(method, path, body), rowIndex: change.rowIndex, mustFindNothing: nil) + QdrantRowWrite(request: try request(method, path, body), rowIndex: change.rowIndex, mustFindNothing: nil, landedCheck: nil) } private static func request(_ method: String, _ path: String, _ body: [String: Any]) throws -> QdrantConsoleRequest { diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift index d21cdfcb2c..f5d84ef033 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift @@ -55,6 +55,11 @@ public enum QdrantFieldType: String, Sendable, Equatable, CaseIterable { } } + static func elementType(of value: Any?) -> QdrantFieldType? { + guard let items = value as? [Any] else { return nil } + return items.compactMap(observed).reduce(nil) { merged, type in merged.map { $0.merged(with: type) } ?? type } + } + func stores(_ stored: QdrantFieldType) -> Bool { switch self { case .keyword, .text, .uuid, .datetime, .string: return stored == .string @@ -129,22 +134,39 @@ public struct QdrantPayloadField: Sendable, Equatable { public let isIndexed: Bool // False while every sampled value was null, so the first real value sets the type outright. public let hasTypedValue: Bool - // What the sampled points actually store. An index does not constrain it: a keyword index - // still lets a point hold 42 or a list. - public let storedType: QdrantFieldType? + // What an indexed field's points actually store, which the index does not constrain: a keyword + // index still lets a point hold 42 or a list. Lists are indexed element by element. + public let scalarType: QdrantFieldType? + public let elementType: QdrantFieldType? + public let holdsLists: Bool public init( name: String, type: QdrantFieldType, isIndexed: Bool, hasTypedValue: Bool = true, - storedType: QdrantFieldType? = nil + scalarType: QdrantFieldType? = nil, + elementType: QdrantFieldType? = nil, + holdsLists: Bool = false ) { self.name = name self.type = type self.isIndexed = isIndexed self.hasTypedValue = hasTypedValue - self.storedType = storedType + self.scalarType = scalarType + self.elementType = elementType + self.holdsLists = holdsLists + } + + public var storedType: QdrantFieldType? { + guard holdsLists else { return scalarType } + return scalarType == nil ? .array : .json + } + + // Whether the index covers every value seen, so a filter typed by it finds them all. + public var fitsIndex: Bool { + guard isIndexed else { return false } + return (scalarType.map(type.stores) ?? true) && (elementType.map(type.stores) ?? true) } // What a cell holds and is written back as, so a value saves as the type it was stored with. @@ -190,8 +212,11 @@ public struct QdrantCollection: Sendable, Equatable { } // Only an index states a field's type; a sample guesses it from 100 points. + // An index does not constrain what points store, so it types a filter only while every value seen + // fits it; a keyword index holding 42 in some point filters like a field nobody indexed. public func indexType(ofColumn column: String) -> QdrantFieldType? { - field(ofColumn: column).flatMap { $0.isIndexed ? $0.type : nil } + guard let field = field(ofColumn: column), field.fitsIndex else { return nil } + return field.type } public func cellTypeName(ofColumn column: String) -> String? { @@ -224,9 +249,15 @@ public struct QdrantCollection: Sendable, Equatable { let field = fields[index] guard let observed else { continue } if field.isIndexed { + let isList = observed == .array + let elements = isList ? QdrantFieldType.elementType(of: point.payload[key]) : nil fields[index] = QdrantPayloadField( name: key, type: field.type, isIndexed: true, - storedType: field.storedType.map { $0.merged(with: observed) } ?? observed + scalarType: isList ? field.scalarType : field.scalarType.map { $0.merged(with: observed) } ?? observed, + elementType: [field.elementType, elements].compactMap { $0 }.reduce(nil) { merged, next in + merged.map { $0.merged(with: next) } ?? next + }, + holdsLists: field.holdsLists || isList ) continue } diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift index f9a5b962cb..b4b7ff3d73 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift @@ -136,6 +136,20 @@ struct QdrantRowWriteTests { + #"{"points":[{"id":"00000000-0000-0000-0000-000000000001","payload":{"active":true,"city":"Hanoi"},"vector":{"":[1,2]}}],"update_filter":{"must":[{"has_id":[]}]}}"#) #expect(write.mustFindNothing?.text == "POST /collections/docs/points\n" + #"{"ids":["00000000-0000-0000-0000-000000000001"],"with_payload":false,"with_vector":false}"#) + let check = try #require(write.landedCheck.flatMap(QdrantInsertCheck.decode)) + #expect(check.collection == "docs") + #expect(check.id as? String == "00000000-0000-0000-0000-000000000001") + #expect(canonical(check.payload) == #"{"active":true,"city":"Hanoi"}"#) + } + + @Test("The landed check passes only for the point the insert wrote") + func landedCheck() throws { + let written: [String: Any] = ["city": "Hanoi", "count": 3] + let same = QdrantPoint(id: 1, payload: ["count": 3, "city": "Hanoi"], vector: nil, score: nil) + let other = QdrantPoint(id: 1, payload: ["city": "Paris"], vector: nil, score: nil) + #expect(QdrantInsertCheck.landed(same, payload: written)) + #expect(!QdrantInsertCheck.landed(other, payload: written)) + #expect(!QdrantInsertCheck.landed(nil, payload: written)) } @Test("Typed values: text stays text, JSON columns parse, and malformed structures are refused") diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift index 988348532e..230d957773 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift @@ -87,6 +87,8 @@ struct QdrantSchemaTests { #expect(types["code"] == "integer") #expect(types["price"] == "float") #expect(types["fresh"] == "integer") + #expect(collection.indexType(ofColumn: "code") == nil) + #expect(collection.indexType(ofColumn: "city") == .keyword) #expect(collection.indexType(ofColumn: "tags") == .keyword) let condition = try QdrantFilterBuilder.condition( for: QdrantFilterSpec(column: "tags", op: "=", value: "42"), collection: collection diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift index e8dfe492bd..13f8fecc77 100644 --- a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift @@ -23,6 +23,22 @@ extension QdrantPluginDriver { if let request = QdrantBrowseQuery.parse(trimmed) { return try await executeBrowse(request, client: client, started: started) } + if let check = QdrantInsertCheck.decode(trimmed) { + let response = try await client.execute(QdrantConsoleRequest( + method: "POST", + path: "/collections/\(QdrantPathEncoding.segment(check.collection))/points", + body: try QdrantJSON.text(["ids": [check.id], "with_payload": true, "with_vector": false]) + )) + let point = QdrantPoint.list(fromResponse: response.json)?.first + let missed = !QdrantInsertCheck.landed(point, payload: check.payload) + return PluginQueryResult( + columns: [QdrantSchema.idColumn], + columnTypeNames: ["id"], + rows: missed ? [[QdrantJSON.displayText(check.id).map(PluginCellValue.text) ?? .null]] : [], + rowsAffected: 0, + executionTime: Date().timeIntervalSince(started) + ) + } if let request = QdrantConsoleParser.parse(trimmed) { return try await executeConsole(request, client: client, started: started) } diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift index 1a5266da47..23d12b02a8 100644 --- a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift @@ -149,7 +149,7 @@ extension QdrantPluginDriver { statement: write.request.text, rowIndices: [write.rowIndex], lookup: write.mustFindNothing.map { - PluginRowLookup(query: $0.text, parameters: [], rowCount: 0, pinsRows: false) + PluginRowLookup(query: $0.text, parameters: [], rowCount: 0, pinsRows: false, check: write.landedCheck) } ) } diff --git a/apple/TablePro/Core/Services/Execution/ExternalStatementGate.swift b/apple/TablePro/Core/Services/Execution/ExternalStatementGate.swift index a0e68187e3..7990c163f8 100644 --- a/apple/TablePro/Core/Services/Execution/ExternalStatementGate.swift +++ b/apple/TablePro/Core/Services/Execution/ExternalStatementGate.swift @@ -177,7 +177,7 @@ internal enum ExternalStatementGate { "ETCD_COUNT:", "ETCD_RANGE:", "ETCD_REFUSAL:", "FIREBASE_BROWSE:", "MEMCACHED_BROWSE:", "MEMCACHED_REFUSAL:", - "QDRANT_BROWSE:", "QDRANT_EXPORT:", + "QDRANT_BROWSE:", "QDRANT_EXPORT:", "QDRANT_INSERT_CHECK:", "SPANNER_BROWSE:", "TYPESENSE_EXPORT:", "TYPESENSE_SEARCH:", "TYPESENSE_WRITE:", "WEAVIATE_EXPORT:", "WEAVIATE_SEARCH:", "WEAVIATE_WRITE:", diff --git a/apple/TableProTests/Core/Execution/ExternalStatementGatePrivateRequestTests.swift b/apple/TableProTests/Core/Execution/ExternalStatementGatePrivateRequestTests.swift index 4993e9e702..00c68f911e 100644 --- a/apple/TableProTests/Core/Execution/ExternalStatementGatePrivateRequestTests.swift +++ b/apple/TableProTests/Core/Execution/ExternalStatementGatePrivateRequestTests.swift @@ -77,6 +77,7 @@ struct ExternalStatementGatePrivateRequestTests { (.weaviate, "WEAVIATE_EXPORT:"), (.qdrant, "QDRANT_BROWSE:"), (.qdrant, "QDRANT_EXPORT:"), + (.qdrant, "QDRANT_INSERT_CHECK:"), (.memcached, "MEMCACHED_BROWSE:"), (.memcached, "MEMCACHED_REFUSAL:"), (.etcd, "ETCD_RANGE:"), diff --git a/apple/scripts/probes/qdrant-probe.swift b/apple/scripts/probes/qdrant-probe.swift index daebc2be68..8dc17e6dd7 100644 --- a/apple/scripts/probes/qdrant-probe.swift +++ b/apple/scripts/probes/qdrant-probe.swift @@ -53,6 +53,7 @@ for i in 1...250 { if i <= 100 { payload["when"] = String(format: "2024-01-01T%02d:%02d:00Z", i / 60, i % 60) } if i == 7 { payload["note"] = NSNull() } if i == 8 { payload["note"] = "" } + if i == 9 { payload["city"] = 42 } if i == 10 { payload["mixed"] = "42" } if i == 11 { payload["mixed"] = 42 } if i == 12 { payload["mixed"] = ["42"] } @@ -91,6 +92,7 @@ func filtered(_ specs: [QdrantFilterSpec], _ mode: String = "AND") async throws return try await client.count(collection: name, filter: filter) } check(try await filtered([QdrantFilterSpec(column: "city", op: "=", value: "even")]) == 125, "match keyword") +check(try await filtered([QdrantFilterSpec(column: "city", op: "=", value: "42")]) == 1, "a number stored under a keyword index is found") check(try await filtered([QdrantFilterSpec(column: "n", op: "BETWEEN", value: "10,19", secondValue: "19")]) == 10, "between") check(try await filtered([QdrantFilterSpec(column: "n", op: ">", value: "190"), QdrantFilterSpec(column: "city", op: "!=", value: "even")]) == 5, "and with not") check(try await filtered([QdrantFilterSpec(column: "n", op: "<", value: "3"), QdrantFilterSpec(column: "n", op: ">", value: "198")], "OR") == 4, "or") @@ -180,6 +182,16 @@ _ = try await run(taken[0].request.text) let untouched = QdrantPoint.list(fromResponse: try await run(#"POST \#(base)/points {"ids":[3],"with_payload":true,"with_vector":true}"#).json) check(untouched?.first?.cell(for: "city") == "odd" && untouched?.first?.cell(for: "vector") == "[3,1]", "insert leaves a taken id untouched") +let raced = try QdrantRowWriter.writes(collection: name, changes: [ + QdrantRowChange(kind: .insert, rowIndex: 0, id: nil, cells: [QdrantCell(column: "id", text: "777"), QdrantCell(column: "city", text: "ours")]) +], serverVersion: client.serverVersion, typeName: { types[$0] })[0] +check(QdrantPoint.list(fromResponse: try await run(raced.mustFindNothing?.text).json)?.isEmpty == true, "raced insert passes its lookup") +_ = try await run(#"PUT \#(base)/points?wait=true {"points":[{"id":777,"vector":[7,7],"payload":{"city":"theirs"}}]}"#) +_ = try await run(raced.request.text) +guard let racedCheck = raced.landedCheck.flatMap(QdrantInsertCheck.decode) else { fatalError("insert has no landed check") } +let held = QdrantPoint.list(fromResponse: try await run(#"POST \#(base)/points {"ids":[777],"with_payload":true}"#).json)?.first +check(!QdrantInsertCheck.landed(held, payload: racedCheck.payload), "an insert that lost a race to the same id is caught after the write") +_ = try await run(#"POST \#(base)/points/delete?wait=true {"points":[777]}"#) let after = try await run(#"POST \#(base)/points {"ids":[1,2,999],"with_payload":true,"with_vector":true}"#) let byID = Dictionary(uniqueKeysWithValues: (QdrantPoint.list(fromResponse: after.json) ?? []).map { ($0.idText ?? "", $0) }) check(byID["1"]?.cell(for: "city") == "edited" && byID["1"]?.cell(for: "n") == "1000", "set payload typed") From 80fc271ccf10bcd53f2aff54a6fda3baf06b0c37 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 23:34:23 +0700 Subject: [PATCH 17/20] refactor(plugin-qdrant): filters match every spelling regardless of index, compare vectors in the insert check --- .../QdrantFilterBuilder.swift | 112 ++++-------------- .../TableProQdrantCore/QdrantRowWrites.swift | 52 ++++++-- .../TableProQdrantCore/QdrantSchema.swift | 27 +---- .../QdrantFilterBuilderTests.swift | 77 ++++-------- .../QdrantRowWriteTests.swift | 14 ++- .../QdrantSchemaTests.swift | 20 ---- .../QdrantPluginDriver+Execution.swift | 6 +- .../QdrantPluginDriver+Metadata.swift | 3 +- .../TablePro/Resources/Localizable.xcstrings | 8 -- apple/scripts/probes/qdrant-probe.swift | 8 +- 10 files changed, 111 insertions(+), 216 deletions(-) diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift index ad7cb6a70b..9e611fc252 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift @@ -19,8 +19,6 @@ public enum QdrantFilterError: Error, LocalizedError, Equatable, Sendable { case missingUpperBound(column: String) case notAPointID(value: String) case emptyList(column: String) - case comparisonNeedsNumberOrDate(column: String, op: String) - case wholeValueNotMatchable(column: String, type: String) case vectorNotFilterable(column: String) case keyNotAddressable(column: String) case rawFilterNotAnObject @@ -38,16 +36,6 @@ public enum QdrantFilterError: Error, LocalizedError, Equatable, Sendable { ) case .emptyList(let column): return String(format: String(localized: "IN on %@ needs at least one value."), column) - case .comparisonNeedsNumberOrDate(let column, let op): - return String( - format: String(localized: "Qdrant compares only numbers and dates with %@, and %@ holds neither."), - op, column - ) - case .wholeValueNotMatchable(let column, let type): - return String( - format: String(localized: "Qdrant cannot match a whole %@ value in %@. Use a raw filter on one of its fields."), - type, column - ) case .vectorNotFilterable(let column): return String(format: String(localized: "Qdrant cannot filter on %@."), column) case .keyNotAddressable(let column): @@ -63,20 +51,20 @@ public enum QdrantFilterError: Error, LocalizedError, Equatable, Sendable { // AND rows go under `must` and OR rows under `should`. A negated row is a nested filter with `must_not`, // which, like `$ne` in a document store, also matches a point the key is missing from. +// +// A payload index does not constrain what points store, and a sample cannot prove it does, so a value +// matches every type its text can spell: `42` finds "42", 42, 42.0 and a list holding either. The +// index still serves the branch of its own type. public enum QdrantFilterBuilder { public static let rawFilterColumn = "__RAW__" - public static func filter( - _ filters: [QdrantFilterSpec], - logicMode: String, - collection: QdrantCollection - ) throws -> [String: Any]? { + public static func filter(_ filters: [QdrantFilterSpec], logicMode: String) throws -> [String: Any]? { guard !filters.isEmpty else { return nil } - let conditions = try filters.map { try condition(for: $0, collection: collection) } + let conditions = try filters.map(condition(for:)) return [logicMode.uppercased() == "OR" ? "should" : "must": conditions] } - public static func condition(for filter: QdrantFilterSpec, collection: QdrantCollection) throws -> [String: Any] { + public static func condition(for filter: QdrantFilterSpec) throws -> [String: Any] { if filter.column == rawFilterColumn { return try raw(filter.value) } @@ -96,24 +84,24 @@ public enum QdrantFilterBuilder { guard let key = QdrantKeyPath.quoted(QdrantSchema.payloadKey(forColumn: filter.column)) else { throw QdrantFilterError.keyNotAddressable(column: filter.column) } - let field = Field(column: filter.column, key: key, type: collection.indexType(ofColumn: filter.column)) + let field = Field(key: key) switch op { case "=": - return try field.equality(filter.value) + return field.equality(filter.value) case "!=", "<>": - return negated([try field.equality(filter.value)]) + return negated([field.equality(filter.value)]) case ">", ">=", "<", "<=": - return try field.range(op: op, bounds: [rangeKey(op): filter.value]) + return field.range([rangeKey(op): filter.value]) case "BETWEEN": guard let upper = filter.secondValue, !upper.trimmingCharacters(in: .whitespaces).isEmpty else { throw QdrantFilterError.missingUpperBound(column: filter.column) } - return try field.range(op: op, bounds: ["gte": lowerBound(of: filter.value, upperBound: upper), "lte": upper]) + return field.range(["gte": lowerBound(of: filter.value, upperBound: upper), "lte": upper]) case "IN": - return try field.anyOf(list(filter.value, column: filter.column)) + return field.anyOf(try list(filter.value, column: filter.column)) case "NOT IN": - return negated(try field.anyOfConditions(list(filter.value, column: filter.column))) + return negated(try list(filter.value, column: filter.column).map(field.equality)) case "CONTAINS": return field.textMatch(filter.value) case "NOT CONTAINS": @@ -132,37 +120,9 @@ public enum QdrantFilterBuilder { } private struct Field { - let column: String let key: String - // The payload index's type. Nil for a field nobody indexed, whose points may hold `"42"`, `42` - // or a list of either, so a value there matches every type its text can spell. - let type: QdrantFieldType? - - func equality(_ value: String) throws -> [String: Any] { - guard let type else { return anyType(equalTo: value) } - switch type { - case .keyword, .text, .string, .uuid: - return match(value) - case .integer: - guard let number = QdrantValue.integer(value) else { - throw QdrantValueError.notAnInteger(column: column, value: value) - } - return match(NSNumber(value: number)) - case .bool: - guard let flag = QdrantValue.bool(value) else { - throw QdrantValueError.notABoolean(column: column, value: value) - } - return match(flag) - case .float, .datetime: - return try range(op: "=", bounds: ["gte": value, "lte": value]) - case .array, .json: - return anyType(equalTo: value) - case .geo, .object: - throw QdrantFilterError.wholeValueNotMatchable(column: column, type: type.rawValue) - } - } - private func anyType(equalTo value: String) -> [String: Any] { + func equality(_ value: String) -> [String: Any] { let trimmed = value.trimmingCharacters(in: .whitespaces) var options = [match(value)] // A range also finds 42.0, but a double rounds an integer past 2^53, so a larger one matches exactly. @@ -177,38 +137,22 @@ public enum QdrantFilterBuilder { return options.count == 1 ? options[0] : ["should": options] } - // Qdrant's `match.any` takes strings alone or integers alone; anything else is one `should`. - func anyOf(_ values: [String]) throws -> [String: Any] { - let conditions = try anyOfConditions(values) - let matched = conditions.compactMap { ($0["match"] as? [String: Any])?["value"] } - if matched.count == conditions.count { - if matched.allSatisfy({ $0 is String }) || matched.allSatisfy(isInteger) { - return ["key": key, "match": ["any": matched]] - } + // Qdrant's `match.any` takes strings alone, so it collapses only a list of plain text. + func anyOf(_ values: [String]) -> [String: Any] { + let conditions = values.map(equality) + let texts = conditions.compactMap { ($0["match"] as? [String: Any])?["value"] as? String } + if texts.count == conditions.count { + return ["key": key, "match": ["any": texts]] } return ["should": conditions] } - func anyOfConditions(_ values: [String]) throws -> [[String: Any]] { - try values.map(equality) - } - - func range(op: String, bounds: [String: String]) throws -> [String: Any] { - if let type, !type.isOrdered, type != .json, type != .array { - throw QdrantFilterError.comparisonNeedsNumberOrDate(column: column, op: op) - } + // A number bounds numbers; any other text is a date, which Qdrant parses. + func range(_ bounds: [String: String]) -> [String: Any] { var range: [String: Any] = [:] for (bound, text) in bounds { let trimmed = text.trimmingCharacters(in: .whitespaces) - if type == .datetime { - range[bound] = trimmed - } else if let number = QdrantValue.number(trimmed) { - range[bound] = NSNumber(value: number) - } else if type?.isOrdered == true { - throw QdrantValueError.notANumber(column: column, value: text) - } else { - range[bound] = trimmed - } + range[bound] = QdrantValue.number(trimmed).map { NSNumber(value: $0) as Any } ?? trimmed } return ["key": key, "range": range] } @@ -225,18 +169,12 @@ public enum QdrantFilterBuilder { } var isBlank: [String: Any] { - if let type, !type.holdsText { return isEmpty } - return ["should": [isEmpty, match("")]] + ["should": [isEmpty, match("")]] } private func match(_ value: Any) -> [String: Any] { ["key": key, "match": ["value": value]] } - - private func isInteger(_ value: Any) -> Bool { - guard let number = value as? NSNumber else { return false } - return !QdrantJSON.isBoolean(number) && !CFNumberIsFloatType(number) - } } private static func raw(_ text: String) throws -> [String: Any] { diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift index a331933e59..44bcbf0fc8 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift @@ -45,24 +45,60 @@ public struct QdrantRowWrite: Sendable, Equatable { public enum QdrantInsertCheck { public static let tag = "QDRANT_INSERT_CHECK:" - public static func encode(collection: String, id: Any, payload: [String: Any]) -> String? { - guard let data = try? QdrantJSON.data(["collection": collection, "id": id, "payload": payload]) else { return nil } + public static func encode(collection: String, id: Any, payload: [String: Any], vectors: [String: Any]) -> String? { + let spec: [String: Any] = ["collection": collection, "id": id, "payload": payload, "vectors": vectors] + guard let data = try? QdrantJSON.data(spec) else { return nil } return tag + data.base64EncodedString() } - public static func decode(_ text: String) -> (collection: String, id: Any, payload: [String: Any])? { + public static func decode(_ text: String) -> (collection: String, id: Any, payload: [String: Any], vectors: [String: Any])? { guard text.hasPrefix(tag), let data = Data(base64Encoded: String(text.dropFirst(tag.count))), let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any], let collection = object["collection"] as? String, let id = object["id"], let payload = object["payload"] as? [String: Any] else { return nil } - return (collection, id, payload) + return (collection, id, payload, object["vectors"] as? [String: Any] ?? [:]) } - public static func landed(_ point: QdrantPoint?, payload: [String: Any]) -> Bool { - guard let point else { return false } - return (try? QdrantJSON.text(point.payload)) == (try? QdrantJSON.text(payload)) + // A vector-only point has an empty payload either way, so the vectors are compared too. + public static func landed(_ point: QdrantPoint?, payload: [String: Any], vectors: [String: Any]) -> Bool { + guard let point, (try? QdrantJSON.text(point.payload)) == (try? QdrantJSON.text(payload)) else { return false } + let stored: [String: Any] + switch point.vector { + case let unnamed as [Any]: stored = ["": unnamed] + case let named as [String: Any]: stored = named + default: stored = [:] + } + guard Set(stored.keys) == Set(vectors.keys) else { return false } + return vectors.allSatisfy { name, sent in stored[name].map { sameVector(sent, $0) } ?? false } + } + + // Qdrant stores a cosine collection's dense vectors normalized, so one equal to what was sent after + // normalizing is the same vector. + static func sameVector(_ sent: Any, _ stored: Any) -> Bool { + if let sentRows = sent as? [[Any]], let storedRows = stored as? [[Any]] { + return sentRows.count == storedRows.count && zip(sentRows, storedRows).allSatisfy { sameVector($0, $1) } + } + if let sentSparse = sent as? [String: Any], let storedSparse = stored as? [String: Any] { + return numbers(sentSparse["indices"]) == numbers(storedSparse["indices"]) + && close(numbers(sentSparse["values"]), numbers(storedSparse["values"])) + } + guard let sentValues = numbers(sent), let storedValues = numbers(stored) else { return false } + if close(sentValues, storedValues) { return true } + let norm = sentValues.reduce(0) { $0 + $1 * $1 }.squareRoot() + return norm > 0 && close(sentValues.map { $0 / norm }, storedValues) + } + + private static func numbers(_ value: Any?) -> [Double]? { + guard let array = value as? [Any] else { return nil } + let numbers = array.compactMap { ($0 as? NSNumber)?.doubleValue } + return numbers.count == array.count ? numbers : nil + } + + private static func close(_ lhs: [Double]?, _ rhs: [Double]?) -> Bool { + guard let lhs, let rhs, lhs.count == rhs.count else { return false } + return zip(lhs, rhs).allSatisfy { abs($0 - $1) <= 1e-4 * max(1, abs($0)) } } } @@ -147,7 +183,7 @@ public enum QdrantRowWriter { request: try request("PUT", "\(points)?wait=true", ["points": [point], "update_filter": Self.matchesNoPoint]), rowIndex: change.rowIndex, mustFindNothing: lookup, - landedCheck: QdrantInsertCheck.encode(collection: collection, id: id, payload: values.payload) + landedCheck: QdrantInsertCheck.encode(collection: collection, id: id, payload: values.payload, vectors: values.vectors) )] case .update: let id = try pointID(change.id) diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift index f5d84ef033..bc60dd925f 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift @@ -55,11 +55,6 @@ public enum QdrantFieldType: String, Sendable, Equatable, CaseIterable { } } - static func elementType(of value: Any?) -> QdrantFieldType? { - guard let items = value as? [Any] else { return nil } - return items.compactMap(observed).reduce(nil) { merged, type in merged.map { $0.merged(with: type) } ?? type } - } - func stores(_ stored: QdrantFieldType) -> Bool { switch self { case .keyword, .text, .uuid, .datetime, .string: return stored == .string @@ -135,9 +130,8 @@ public struct QdrantPayloadField: Sendable, Equatable { // False while every sampled value was null, so the first real value sets the type outright. public let hasTypedValue: Bool // What an indexed field's points actually store, which the index does not constrain: a keyword - // index still lets a point hold 42 or a list. Lists are indexed element by element. + // index still lets a point hold 42 or a list. public let scalarType: QdrantFieldType? - public let elementType: QdrantFieldType? public let holdsLists: Bool public init( @@ -146,7 +140,6 @@ public struct QdrantPayloadField: Sendable, Equatable { isIndexed: Bool, hasTypedValue: Bool = true, scalarType: QdrantFieldType? = nil, - elementType: QdrantFieldType? = nil, holdsLists: Bool = false ) { self.name = name @@ -154,7 +147,6 @@ public struct QdrantPayloadField: Sendable, Equatable { self.isIndexed = isIndexed self.hasTypedValue = hasTypedValue self.scalarType = scalarType - self.elementType = elementType self.holdsLists = holdsLists } @@ -163,12 +155,6 @@ public struct QdrantPayloadField: Sendable, Equatable { return scalarType == nil ? .array : .json } - // Whether the index covers every value seen, so a filter typed by it finds them all. - public var fitsIndex: Bool { - guard isIndexed else { return false } - return (scalarType.map(type.stores) ?? true) && (elementType.map(type.stores) ?? true) - } - // What a cell holds and is written back as, so a value saves as the type it was stored with. // The index's name stands in only when every value seen fits it. public var cellTypeName: String { @@ -212,13 +198,6 @@ public struct QdrantCollection: Sendable, Equatable { } // Only an index states a field's type; a sample guesses it from 100 points. - // An index does not constrain what points store, so it types a filter only while every value seen - // fits it; a keyword index holding 42 in some point filters like a field nobody indexed. - public func indexType(ofColumn column: String) -> QdrantFieldType? { - guard let field = field(ofColumn: column), field.fitsIndex else { return nil } - return field.type - } - public func cellTypeName(ofColumn column: String) -> String? { field(ofColumn: column)?.cellTypeName } @@ -250,13 +229,9 @@ public struct QdrantCollection: Sendable, Equatable { guard let observed else { continue } if field.isIndexed { let isList = observed == .array - let elements = isList ? QdrantFieldType.elementType(of: point.payload[key]) : nil fields[index] = QdrantPayloadField( name: key, type: field.type, isIndexed: true, scalarType: isList ? field.scalarType : field.scalarType.map { $0.merged(with: observed) } ?? observed, - elementType: [field.elementType, elements].compactMap { $0 }.reduce(nil) { merged, next in - merged.map { $0.merged(with: next) } ?? next - }, holdsLists: field.holdsLists || isList ) continue diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift index 6a5b49dbb2..8b9a6aee54 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift @@ -4,27 +4,22 @@ import Testing struct QdrantFilterBuilderTests { private func condition(_ column: String, _ op: String, _ value: String = "", _ second: String? = nil) throws -> String { - canonical(try QdrantFilterBuilder.condition( - for: QdrantFilterSpec(column: column, op: op, value: value, secondValue: second), - collection: sampleCollection - )) + canonical(try QdrantFilterBuilder.condition(for: QdrantFilterSpec(column: column, op: op, value: value, secondValue: second))) } - @Test("An indexed field matches as its index's type") - func indexedEquality() throws { + @Test("Text matches as text alone") + func textEquality() throws { #expect(try condition("city", "=", "Berlin") == #"{"key":"\"city\"","match":{"value":"Berlin"}}"#) - #expect(try condition("rank", "=", "7") == #"{"key":"\"rank\"","match":{"value":7}}"#) - #expect(try condition("flag", "=", "1") == #"{"key":"\"flag\"","match":{"value":true}}"#) - #expect(throws: QdrantValueError.notAnInteger(column: "rank", value: "x")) { try condition("rank", "=", "x") } } - @Test("A field nobody indexed matches every type its text can spell, whatever the sample guessed") - func unindexedEquality() throws { + @Test("A value matches every type its text spells, whatever an index or a sample says the field holds") + func everySpelling() throws { #expect(try condition("count", "=", "42") == #"{"should":[{"key":"\"count\"","match":{"value":"42"}},{"key":"\"count\"","range":{"gte":42,"lte":42}}]}"#) - #expect(try condition("note", "=", "1") - == #"{"should":[{"key":"\"note\"","match":{"value":"1"}},{"key":"\"note\"","range":{"gte":1,"lte":1}},{"key":"\"note\"","match":{"value":true}}]}"#) - #expect(try condition("note", "=", "Berlin") == #"{"key":"\"note\"","match":{"value":"Berlin"}}"#) + #expect(try condition("flag", "=", "1") + == #"{"should":[{"key":"\"flag\"","match":{"value":"1"}},{"key":"\"flag\"","range":{"gte":1,"lte":1}},{"key":"\"flag\"","match":{"value":true}}]}"#) + #expect(try condition("pop", "=", "3.5") + == #"{"should":[{"key":"\"pop\"","match":{"value":"3.5"}},{"key":"\"pop\"","range":{"gte":3.5,"lte":3.5}}]}"#) #expect(try condition("count", "=", "9007199254740993") == #"{"should":[{"key":"\"count\"","match":{"value":"9007199254740993"}},{"key":"\"count\"","match":{"value":9007199254740993}}]}"#) } @@ -32,19 +27,7 @@ struct QdrantFilterBuilderTests { @Test("A key holding a dot stays one top-level key, and one holding a quote is refused") func keyQuoting() throws { #expect(try condition("a.b", "=", "x") == #"{"key":"\"a.b\"","match":{"value":"x"}}"#) - let collection = QdrantCollection(name: "docs", vectors: [], payloadFields: [ - QdrantPayloadField(name: "q\"x", type: .string, isIndexed: false) - ]) - #expect(throws: QdrantFilterError.keyNotAddressable(column: "q\"x")) { - try QdrantFilterBuilder.condition(for: QdrantFilterSpec(column: "q\"x", op: "=", value: "1"), collection: collection) - } - } - - @Test("A float and a datetime have no exact match, so equality is the closed range around the value") - func rangeEquality() throws { - #expect(try condition("pop", "=", "3.5") == #"{"key":"\"pop\"","range":{"gte":3.5,"lte":3.5}}"#) - #expect(try condition("seen", "=", "2024-01-02T00:00:00Z") - == #"{"key":"\"seen\"","range":{"gte":"2024-01-02T00:00:00Z","lte":"2024-01-02T00:00:00Z"}}"#) + #expect(throws: QdrantFilterError.keyNotAddressable(column: "q\"x")) { try condition("q\"x", "=", "1") } } @Test("Not equal nests the match under must_not") @@ -52,46 +35,31 @@ struct QdrantFilterBuilderTests { #expect(try condition("city", "!=", "Berlin") == #"{"must_not":[{"key":"\"city\"","match":{"value":"Berlin"}}]}"#) } - @Test("Comparisons and BETWEEN become a range, and text is refused") + @Test("Comparisons and BETWEEN become a range on numbers, or on dates for other text") func ranges() throws { #expect(try condition("pop", ">", "2") == #"{"key":"\"pop\"","range":{"gt":2}}"#) #expect(try condition("count", "BETWEEN", "1,5", "5") == #"{"key":"\"count\"","range":{"gte":1,"lte":5}}"#) - #expect(throws: QdrantFilterError.comparisonNeedsNumberOrDate(column: "city", op: ">")) { - try condition("city", ">", "a") - } + #expect(try condition("seen", ">=", "2024-01-02T00:00:00Z") == #"{"key":"\"seen\"","range":{"gte":"2024-01-02T00:00:00Z"}}"#) + #expect(throws: QdrantFilterError.missingUpperBound(column: "count")) { try condition("count", "BETWEEN", "1", "") } } - @Test("IN is the equality of each value: match any for strings or integers, ranges for floats") + @Test("IN collapses plain text to match any, and keeps every spelling of a number") func inLists() throws { #expect(try condition("city", "IN", "Berlin, Paris") == #"{"key":"\"city\"","match":{"any":["Berlin","Paris"]}}"#) - #expect(try condition("rank", "IN", "1,2") == #"{"key":"\"rank\"","match":{"any":[1,2]}}"#) - #expect(try condition("note", "IN", "a, b") == #"{"key":"\"note\"","match":{"any":["a","b"]}}"#) - #expect(try condition("pop", "IN", "1.5, 2.5") - == #"{"should":[{"key":"\"pop\"","range":{"gte":1.5,"lte":1.5}},{"key":"\"pop\"","range":{"gte":2.5,"lte":2.5}}]}"#) - #expect(try condition("flag", "IN", "true") - == #"{"should":[{"key":"\"flag\"","match":{"value":true}}]}"#) + #expect(try condition("rank", "IN", "1") + == #"{"should":[{"should":[{"key":"\"rank\"","match":{"value":"1"}},{"key":"\"rank\"","range":{"gte":1,"lte":1}},{"key":"\"rank\"","match":{"value":true}}]}]}"#) } @Test("NOT IN excludes every value the way != excludes one") func notInLists() throws { - #expect(try condition("rank", "NOT IN", "1,2") - == #"{"must_not":[{"key":"\"rank\"","match":{"value":1}},{"key":"\"rank\"","match":{"value":2}}]}"#) - #expect(try condition("pop", "NOT IN", "1.5") == #"{"must_not":[{"key":"\"pop\"","range":{"gte":1.5,"lte":1.5}}]}"#) - } - - @Test("A geo point cannot be matched whole") - func geoEquality() { - #expect(throws: QdrantFilterError.wholeValueNotMatchable(column: "home", type: "geo")) { - try condition("home", "IN", "1") - } + #expect(try condition("city", "NOT IN", "a, b") + == #"{"must_not":[{"key":"\"city\"","match":{"value":"a"}},{"key":"\"city\"","match":{"value":"b"}}]}"#) } @Test("CONTAINS matches words in any field, indexed or not, and a value that is not text matches nothing") func contains() throws { #expect(try condition("body", "CONTAINS", "vector") == #"{"key":"\"body\"","match":{"text":"vector"}}"#) - #expect(try condition("note", "CONTAINS", "Ber") == #"{"key":"\"note\"","match":{"text":"Ber"}}"#) #expect(try condition("rank", "CONTAINS", "1") == #"{"key":"\"rank\"","match":{"text":"1"}}"#) - #expect(try condition("count", "CONTAINS", "1") == #"{"key":"\"count\"","match":{"text":"1"}}"#) } @Test("IS NULL matches a missing key as well as null, and IS EMPTY adds the empty string") @@ -100,7 +68,6 @@ struct QdrantFilterBuilderTests { #expect(try condition("note", "IS NOT NULL") == #"{"must_not":[{"is_empty":{"key":"\"note\""}}]}"#) #expect(try condition("note", "IS EMPTY") == #"{"should":[{"is_empty":{"key":"\"note\""}},{"key":"\"note\"","match":{"value":""}}]}"#) - #expect(try condition("rank", "IS EMPTY") == #"{"is_empty":{"key":"\"rank\""}}"#) } @Test("The id column filters with has_id, keeping integer and UUID ids apart") @@ -113,7 +80,7 @@ struct QdrantFilterBuilderTests { #expect(try condition("id", "CONTAINS", "Berlin") == #"{"has_id":[]}"#) } - @Test("Vectors cannot be filtered and unsupported operators say so") + @Test("Vectors cannot be filtered, contains on one finds nothing, and unsupported operators say so") func refusals() throws { #expect(throws: QdrantFilterError.vectorNotFilterable(column: "vector")) { try condition("vector", "=", "1") } #expect(try condition("vector.sp", "CONTAINS", "a") == #"{"has_id":[]}"#) @@ -132,10 +99,10 @@ struct QdrantFilterBuilderTests { QdrantFilterSpec(column: "city", op: "=", value: "Berlin"), QdrantFilterSpec(column: "count", op: "<", value: "3") ] - let and = try #require(try QdrantFilterBuilder.filter(specs, logicMode: "AND", collection: sampleCollection)) - let or = try #require(try QdrantFilterBuilder.filter(specs, logicMode: "OR", collection: sampleCollection)) + let and = try #require(try QdrantFilterBuilder.filter(specs, logicMode: "AND")) + let or = try #require(try QdrantFilterBuilder.filter(specs, logicMode: "OR")) #expect((and["must"] as? [Any])?.count == 2) #expect((or["should"] as? [Any])?.count == 2) - #expect(try QdrantFilterBuilder.filter([], logicMode: "AND", collection: sampleCollection) == nil) + #expect(try QdrantFilterBuilder.filter([], logicMode: "AND") == nil) } } diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift index b4b7ff3d73..3a26b0c6ff 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift @@ -147,9 +147,17 @@ struct QdrantRowWriteTests { let written: [String: Any] = ["city": "Hanoi", "count": 3] let same = QdrantPoint(id: 1, payload: ["count": 3, "city": "Hanoi"], vector: nil, score: nil) let other = QdrantPoint(id: 1, payload: ["city": "Paris"], vector: nil, score: nil) - #expect(QdrantInsertCheck.landed(same, payload: written)) - #expect(!QdrantInsertCheck.landed(other, payload: written)) - #expect(!QdrantInsertCheck.landed(nil, payload: written)) + #expect(QdrantInsertCheck.landed(same, payload: written, vectors: [:])) + #expect(!QdrantInsertCheck.landed(other, payload: written, vectors: [:])) + #expect(!QdrantInsertCheck.landed(nil, payload: written, vectors: [:])) + + let sent: [String: Any] = ["": [3, 4], "sp": ["indices": [1], "values": [0.5]]] + let normalized = QdrantPoint(id: 1, payload: [:], vector: ["": [0.6, 0.8], "sp": ["indices": [1], "values": [0.5]]], score: nil) + let theirs = QdrantPoint(id: 1, payload: [:], vector: ["": [4, 3], "sp": ["indices": [1], "values": [0.5]]], score: nil) + let missing = QdrantPoint(id: 1, payload: [:], vector: ["": [3, 4]], score: nil) + #expect(QdrantInsertCheck.landed(normalized, payload: [:], vectors: sent)) + #expect(!QdrantInsertCheck.landed(theirs, payload: [:], vectors: sent)) + #expect(!QdrantInsertCheck.landed(missing, payload: [:], vectors: sent)) } @Test("Typed values: text stays text, JSON columns parse, and malformed structures are refused") diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift index 230d957773..446dec447a 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift @@ -87,13 +87,6 @@ struct QdrantSchemaTests { #expect(types["code"] == "integer") #expect(types["price"] == "float") #expect(types["fresh"] == "integer") - #expect(collection.indexType(ofColumn: "code") == nil) - #expect(collection.indexType(ofColumn: "city") == .keyword) - #expect(collection.indexType(ofColumn: "tags") == .keyword) - let condition = try QdrantFilterBuilder.condition( - for: QdrantFilterSpec(column: "tags", op: "=", value: "42"), collection: collection - ) - #expect(canonical(condition) == #"{"key":"\"tags\"","match":{"value":"42"}}"#) } @Test("A page's points widen a sampled type without adding columns the table did not open with") @@ -111,19 +104,6 @@ struct QdrantSchemaTests { #expect(types["late"] == nil) } - @Test("Only an index types a filter; a sampled field's type never decides what a value matches") - func filtersIgnoreSampledTypes() throws { - let collection = QdrantCollection.parse( - name: "docs", info: info(vectors: ["size": 2], payloadSchema: ["rank": ["data_type": "integer"]]), - sample: [try point(#"{"id":1,"payload":{"tags":["42","a"],"mixed":"42","rank":1}}"#), - try point(#"{"id":2,"payload":{"mixed":["42"]}}"#)] - ) - #expect(collection.indexType(ofColumn: "rank") == .integer) - #expect(collection.indexType(ofColumn: "tags") == nil) - #expect(collection.indexType(ofColumn: "mixed") == nil) - #expect(QdrantSchema.columns(for: collection).first { $0.name == "tags" }?.type == "array") - } - @Test("A payload key named score never stands in for the similarity score column") func scoreColumnIsThePoints() throws { let hit = try point(#"{"id":1,"score":0.75,"payload":{"score":"high"}}"#) diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift index 13f8fecc77..4bf1ae92c0 100644 --- a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift @@ -27,10 +27,10 @@ extension QdrantPluginDriver { let response = try await client.execute(QdrantConsoleRequest( method: "POST", path: "/collections/\(QdrantPathEncoding.segment(check.collection))/points", - body: try QdrantJSON.text(["ids": [check.id], "with_payload": true, "with_vector": false]) + body: try QdrantJSON.text(["ids": [check.id], "with_payload": true, "with_vector": true]) )) let point = QdrantPoint.list(fromResponse: response.json)?.first - let missed = !QdrantInsertCheck.landed(point, payload: check.payload) + let missed = !QdrantInsertCheck.landed(point, payload: check.payload, vectors: check.vectors) return PluginQueryResult( columns: [QdrantSchema.idColumn], columnTypeNames: ["id"], @@ -53,7 +53,7 @@ extension QdrantPluginDriver { started: Date ) async throws -> PluginQueryResult { let collection = try await collection(request.collection) - let filter = try QdrantFilterBuilder.filter(request.filters, logicMode: request.logicMode, collection: collection) + let filter = try QdrantFilterBuilder.filter(request.filters, logicMode: request.logicMode) let points: [QdrantPoint] switch try QdrantSortPlan.resolve(request.sorts) { case .byID: diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift index 23d12b02a8..ca822ad6e3 100644 --- a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift @@ -65,8 +65,7 @@ extension QdrantPluginDriver { } func fetchFilteredRowCount(table: String, queryFilters: [PluginQueryFilter], logicMode: String) async throws -> Int? { - let collection = try await collection(table) - let filter = try QdrantFilterBuilder.filter(specs(queryFilters), logicMode: logicMode, collection: collection) + let filter = try QdrantFilterBuilder.filter(specs(queryFilters), logicMode: logicMode) return try await requireClient().count(collection: table, filter: filter) } diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index fdac408d4b..78a9f13704 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -139366,18 +139366,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, - "Qdrant cannot match a whole %@ value in %@. Use a raw filter on one of its fields." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, "Qdrant cannot sort by %@." : { "extractionState" : "manual", "generatesSymbol" : false }, - "Qdrant compares only numbers and dates with %@, and %@ holds neither." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, "Qdrant does not support views." : { "extractionState" : "manual", "generatesSymbol" : false diff --git a/apple/scripts/probes/qdrant-probe.swift b/apple/scripts/probes/qdrant-probe.swift index 8dc17e6dd7..1e0c7b0a23 100644 --- a/apple/scripts/probes/qdrant-probe.swift +++ b/apple/scripts/probes/qdrant-probe.swift @@ -88,7 +88,7 @@ check(shifted.map { $0.idText ?? "" } == ["102", "103"], "a page after another c _ = try await run(#"PUT \#(base)/points?wait=true {"points":[{"id":50,"vector":[50,1],"payload":{"city":"even","body":"Hello World 50","f":50.5,"n":50,"meta":{"source":"doc50.pdf"},"a.b":"dotted","when":"2024-01-01T00:50:00Z"}}]}"#) func filtered(_ specs: [QdrantFilterSpec], _ mode: String = "AND") async throws -> Int { - let filter = try QdrantFilterBuilder.filter(specs, logicMode: mode, collection: collection) + let filter = try QdrantFilterBuilder.filter(specs, logicMode: mode) return try await client.count(collection: name, filter: filter) } check(try await filtered([QdrantFilterSpec(column: "city", op: "=", value: "even")]) == 125, "match keyword") @@ -109,7 +109,7 @@ let searchNumbers = [ QdrantFilterSpec(column: "city", op: "CONTAINS", value: "even") ] check(try await filtered(searchNumbers, "OR") == 125, "search all rows runs on number and vector columns") -let evenFilter = try QdrantFilterBuilder.filter([QdrantFilterSpec(column: "city", op: "=", value: "even")], logicMode: "AND", collection: collection) +let evenFilter = try QdrantFilterBuilder.filter([QdrantFilterSpec(column: "city", op: "=", value: "even")], logicMode: "AND") let evenBefore = try await client.pointPage(collection: name, filter: evenFilter, offset: 10, limit: 2) _ = try await run(#"POST \#(base)/points/payload?wait=true {"payload":{"city":"odd"},"points":[2]}"#) let evenAfter = try await client.pointPage(collection: name, filter: evenFilter, offset: 10, limit: 2) @@ -189,8 +189,8 @@ check(QdrantPoint.list(fromResponse: try await run(raced.mustFindNothing?.text). _ = try await run(#"PUT \#(base)/points?wait=true {"points":[{"id":777,"vector":[7,7],"payload":{"city":"theirs"}}]}"#) _ = try await run(raced.request.text) guard let racedCheck = raced.landedCheck.flatMap(QdrantInsertCheck.decode) else { fatalError("insert has no landed check") } -let held = QdrantPoint.list(fromResponse: try await run(#"POST \#(base)/points {"ids":[777],"with_payload":true}"#).json)?.first -check(!QdrantInsertCheck.landed(held, payload: racedCheck.payload), "an insert that lost a race to the same id is caught after the write") +let held = QdrantPoint.list(fromResponse: try await run(#"POST \#(base)/points {"ids":[777],"with_payload":true,"with_vector":true}"#).json)?.first +check(!QdrantInsertCheck.landed(held, payload: racedCheck.payload, vectors: racedCheck.vectors), "an insert that lost a race to the same id is caught after the write") _ = try await run(#"POST \#(base)/points/delete?wait=true {"points":[777]}"#) let after = try await run(#"POST \#(base)/points {"ids":[1,2,999],"with_payload":true,"with_vector":true}"#) let byID = Dictionary(uniqueKeysWithValues: (QdrantPoint.list(fromResponse: after.json) ?? []).map { ($0.idText ?? "", $0) }) From 19528846455df63793d87356db72f296887dc672 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 23:43:00 +0700 Subject: [PATCH 18/20] fix(plugin-qdrant): check an insert by payload and vector names, not vector values --- .../TableProQdrantCore/QdrantRowWrites.swift | 42 ++++--------------- .../QdrantRowWriteTests.swift | 8 ++-- 2 files changed, 11 insertions(+), 39 deletions(-) diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift index 44bcbf0fc8..ce87b6ac90 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift @@ -61,44 +61,18 @@ public enum QdrantInsertCheck { return (collection, id, payload, object["vectors"] as? [String: Any] ?? [:]) } - // A vector-only point has an empty payload either way, so the vectors are compared too. + // The payload and which vectors the point holds. Vector values are not compared: Qdrant sorts a + // sparse vector's indices, stores float16 and uint8 vectors rounded and cosine vectors normalized, so + // a vector written exactly reads back different. public static func landed(_ point: QdrantPoint?, payload: [String: Any], vectors: [String: Any]) -> Bool { guard let point, (try? QdrantJSON.text(point.payload)) == (try? QdrantJSON.text(payload)) else { return false } - let stored: [String: Any] + let names: Set switch point.vector { - case let unnamed as [Any]: stored = ["": unnamed] - case let named as [String: Any]: stored = named - default: stored = [:] + case is [Any]: names = [""] + case let named as [String: Any]: names = Set(named.keys) + default: names = [] } - guard Set(stored.keys) == Set(vectors.keys) else { return false } - return vectors.allSatisfy { name, sent in stored[name].map { sameVector(sent, $0) } ?? false } - } - - // Qdrant stores a cosine collection's dense vectors normalized, so one equal to what was sent after - // normalizing is the same vector. - static func sameVector(_ sent: Any, _ stored: Any) -> Bool { - if let sentRows = sent as? [[Any]], let storedRows = stored as? [[Any]] { - return sentRows.count == storedRows.count && zip(sentRows, storedRows).allSatisfy { sameVector($0, $1) } - } - if let sentSparse = sent as? [String: Any], let storedSparse = stored as? [String: Any] { - return numbers(sentSparse["indices"]) == numbers(storedSparse["indices"]) - && close(numbers(sentSparse["values"]), numbers(storedSparse["values"])) - } - guard let sentValues = numbers(sent), let storedValues = numbers(stored) else { return false } - if close(sentValues, storedValues) { return true } - let norm = sentValues.reduce(0) { $0 + $1 * $1 }.squareRoot() - return norm > 0 && close(sentValues.map { $0 / norm }, storedValues) - } - - private static func numbers(_ value: Any?) -> [Double]? { - guard let array = value as? [Any] else { return nil } - let numbers = array.compactMap { ($0 as? NSNumber)?.doubleValue } - return numbers.count == array.count ? numbers : nil - } - - private static func close(_ lhs: [Double]?, _ rhs: [Double]?) -> Bool { - guard let lhs, let rhs, lhs.count == rhs.count else { return false } - return zip(lhs, rhs).allSatisfy { abs($0 - $1) <= 1e-4 * max(1, abs($0)) } + return names == Set(vectors.keys) } } diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift index 3a26b0c6ff..cd0bbbdaab 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift @@ -151,12 +151,10 @@ struct QdrantRowWriteTests { #expect(!QdrantInsertCheck.landed(other, payload: written, vectors: [:])) #expect(!QdrantInsertCheck.landed(nil, payload: written, vectors: [:])) - let sent: [String: Any] = ["": [3, 4], "sp": ["indices": [1], "values": [0.5]]] - let normalized = QdrantPoint(id: 1, payload: [:], vector: ["": [0.6, 0.8], "sp": ["indices": [1], "values": [0.5]]], score: nil) - let theirs = QdrantPoint(id: 1, payload: [:], vector: ["": [4, 3], "sp": ["indices": [1], "values": [0.5]]], score: nil) + let sent: [String: Any] = ["": [3, 4], "sp": ["indices": [5, 1], "values": [0.5, 0.2]]] + let stored = QdrantPoint(id: 1, payload: [:], vector: ["": [0.6, 0.8], "sp": ["indices": [1, 5], "values": [0.2, 0.5]]], score: nil) let missing = QdrantPoint(id: 1, payload: [:], vector: ["": [3, 4]], score: nil) - #expect(QdrantInsertCheck.landed(normalized, payload: [:], vectors: sent)) - #expect(!QdrantInsertCheck.landed(theirs, payload: [:], vectors: sent)) + #expect(QdrantInsertCheck.landed(stored, payload: [:], vectors: sent)) #expect(!QdrantInsertCheck.landed(missing, payload: [:], vectors: sent)) } From 20583740e491bda3d0cfc2dc9b7fb4f37cc0457a Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 23:52:15 +0700 Subject: [PATCH 19/20] fix(plugin-qdrant): refuse range bounds a float cannot hold, reserve the raw-filter name --- .../QdrantFilterBuilder.swift | 21 ++++++++++++++----- .../TableProQdrantCore/QdrantSchema.swift | 7 ++++--- .../QdrantFilterBuilderTests.swift | 3 +++ .../QdrantSchemaTests.swift | 2 +- .../TablePro/Resources/Localizable.xcstrings | 4 ++++ 5 files changed, 28 insertions(+), 9 deletions(-) diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift index 9e611fc252..8b20a24cde 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift @@ -21,6 +21,7 @@ public enum QdrantFilterError: Error, LocalizedError, Equatable, Sendable { case emptyList(column: String) case vectorNotFilterable(column: String) case keyNotAddressable(column: String) + case boundNotExact(column: String, value: String) case rawFilterNotAnObject public var errorDescription: String? { @@ -43,6 +44,11 @@ public enum QdrantFilterError: Error, LocalizedError, Equatable, Sendable { format: String(localized: "Qdrant cannot filter on %@, because a key holding a double quote has no path it accepts."), column ) + case .boundNotExact(let column, let value): + return String( + format: String(localized: "Qdrant compares %@ as a 64-bit float, which cannot hold %@ exactly."), + column, value + ) case .rawFilterNotAnObject: return String(localized: "A raw filter for Qdrant is a JSON condition, such as {\"key\": \"city\", \"match\": {\"value\": \"Berlin\"}}.") } @@ -84,7 +90,7 @@ public enum QdrantFilterBuilder { guard let key = QdrantKeyPath.quoted(QdrantSchema.payloadKey(forColumn: filter.column)) else { throw QdrantFilterError.keyNotAddressable(column: filter.column) } - let field = Field(key: key) + let field = Field(column: filter.column, key: key) switch op { case "=": @@ -92,12 +98,12 @@ public enum QdrantFilterBuilder { case "!=", "<>": return negated([field.equality(filter.value)]) case ">", ">=", "<", "<=": - return field.range([rangeKey(op): filter.value]) + return try field.range([rangeKey(op): filter.value]) case "BETWEEN": guard let upper = filter.secondValue, !upper.trimmingCharacters(in: .whitespaces).isEmpty else { throw QdrantFilterError.missingUpperBound(column: filter.column) } - return field.range(["gte": lowerBound(of: filter.value, upperBound: upper), "lte": upper]) + return try field.range(["gte": lowerBound(of: filter.value, upperBound: upper), "lte": upper]) case "IN": return field.anyOf(try list(filter.value, column: filter.column)) case "NOT IN": @@ -120,6 +126,7 @@ public enum QdrantFilterBuilder { } private struct Field { + let column: String let key: String func equality(_ value: String) -> [String: Any] { @@ -147,11 +154,15 @@ public enum QdrantFilterBuilder { return ["should": conditions] } - // A number bounds numbers; any other text is a date, which Qdrant parses. - func range(_ bounds: [String: String]) -> [String: Any] { + // A number bounds numbers; any other text is a date, which Qdrant parses. Qdrant reads a bound as + // a 64-bit float, so an integer past 2^53 would be rounded to a different cutoff. + func range(_ bounds: [String: String]) throws -> [String: Any] { var range: [String: Any] = [:] for (bound, text) in bounds { let trimmed = text.trimmingCharacters(in: .whitespaces) + if let integer = QdrantValue.integer(trimmed), integer.magnitude > 1 << 53 { + throw QdrantFilterError.boundNotExact(column: column, value: trimmed) + } range[bound] = QdrantValue.number(trimmed).map { NSNumber(value: $0) as Any } ?? trimmed } return ["key": key, "range": range] diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift index bc60dd925f..0894a0ae07 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift @@ -320,11 +320,12 @@ public enum QdrantSchema { return String(column.dropFirst(vectorColumn.count + 1)) } - // A payload key named `id`, `score` or `vector...` would take a point column's name, so it shows as - // `payload.`. A key already starting with `payload.` is prefixed too, so every column maps back - // to exactly one key. + // A payload key named `id`, `score`, `vector...` or the raw filter's `__RAW__` would take a name the + // grid gives another meaning, so it shows as `payload.`. A key already starting with `payload.` + // is prefixed too, so every column maps back to exactly one key. public static func isReserved(_ key: String) -> Bool { key == idColumn || key == scoreColumn || isVectorColumn(key) || key.hasPrefix(payloadPrefix) + || key == QdrantFilterBuilder.rawFilterColumn } public static func column(forPayloadKey key: String) -> String { diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift index 8b9a6aee54..d2a4a83fcb 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift @@ -41,6 +41,9 @@ struct QdrantFilterBuilderTests { #expect(try condition("count", "BETWEEN", "1,5", "5") == #"{"key":"\"count\"","range":{"gte":1,"lte":5}}"#) #expect(try condition("seen", ">=", "2024-01-02T00:00:00Z") == #"{"key":"\"seen\"","range":{"gte":"2024-01-02T00:00:00Z"}}"#) #expect(throws: QdrantFilterError.missingUpperBound(column: "count")) { try condition("count", "BETWEEN", "1", "") } + #expect(throws: QdrantFilterError.boundNotExact(column: "count", value: "9007199254740993")) { + try condition("count", ">=", "9007199254740993") + } } @Test("IN collapses plain text to match any, and keeps every spelling of a number") diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift index 446dec447a..6ec0c041d5 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift @@ -60,7 +60,7 @@ struct QdrantSchemaTests { @Test("Every payload key maps to one column and back, including one that starts with payload.") func payloadKeysRoundTrip() { - let keys = ["city", "id", "score", "vector", "vector.image", "payload.id", "payload.city", "payload."] + let keys = ["city", "id", "score", "vector", "vector.image", "payload.id", "payload.city", "payload.", "__RAW__"] let columns = keys.map(QdrantSchema.column(forPayloadKey:)) #expect(Set(columns).count == keys.count) #expect(columns.map(QdrantSchema.payloadKey(forColumn:)) == keys) diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index 78a9f13704..f98b67fa35 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -139370,6 +139370,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "Qdrant compares %@ as a 64-bit float, which cannot hold %@ exactly." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Qdrant does not support views." : { "extractionState" : "manual", "generatesSymbol" : false From b48dd47ea2ae3606808736f9fc8a796f57ced81f Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Mon, 2 Nov 2026 00:18:55 +0700 Subject: [PATCH 20/20] fix(plugin-qdrant): rank payload overwrite, cluster and alias requests as destructive --- .../Sources/TableProQdrantCore/QdrantConsole.swift | 10 ++++++++-- .../TableProQdrantCoreTests/QdrantConsoleTests.swift | 3 +++ .../Utilities/SQL/QueryClassifierHardeningTests.swift | 7 +++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConsole.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConsole.swift index 9b2880f24a..01945bf62a 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConsole.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConsole.swift @@ -92,8 +92,14 @@ public struct QdrantRequestKind: Sendable, Equatable { if verb == "POST", readRoutes.contains(subroute) { return QdrantRequestKind(tier: .read, touchesServerFiles: touchesServerFiles) } - // A batch can carry deletes, and a POST ending in `delete` or `clear` removes points or payload. - let removes = verb == "DELETE" || route.last == "delete" || route.last == "clear" || subroute == ["points", "batch"] + // A batch can carry deletes, and a POST ending in `delete` or `clear` removes points or payload. Some + // routes delete through the body: a PUT of the payload replaces it whole, the cluster route drops + // shard keys and replicas, and the aliases route deletes aliases. + let removes = verb == "DELETE" || route.last == "delete" || route.last == "clear" + || subroute == ["points", "batch"] + || (verb == "PUT" && subroute == ["points", "payload"]) + || subroute.first == "cluster" + || route == ["collections", "aliases"] return QdrantRequestKind(tier: removes ? .remove : .write, touchesServerFiles: touchesServerFiles) } } diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantConsoleTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantConsoleTests.swift index 5779368989..33f6cb4555 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantConsoleTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantConsoleTests.swift @@ -35,6 +35,9 @@ struct QdrantConsoleTests { #expect(tier("POST", "/collections/docs/points/payload/clear") == .remove) #expect(tier("POST", "/collections/docs/points/batch") == .remove) #expect(tier("DELETE", "/collections/docs") == .remove) + #expect(tier("PUT", "/collections/docs/points/payload?wait=true") == .remove) + #expect(tier("POST", "/collections/docs/cluster") == .remove) + #expect(tier("POST", "/collections/aliases") == .remove) } @Test("A snapshot touches the server's disk, and a dot segment is never read as another route") diff --git a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift index 44ab29ab76..0bc9e4318b 100644 --- a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift @@ -916,6 +916,13 @@ struct QueryClassifierNonSqlTests { #expect(QueryClassifier.classifyTier("POST /collections/docs/points/payload/clear {}", databaseType: .qdrant) == .destructive) #expect(QueryClassifier.classifyTier("POST /collections/docs/points/batch {}", databaseType: .qdrant) == .destructive) #expect(QueryClassifier.classifyTier("DELETE /collections/docs", databaseType: .qdrant) == .destructive) + #expect(QueryClassifier.classifyTier( + "PUT /collections/docs/points/payload?wait=true\n{\"payload\":{},\"filter\":{}}", databaseType: .qdrant + ) == .destructive) + #expect(QueryClassifier.classifyTier( + "POST /collections/docs/cluster {\"drop_sharding_key\":{\"shard_key\":\"a\"}}", databaseType: .qdrant + ) == .destructive) + #expect(QueryClassifier.classifyTier("POST /collections/aliases {\"actions\":[]}", databaseType: .qdrant) == .destructive) #expect(QueryClassifier.isWriteQuery("POST /collections/docs/points/query/../delete {}", databaseType: .qdrant)) #expect(!QueryClassifier.isWriteQuery("POST /collections/docs/facet {\"key\": \"city\"}", databaseType: .qdrant)) #expect(QueryClassifier.classifyTier("DELETE\t/collections/docs", databaseType: .qdrant) == .destructive)