From da3cc85394a0ce283763862dd62965775b63d17e Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Wed, 21 Oct 2026 08:39:07 +0700 Subject: [PATCH 1/5] feat(export): export a data dictionary of tables and columns as Markdown or HTML --- .../DataDictionaryDocument.swift | 257 ++++++++++++++ .../DataDictionaryExportOptions.swift | 31 ++ .../DataDictionaryExportOptionsView.swift | 23 ++ .../DataDictionaryExportPlugin.swift | 171 ++++++++++ .../DataDictionaryHTMLRenderer.swift | 172 ++++++++++ .../DataDictionaryMarkdownRenderer.swift | 133 ++++++++ .../DataDictionaryExportPlugin/Info.plist | 12 + .../DataDictionaryExportTests.swift | 319 ++++++++++++++++++ .../SQLExportReadsRowsTests.swift | 18 + .../SQLExportPlugin/SQLExportPlugin.swift | 4 + .../ExportFormatPlugin.swift | 4 + .../PluginExportDataSource.swift | 9 + .../DatabaseObjectToolEligibility.swift | 11 +- .../TablePro/Core/Menu/FileMenuBuilder.swift | 4 + .../Plugins/ExportDataSourceAdapter.swift | 30 ++ .../Services/Export/ExportFormatCatalog.swift | 19 +- .../Core/Services/Export/ExportService.swift | 15 +- ...nSplitViewController+FileMenuActions.swift | 4 + ...inSplitViewController+MenuValidation.swift | 6 + .../TablePro/Resources/Localizable.xcstrings | 22 ++ .../TablePro/Views/Export/ExportDialog.swift | 12 +- .../Views/Export/ExportProgressView.swift | 8 +- ...ainContentCoordinator+SidebarActions.swift | 8 + .../MainContentView+TransferSheets.swift | 3 +- ...ContentCommandActions+DataDictionary.swift | 25 ++ .../Views/Main/MainContentCoordinator.swift | 1 + .../TablePro/Views/Main/MainContentView.swift | 1 + .../Export/ExportFormatCatalogTests.swift | 29 ++ .../Export/ExportServiceRowCountTests.swift | 78 +++++ .../Core/Menu/ChromeSnapshot.json | 1 + .../ExportDataSourceAdapterCatalogTests.swift | 68 ++++ apple/TableProUITests/ExportUITests.swift | 68 +++- apple/project.yml | 18 +- 33 files changed, 1554 insertions(+), 30 deletions(-) create mode 100644 apple/Plugins/DataDictionaryExportPlugin/DataDictionaryDocument.swift create mode 100644 apple/Plugins/DataDictionaryExportPlugin/DataDictionaryExportOptions.swift create mode 100644 apple/Plugins/DataDictionaryExportPlugin/DataDictionaryExportOptionsView.swift create mode 100644 apple/Plugins/DataDictionaryExportPlugin/DataDictionaryExportPlugin.swift create mode 100644 apple/Plugins/DataDictionaryExportPlugin/DataDictionaryHTMLRenderer.swift create mode 100644 apple/Plugins/DataDictionaryExportPlugin/DataDictionaryMarkdownRenderer.swift create mode 100644 apple/Plugins/DataDictionaryExportPlugin/Info.plist create mode 100644 apple/Plugins/FormatPluginTests/DataDictionaryExportTests.swift create mode 100644 apple/Plugins/FormatPluginTests/SQLExportReadsRowsTests.swift create mode 100644 apple/TablePro/Views/Main/MainContentCommandActions+DataDictionary.swift create mode 100644 apple/TableProTests/Core/Export/ExportServiceRowCountTests.swift create mode 100644 apple/TableProTests/Core/Plugins/ExportDataSourceAdapterCatalogTests.swift diff --git a/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryDocument.swift b/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryDocument.swift new file mode 100644 index 0000000000..9e7bf4af49 --- /dev/null +++ b/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryDocument.swift @@ -0,0 +1,257 @@ +// +// DataDictionaryDocument.swift +// DataDictionaryExportPlugin +// + +import Foundation +import TableProPluginKit + +struct DataDictionaryObject: Sendable { + let name: String + let container: String? + let kind: PluginExportObjectKind + var comment: String? + var columns: [PluginColumnInfo] = [] + var indexes: [PluginIndexInfo] = [] + var foreignKeys: [PluginForeignKeyInfo] = [] + var columnsError: String? + + var kindLabel: String { + switch kind { + case .view: "View" + case .materializedView: "Materialized view" + case .foreignTable: "Foreign table" + default: "Table" + } + } +} + +struct DataDictionaryColumnRow: Equatable { + let name: String + let type: String + let isNullable: Bool + let defaultText: String? + let isPrimaryKey: Bool + let references: [DataDictionaryReference] + let comment: String? +} + +struct DataDictionaryReference: Equatable { + let table: String + let column: String? + let anchor: String? + + var text: String { + guard let column else { return table } + return "\(table).\(column)" + } +} + +struct DataDictionaryIndexRow: Equatable { + let name: String + let columns: String + let kind: String + let method: String +} + +struct DataDictionaryForeignKeyRow: Equatable { + let name: String + let columns: [String] + let target: DataDictionaryReference + let referencedColumns: [String] + let onDelete: String + let onUpdate: String +} + +// Everything both renderers need, decided once so the Markdown and HTML files never disagree. +struct DataDictionaryDocument { + let title: String + let databaseTypeId: String + let generatedAt: Date + let objects: [DataDictionaryObject] + let anchors: [String] + private let anchorByTable: [String: String] + let qualifiesNames: Bool + let showsDefault: Bool + let showsKey: Bool + let showsReferences: Bool + let showsColumnComment: Bool + + init(title: String, databaseTypeId: String, generatedAt: Date, objects: [DataDictionaryObject]) { + self.title = title + self.databaseTypeId = databaseTypeId + self.generatedAt = generatedAt + self.objects = objects + let qualifies = Set(objects.map { $0.container ?? "" }).count > 1 + self.qualifiesNames = qualifies + let anchors = DataDictionaryAnchors.make(for: objects.map { Self.displayName(of: $0, qualified: qualifies) }) + self.anchors = anchors + var anchorByTable: [String: String] = [:] + for (object, anchor) in zip(objects, anchors) { + let key = Self.tableKey(container: object.container, name: object.name) + if anchorByTable[key] == nil { anchorByTable[key] = anchor } + } + self.anchorByTable = anchorByTable + self.showsDefault = objects.contains { object in object.columns.contains { Self.defaultText(of: $0) != nil } } + self.showsKey = objects.contains { object in object.columns.contains(where: \.isPrimaryKey) } + self.showsReferences = objects.contains { !$0.foreignKeys.isEmpty } + self.showsColumnComment = objects.contains { object in object.columns.contains { Self.hasText($0.comment) } } + } + + func displayName(at index: Int) -> String { + Self.displayName(of: objects[index], qualified: qualifiesNames) + } + + func summary() -> String { + let counts = Dictionary(grouping: objects, by: \.kindLabel).mapValues(\.count) + let parts = ["Table", "View", "Materialized view", "Foreign table"].compactMap { label -> String? in + guard let count = counts[label] else { return nil } + let noun = label.lowercased() + return count == 1 ? "1 \(noun)" : "\(count) \(noun)s" + } + let formatter = DateFormatter() + formatter.locale = Locale(identifier: "en_US_POSIX") + formatter.dateFormat = "yyyy-MM-dd HH:mm xxx" + let when = formatter.string(from: generatedAt) + let source = databaseTypeId.isEmpty ? "" : " from \(databaseTypeId)" + let contents = parts.isEmpty ? "" : ": \(parts.joined(separator: ", "))" + return "Data dictionary generated \(when)\(source)\(contents)." + } + + func columnRows(at index: Int) -> [DataDictionaryColumnRow] { + let object = objects[index] + return object.columns.map { column in + let references = object.foreignKeys + .filter { $0.column == column.name } + .map { reference(to: $0, from: object) } + return DataDictionaryColumnRow( + name: column.name, + type: column.dataType, + isNullable: column.isNullable, + defaultText: Self.defaultText(of: column), + isPrimaryKey: column.isPrimaryKey, + references: references, + comment: Self.hasText(column.comment) ? column.comment : nil + ) + } + } + + func indexRows(at index: Int) -> [DataDictionaryIndexRow] { + objects[index].indexes.map { info in + var keys = info.columns.joined(separator: ", ") + if let included = info.includedColumns, !included.isEmpty { + keys += " INCLUDE (\(included.joined(separator: ", ")))" + } + if let condition = info.whereClause, Self.hasText(condition) { + keys += " WHERE \(condition)" + } + let kind = info.isPrimary ? "Primary key" : (info.isUnique ? "Unique" : "Index") + return DataDictionaryIndexRow(name: info.name, columns: keys, kind: kind, method: info.type) + } + } + + // A composite key arrives as one entry per column under one name. + func foreignKeyRows(at index: Int) -> [DataDictionaryForeignKeyRow] { + let object = objects[index] + var order: [String] = [] + var groups: [String: [PluginForeignKeyInfo]] = [:] + for key in object.foreignKeys { + if groups[key.name] == nil { order.append(key.name) } + groups[key.name, default: []].append(key) + } + return order.compactMap { name in + guard let group = groups[name], let first = group.first else { return nil } + return DataDictionaryForeignKeyRow( + name: name, + columns: group.map(\.column), + target: reference(to: first, from: object, includesColumn: false), + referencedColumns: group.map(\.referencedColumn), + onDelete: first.onDelete, + onUpdate: first.onUpdate + ) + } + } + + private func reference( + to key: PluginForeignKeyInfo, + from object: DataDictionaryObject, + includesColumn: Bool = true + ) -> DataDictionaryReference { + let schema = Self.hasText(key.referencedSchema) ? key.referencedSchema : nil + let database = Self.hasText(key.referencedDatabase) ? key.referencedDatabase : nil + let targetContainer = schema ?? object.container + var parts: [String] = [] + if let database { parts.append(database) } + if let schema, database != nil || schema != object.container { parts.append(schema) } + parts.append(key.referencedTable) + let anchor = database == nil + ? anchorByTable[Self.tableKey(container: targetContainer, name: key.referencedTable)] + : nil + return DataDictionaryReference( + table: parts.joined(separator: "."), + column: includesColumn ? key.referencedColumn : nil, + anchor: anchor + ) + } + + private static func tableKey(container: String?, name: String) -> String { + "\(container ?? "")\u{1F}\(name)" + } + + static func displayName(of object: DataDictionaryObject, qualified: Bool) -> String { + guard qualified, let container = object.container, !container.isEmpty else { return object.name } + return "\(container).\(object.name)" + } + + static func defaultText(of column: PluginColumnInfo) -> String? { + if let value = column.defaultValue, hasText(value) { return value } + if let expression = column.generationExpression, hasText(expression) { + let storage = column.generationKind.map { " \($0.rawValue)" } ?? "" + return "GENERATED ALWAYS AS (\(expression))\(storage)" + } + if let identity = column.identityKind { return "GENERATED \(identity.rawValue) AS IDENTITY" } + if column.extra?.lowercased().contains("auto_increment") == true { return "AUTO_INCREMENT" } + return nil + } + + static func hasText(_ value: String?) -> Bool { + guard let value else { return false } + return !value.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + } +} + +enum DataDictionaryAnchors { + static let contents = "contents" + + // Explicit ids, because every Markdown renderer derives heading slugs its own way. + static func make(for names: [String]) -> [String] { + var used: Set = [contents] + return names.map { name in + let base = slug(name) + var candidate = base + var suffix = 2 + while used.contains(candidate) { + candidate = "\(base)-\(suffix)" + suffix += 1 + } + used.insert(candidate) + return candidate + } + } + + static func slug(_ name: String) -> String { + var result = "" + var lastWasDash = false + for character in name.lowercased() { + if character.isLetter || character.isNumber || character == "_" { + result.append(character) + lastWasDash = false + } else if !lastWasDash, !result.isEmpty { + result.append("-") + lastWasDash = true + } + } + while result.hasSuffix("-") { result.removeLast() } + return result.isEmpty ? "object" : result + } +} diff --git a/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryExportOptions.swift b/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryExportOptions.swift new file mode 100644 index 0000000000..e8dbff1f53 --- /dev/null +++ b/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryExportOptions.swift @@ -0,0 +1,31 @@ +// +// DataDictionaryExportOptions.swift +// DataDictionaryExportPlugin +// + +import Foundation + +public enum DataDictionaryFileType: String, Codable, CaseIterable, Sendable { + case markdown + case html + + public var fileExtension: String { + switch self { + case .markdown: "md" + case .html: "html" + } + } +} + +public struct DataDictionaryExportOptions: Equatable, Codable { + public var fileType: DataDictionaryFileType = .markdown + + public init() {} + + // A synthesized decoder throws on a key the saved payload predates instead of using the default. + public init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + fileType = (try? container.decodeIfPresent(DataDictionaryFileType.self, forKey: .fileType)) + ?? DataDictionaryExportOptions().fileType + } +} diff --git a/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryExportOptionsView.swift b/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryExportOptionsView.swift new file mode 100644 index 0000000000..e70404cd55 --- /dev/null +++ b/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryExportOptionsView.swift @@ -0,0 +1,23 @@ +// +// DataDictionaryExportOptionsView.swift +// DataDictionaryExportPlugin +// + +import SwiftUI + +struct DataDictionaryExportOptionsView: View { + @ObservedObject var plugin: DataDictionaryExportPlugin + + var body: some View { + Picker(String(localized: "Write as", bundle: .main), selection: $plugin.settings.fileType) { + Text(verbatim: "Markdown").tag(DataDictionaryFileType.markdown) + Text(verbatim: "HTML").tag(DataDictionaryFileType.html) + } + .pickerStyle(.radioGroup) + .help(String( + localized: "Markdown has a table of contents for a README or a wiki. HTML is one file that opens in any browser and prints.", + bundle: .main + )) + .font(.system(size: 13)) + } +} diff --git a/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryExportPlugin.swift b/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryExportPlugin.swift new file mode 100644 index 0000000000..f86938b18e --- /dev/null +++ b/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryExportPlugin.swift @@ -0,0 +1,171 @@ +// +// DataDictionaryExportPlugin.swift +// DataDictionaryExportPlugin +// + +import Combine +import Foundation +import SwiftUI +import TableProPluginKit + +final class DataDictionaryExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugin, @unchecked Sendable { + static let pluginName = "Data Dictionary Export" + static let pluginVersion = "1.0.0" + static let pluginDescription = "Describe every table and column as Markdown or HTML" + static let formatId = "datadict" + static var formatDisplayName: String { String(localized: "Data Dictionary", bundle: .main) } + static let defaultFileExtension = "md" + static let iconName = "book.closed" + static let supportedObjectKinds: [PluginExportObjectKind] = [.table, .view, .materializedView, .foreignTable] + + typealias Settings = DataDictionaryExportOptions + static let settingsStorageId = "datadict" + + @Published var settings = DataDictionaryExportOptions() { + didSet { saveSettings() } + } + + required init() { loadSettings() } + + @MainActor + func settingsView() -> AnyView? { + AnyView(DataDictionaryExportOptionsView(plugin: self)) + } + + func resetSettingsToDefaults() { + settings = DataDictionaryExportOptions() + } + + var currentFileExtension: String { + settings.fileType.fileExtension + } + + func readsRows(optionValues: [Bool], kind: PluginExportObjectKind) -> Bool { + false + } + + func export( + tables: [PluginExportTable], + dataSource: any PluginExportDataSource, + destination: URL, + progress: PluginExportProgress + ) async throws -> ExportFormatResult { + // Read once: every window shares this instance, and another one's options pane can write it. + let fileType = settings.fileType + var failures = DataDictionaryReadFailures() + var objects: [DataDictionaryObject] = [] + objects.reserveCapacity(tables.count) + for (index, table) in tables.enumerated() { + try progress.checkCancellation() + progress.setCurrentTable(table.qualifiedName, index: index + 1) + objects.append(try await DataDictionaryReader.read(table, from: dataSource, progress: progress, failures: &failures)) + } + try progress.checkCancellation() + + let document = DataDictionaryDocument( + title: destination.deletingPathExtension().lastPathComponent, + databaseTypeId: dataSource.databaseTypeId, + generatedAt: Date(), + objects: objects + ) + let text = switch fileType { + case .markdown: DataDictionaryMarkdownRenderer.render(document) + case .html: DataDictionaryHTMLRenderer.render(document) + } + + let (fileHandle, tempURL) = try PluginExportUtilities.beginAtomicWrite(for: destination) + var committed = false + defer { + if !committed { PluginExportUtilities.rollbackAtomicWrite(at: tempURL) } + } + try fileHandle.write(contentsOf: text.toUTF8Data()) + try fileHandle.close() + try PluginExportUtilities.commitAtomicWrite(from: tempURL, to: destination) + committed = true + progress.finalizeTable() + return ExportFormatResult(warnings: failures.warnings) + } +} + +struct DataDictionaryReadFailures { + var columns: [String] = [] + var indexes: [String] = [] + var foreignKeys: [String] = [] + var comments: [String] = [] + + var warnings: [String] { + var warnings: [String] = [] + if !columns.isEmpty { + warnings.append(String( + format: String(localized: "Could not fetch table structure for: %@", bundle: .main), + columns.joined(separator: ", "))) + } + if !indexes.isEmpty { + warnings.append(String( + format: String(localized: "Could not fetch indexes for: %@", bundle: .main), + indexes.joined(separator: ", "))) + } + if !foreignKeys.isEmpty { + warnings.append(String( + format: String(localized: "Could not fetch foreign keys for: %@", bundle: .main), + foreignKeys.joined(separator: ", "))) + } + if !comments.isEmpty { + warnings.append(String( + format: String(localized: "Could not fetch comments for: %@", bundle: .main), + comments.joined(separator: ", "))) + } + return warnings + } +} + +// Catalog reads only. A failed read is named in the summary and in the file, never left as a gap +// that reads like an object with no indexes or no comment. +enum DataDictionaryReader { + static func read( + _ table: PluginExportTable, + from dataSource: any PluginExportDataSource, + progress: PluginExportProgress, + failures: inout DataDictionaryReadFailures + ) async throws -> DataDictionaryObject { + var object = DataDictionaryObject(name: table.name, container: table.containerName, kind: table.kind) + do { + object.columns = try await dataSource.fetchColumns(table: table.name, databaseName: table.databaseName) + } catch { + try rethrowIfCancelled(error, progress: progress) + object.columnsError = error.localizedDescription + failures.columns.append(table.qualifiedName) + } + + if table.kind == .table || table.kind == .materializedView { + do { + object.indexes = try await dataSource.fetchIndexes(table: table.name, databaseName: table.databaseName) + } catch { + try rethrowIfCancelled(error, progress: progress) + failures.indexes.append(table.qualifiedName) + } + } + + if table.kind == .table { + do { + object.foreignKeys = try await dataSource.fetchForeignKeys(table: table.name, databaseName: table.databaseName) + } catch { + try rethrowIfCancelled(error, progress: progress) + failures.foreignKeys.append(table.qualifiedName) + } + } + + do { + object.comment = try await dataSource.fetchObjectComment(table) + } catch { + try rethrowIfCancelled(error, progress: progress) + failures.comments.append(table.qualifiedName) + } + return object + } + + private static func rethrowIfCancelled(_ error: Error, progress: PluginExportProgress) throws { + if error is CancellationError || error is PluginExportCancellationError { throw error } + try progress.checkCancellation() + } +} diff --git a/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryHTMLRenderer.swift b/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryHTMLRenderer.swift new file mode 100644 index 0000000000..bdff96a027 --- /dev/null +++ b/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryHTMLRenderer.swift @@ -0,0 +1,172 @@ +// +// DataDictionaryHTMLRenderer.swift +// DataDictionaryExportPlugin +// + +import Foundation +import TableProPluginKit + +// One file with its style inline and no script, so it opens, mails and prints the same anywhere. +enum DataDictionaryHTMLRenderer { + static func render(_ document: DataDictionaryDocument) -> String { + var html = """ + + + + + + + \(escape(document.title)) + + + +
+

\(escape(document.title))

+

\(escape(document.summary()))

+
+ + """ + if !document.objects.isEmpty { + html += "\n" + } + for index in document.objects.indices { + html += section(at: index, in: document) + } + html += "\n\n" + return html + } + + private static func section(at index: Int, in document: DataDictionaryDocument) -> String { + let object = document.objects[index] + var html = "
\n" + html += "

\(escape(document.displayName(at: index)))

\n" + html += "

\(escape(object.kindLabel))." + if let comment = object.comment { + html += " \(escape(comment))" + } + html += "

\n" + + if let error = object.columnsError { + html += "

Columns could not be read: \(escape(error))

\n" + } else if object.columns.isEmpty { + html += "

No columns listed.

\n" + } else { + html += columnTable(document.columnRows(at: index), in: document) + } + + let indexes = document.indexRows(at: index) + if !indexes.isEmpty { + html += "

Indexes

\n" + html += table( + header: ["Name", "Columns", "Kind", "Method"], + rows: indexes.map { ["\(escape($0.name))", "\(escape($0.columns))", escape($0.kind), escape($0.method)] } + ) + } + + let keys = document.foreignKeyRows(at: index) + if !keys.isEmpty { + html += "

Foreign keys

\n" + html += table( + header: ["Name", "Columns", "References", "On delete", "On update"], + rows: keys.map { key in + [ + "\(escape(key.name))", + key.columns.map { "\(escape($0))" }.joined(separator: ", "), + "\(link(key.target)) (\(key.referencedColumns.map { "\(escape($0))" }.joined(separator: ", ")))", + escape(key.onDelete), + escape(key.onUpdate) + ] + } + ) + } + html += "
\n" + return html + } + + private static func columnTable(_ rows: [DataDictionaryColumnRow], in document: DataDictionaryDocument) -> String { + var header = ["Column", "Type", "Nullable"] + if document.showsDefault { header.append("Default") } + if document.showsKey { header.append("Key") } + if document.showsReferences { header.append("References") } + if document.showsColumnComment { header.append("Comment") } + let cells = rows.map { row -> [String] in + var cells = ["\(escape(row.name))", "\(escape(row.type))", row.isNullable ? "Yes" : "No"] + if document.showsDefault { cells.append(row.defaultText.map { "\(escape($0))" } ?? "") } + if document.showsKey { cells.append(row.isPrimaryKey ? "PK" : "") } + if document.showsReferences { cells.append(row.references.map { link($0) }.joined(separator: ", ")) } + if document.showsColumnComment { cells.append(row.comment.map(escape) ?? "") } + return cells + } + return table(header: header, rows: cells) + } + + private static func table(header: [String], rows: [[String]]) -> String { + var html = "\n" + html += header.map { "" }.joined() + html += "\n\n" + for row in rows { + html += "" + row.map { "" }.joined() + "\n" + } + html += "\n
\(escape($0))
\($0)
\n" + return html + } + + private static func link(_ reference: DataDictionaryReference) -> String { + let label = "\(escape(reference.text))" + guard let anchor = reference.anchor else { return label } + return "\(label)" + } + + static func escape(_ value: String) -> String { + var escaped = "" + escaped.reserveCapacity(value.count) + for character in value { + switch character { + case "&": escaped += "&" + case "<": escaped += "<" + case ">": escaped += ">" + case "\"": escaped += """ + case "'": escaped += "'" + default: escaped.append(character) + } + } + return escaped + } + + private static let style = """ + :root { color-scheme: light dark; --text: #1d1d1f; --muted: #6e6e73; --rule: #d2d2d7; --head: #f5f5f7; --link: #0066cc; } + @media (prefers-color-scheme: dark) { + :root { --text: #f5f5f7; --muted: #a1a1a6; --rule: #424245; --head: #1c1c1e; --link: #4da3ff; } + body { background: #000; } + } + body { font: 14px/1.5 -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif; color: var(--text); margin: 32px auto; max-width: 1100px; padding: 0 16px; } + h1 { font-size: 26px; margin: 0 0 4px; } + h2 { font-size: 19px; margin: 32px 0 6px; } + h3 { font-size: 15px; margin: 18px 0 6px; } + .summary, .kind { color: var(--muted); } + .error { color: #c9252d; } + a { color: var(--link); text-decoration: none; } + code { font: 12.5px/1.4 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; overflow-wrap: anywhere; } + nav ol { columns: 2 260px; padding-left: 20px; } + table { border-collapse: collapse; width: 100%; margin: 6px 0 12px; } + th, td { border: 1px solid var(--rule); padding: 4px 8px; text-align: left; vertical-align: top; } + th { background: var(--head); font-weight: 600; } + @media print { + :root { --text: #000; --muted: #444; --rule: #999; --head: #eee; --link: #000; } + body { background: #fff; margin: 0; max-width: none; font-size: 10.5pt; } + code { font-size: 9.5pt; } + nav ol { columns: 2; } + h2, h3 { break-after: avoid; } + thead { display: table-header-group; } + tr { break-inside: avoid; } + } + """ +} diff --git a/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryMarkdownRenderer.swift b/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryMarkdownRenderer.swift new file mode 100644 index 0000000000..1133a1cb25 --- /dev/null +++ b/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryMarkdownRenderer.swift @@ -0,0 +1,133 @@ +// +// DataDictionaryMarkdownRenderer.swift +// DataDictionaryExportPlugin +// + +import Foundation +import TableProPluginKit + +enum DataDictionaryMarkdownRenderer { + static func render(_ document: DataDictionaryDocument) -> String { + var lines = ["# \(text(document.title))", "", text(document.summary()), ""] + if !document.objects.isEmpty { + lines += ["", "", "## Contents", ""] + for index in document.objects.indices { + let object = document.objects[index] + let kind = object.kind == .table ? "" : " (\(object.kindLabel.lowercased()))" + lines.append("- [\(code(document.displayName(at: index)))](#\(document.anchors[index]))\(kind)") + } + lines.append("") + } + for index in document.objects.indices { + lines += section(at: index, in: document) + } + return lines.joined(separator: "\n") + } + + private static func section(at index: Int, in document: DataDictionaryDocument) -> [String] { + let object = document.objects[index] + var lines = ["", "", "## \(code(document.displayName(at: index)))", ""] + if let comment = object.comment { + lines.append("**\(object.kindLabel).** \(text(comment))") + } else { + lines.append("**\(object.kindLabel).**") + } + lines.append("") + + if let error = object.columnsError { + lines += ["Columns could not be read: \(text(error))", ""] + } else if object.columns.isEmpty { + lines += ["No columns listed.", ""] + } else { + lines += columnTable(document.columnRows(at: index), in: document) + [""] + } + + let indexes = document.indexRows(at: index) + if !indexes.isEmpty { + lines += ["### Indexes", ""] + lines += table( + header: ["Name", "Columns", "Kind", "Method"], + rows: indexes.map { [code($0.name, inTable: true), code($0.columns, inTable: true), $0.kind, text($0.method, inTable: true)] } + ) + lines.append("") + } + + let keys = document.foreignKeyRows(at: index) + if !keys.isEmpty { + lines += ["### Foreign keys", ""] + lines += table( + header: ["Name", "Columns", "References", "On delete", "On update"], + rows: keys.map { key in + [ + code(key.name, inTable: true), + key.columns.map { code($0, inTable: true) }.joined(separator: ", "), + "\(link(key.target)) (\(key.referencedColumns.map { code($0, inTable: true) }.joined(separator: ", ")))", + text(key.onDelete, inTable: true), + text(key.onUpdate, inTable: true) + ] + } + ) + lines.append("") + } + return lines + } + + private static func columnTable(_ rows: [DataDictionaryColumnRow], in document: DataDictionaryDocument) -> [String] { + var header = ["Column", "Type", "Nullable"] + if document.showsDefault { header.append("Default") } + if document.showsKey { header.append("Key") } + if document.showsReferences { header.append("References") } + if document.showsColumnComment { header.append("Comment") } + let cells = rows.map { row -> [String] in + var cells = [code(row.name, inTable: true), code(row.type, inTable: true), row.isNullable ? "Yes" : "No"] + if document.showsDefault { cells.append(row.defaultText.map { code($0, inTable: true) } ?? "") } + if document.showsKey { cells.append(row.isPrimaryKey ? "PK" : "") } + if document.showsReferences { cells.append(row.references.map { link($0) }.joined(separator: ", ")) } + if document.showsColumnComment { cells.append(row.comment.map { text($0, inTable: true) } ?? "") } + return cells + } + return table(header: header, rows: cells) + } + + private static func table(header: [String], rows: [[String]]) -> [String] { + let line = { (cells: [String]) in "| " + cells.joined(separator: " | ") + " |" } + return [line(header), line(header.map { _ in "---" })] + rows.map(line) + } + + private static func link(_ reference: DataDictionaryReference) -> String { + let label = code(reference.text, inTable: true) + guard let anchor = reference.anchor else { return label } + return "[\(label)](#\(anchor))" + } + + // A code span shows its content literally, so only its own fence and a table's pipe need care. + static func code(_ value: String, inTable: Bool = false) -> String { + var content = flattened(value) + guard !content.isEmpty else { return "" } + if inTable { content = content.replacingOccurrences(of: "|", with: "\\|") } + var longestRun = 0 + var run = 0 + for character in content { + run = character == "`" ? run + 1 : 0 + longestRun = max(longestRun, run) + } + let fence = String(repeating: "`", count: longestRun + 1) + let padded = content.hasPrefix("`") || content.hasSuffix("`") ? " \(content) " : content + return fence + padded + fence + } + + static func text(_ value: String, inTable: Bool = false) -> String { + var escaped = "" + for character in flattened(value) { + if "\\`*_[]<>#&".contains(character) || (inTable && character == "|") { + escaped.append("\\") + } + escaped.append(character) + } + return escaped + } + + private static func flattened(_ value: String) -> String { + value.split(omittingEmptySubsequences: true, whereSeparator: \.isNewline).joined(separator: " ") + } +} diff --git a/apple/Plugins/DataDictionaryExportPlugin/Info.plist b/apple/Plugins/DataDictionaryExportPlugin/Info.plist new file mode 100644 index 0000000000..dfe303774b --- /dev/null +++ b/apple/Plugins/DataDictionaryExportPlugin/Info.plist @@ -0,0 +1,12 @@ + + + + + TableProPluginKitVersion + TABLEPRO_PLUGINKIT_VERSION + TableProProvidesExportFormatIds + + datadict + + + diff --git a/apple/Plugins/FormatPluginTests/DataDictionaryExportTests.swift b/apple/Plugins/FormatPluginTests/DataDictionaryExportTests.swift new file mode 100644 index 0000000000..6b0633d4b1 --- /dev/null +++ b/apple/Plugins/FormatPluginTests/DataDictionaryExportTests.swift @@ -0,0 +1,319 @@ +// +// DataDictionaryExportTests.swift +// FormatPluginTests +// + +import Foundation +import TableProPluginKit +import Testing + +private final class CatalogOnlyDataSource: PluginExportDataSource, @unchecked Sendable { + let databaseTypeId: String + var columns: [String: [PluginColumnInfo]] = [:] + var indexes: [String: [PluginIndexInfo]] = [:] + var foreignKeys: [String: [PluginForeignKeyInfo]] = [:] + var comments: [String: String] = [:] + var failingColumns: Set = [] + var failingIndexes: Set = [] + var onColumnRead: ((String) -> Void)? + private(set) var rowReads = 0 + private(set) var indexReads: [String] = [] + private(set) var foreignKeyReads: [String] = [] + + init(databaseTypeId: String = "PostgreSQL") { + self.databaseTypeId = databaseTypeId + } + + func streamRows(table: String, databaseName: String) -> AsyncThrowingStream { + rowReads += 1 + return AsyncThrowingStream { $0.finish() } + } + + func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? { + rowReads += 1 + return nil + } + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { "" } + + func execute(query: String) async throws -> PluginQueryResult { + rowReads += 1 + return PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } + + func quoteIdentifier(_ identifier: String) -> String { "\"\(identifier)\"" } + + func escapeStringLiteral(_ value: String) -> String { value } + + func fetchColumns(table: String, databaseName: String) async throws -> [PluginColumnInfo] { + onColumnRead?(table) + if failingColumns.contains(table) { throw StubError.denied } + return columns[table] ?? [] + } + + func fetchIndexes(table: String, databaseName: String) async throws -> [PluginIndexInfo] { + indexReads.append(table) + if failingIndexes.contains(table) { throw StubError.denied } + return indexes[table] ?? [] + } + + func fetchForeignKeys(table: String, databaseName: String) async throws -> [PluginForeignKeyInfo] { + foreignKeyReads.append(table) + return foreignKeys[table] ?? [] + } + + func fetchObjectComment(_ object: PluginExportTable) async throws -> String? { + comments[object.name] + } +} + +private enum StubError: LocalizedError { + case denied + + var errorDescription: String? { "permission denied" } +} + +private func object(_ name: String, kind: PluginExportObjectKind = .table, schema: String = "public") -> PluginExportTable { + PluginExportTable(name: name, databaseName: "", tableType: kind.rawValue, schema: schema, kind: kind) +} + +private func shopSource() -> CatalogOnlyDataSource { + let source = CatalogOnlyDataSource() + source.columns = [ + "orgs": [ + PluginColumnInfo(name: "id", dataType: "integer", isNullable: false, isPrimaryKey: true, + defaultValue: "nextval('orgs_id_seq'::regclass)"), + PluginColumnInfo(name: "name", dataType: "text", isNullable: false, comment: "Shown on invoices") + ], + "users": [ + PluginColumnInfo(name: "id", dataType: "bigint", isNullable: false, isPrimaryKey: true), + PluginColumnInfo(name: "org_id", dataType: "integer", isNullable: true) + ], + "active_users": [ + PluginColumnInfo(name: "id", dataType: "bigint", isNullable: true) + ] + ] + source.indexes = [ + "users": [ + PluginIndexInfo(name: "users_pkey", columns: ["id"], isUnique: true, isPrimary: true, type: "btree"), + PluginIndexInfo(name: "users_org_idx", columns: ["org_id"], type: "btree", whereClause: "org_id IS NOT NULL") + ] + ] + source.foreignKeys = [ + "users": [ + PluginForeignKeyInfo(name: "users_org_fk", column: "org_id", referencedTable: "orgs", + referencedColumn: "id", referencedDatabase: nil, referencedSchema: "public", + onDelete: "CASCADE") + ] + ] + source.comments = ["orgs": "Companies that own accounts"] + return source +} + +private func readDocument( + _ tables: [PluginExportTable], + from source: CatalogOnlyDataSource +) async throws -> (DataDictionaryDocument, DataDictionaryReadFailures) { + var failures = DataDictionaryReadFailures() + var objects: [DataDictionaryObject] = [] + let progress = PluginExportProgress(progress: Progress(totalUnitCount: 0)) + for table in tables { + objects.append(try await DataDictionaryReader.read(table, from: source, progress: progress, failures: &failures)) + } + let document = DataDictionaryDocument( + title: "shop", + databaseTypeId: source.databaseTypeId, + generatedAt: Date(timeIntervalSince1970: 0), + objects: objects + ) + return (document, failures) +} + +struct DataDictionaryExportTests { + @Test("The plugin reads no rows and names its file after the chosen output") + func readsNoRows() { + let plugin = DataDictionaryExportPlugin() + for kind in DataDictionaryExportPlugin.supportedObjectKinds { + #expect(!plugin.readsRows(optionValues: [], kind: kind)) + } + #expect(DataDictionaryFileType.markdown.fileExtension == "md") + #expect(DataDictionaryFileType.html.fileExtension == "html") + } + + @Test("A schema is read from the catalog only: columns, indexes, keys and comments, never a row") + func readsTheCatalogOnly() async throws { + let source = shopSource() + let (document, failures) = try await readDocument( + [object("orgs"), object("users"), object("active_users", kind: .view)], from: source) + + #expect(source.rowReads == 0) + #expect(failures.warnings.isEmpty) + #expect(source.indexReads == ["orgs", "users"]) + #expect(source.foreignKeyReads == ["orgs", "users"]) + #expect(document.objects.map { $0.columns.count } == [2, 2, 1]) + #expect(document.objects[0].comment == "Companies that own accounts") + } + + @Test("Markdown lists every object in a table of contents that links to its anchor") + func markdownContents() async throws { + let (document, _) = try await readDocument( + [object("orgs"), object("users"), object("active_users", kind: .view)], from: shopSource()) + let markdown = DataDictionaryMarkdownRenderer.render(document) + + #expect(markdown.hasPrefix("# shop\n\nData dictionary generated ")) + #expect(markdown.contains(" from PostgreSQL: 2 tables, 1 view.\n")) + #expect(markdown.contains("- [`orgs`](#orgs)\n- [`users`](#users)\n- [`active_users`](#active_users) (view)")) + #expect(markdown.contains("\n\n## `users`\n\n**Table.**")) + #expect(markdown.contains("**Table.** Companies that own accounts")) + #expect(markdown.contains("**View.**")) + } + + @Test("Each column row carries its type, nullability, default, key, reference and comment") + func markdownColumns() async throws { + let (document, _) = try await readDocument([object("orgs"), object("users")], from: shopSource()) + let markdown = DataDictionaryMarkdownRenderer.render(document) + + #expect(markdown.contains("| Column | Type | Nullable | Default | Key | References | Comment |")) + #expect(markdown.contains( + "| `id` | `integer` | No | `nextval('orgs_id_seq'::regclass)` | PK | | |")) + #expect(markdown.contains("| `name` | `text` | No | | | | Shown on invoices |")) + #expect(markdown.contains("| `org_id` | `integer` | Yes | | | [`orgs.id`](#orgs) | |")) + #expect(markdown.contains("| `users_org_idx` | `org_id WHERE org_id IS NOT NULL` | Index | btree |")) + #expect(markdown.contains("| `users_pkey` | `id` | Primary key | btree |")) + #expect(markdown.contains("| `users_org_fk` | `org_id` | [`orgs`](#orgs) (`id`) | CASCADE | NO ACTION |")) + } + + @Test("Columns nothing in the document fills are left out rather than printed empty") + func emptyColumnsAreLeftOut() async throws { + let source = CatalogOnlyDataSource(databaseTypeId: "SQLite") + source.columns = ["notes": [PluginColumnInfo(name: "body", dataType: "TEXT")]] + let (document, _) = try await readDocument([object("notes", schema: "main")], from: source) + let markdown = DataDictionaryMarkdownRenderer.render(document) + let html = DataDictionaryHTMLRenderer.render(document) + + #expect(markdown.contains("| Column | Type | Nullable |\n| --- | --- | --- |\n| `body` | `TEXT` | Yes |")) + #expect(!markdown.contains("### Indexes")) + #expect(!markdown.contains("### Foreign keys")) + #expect(html.contains("ColumnTypeNullable")) + } + + @Test("A pipe, a backtick, a line break and markup in metadata cannot break the Markdown table") + func markdownEscaping() async throws { + let source = CatalogOnlyDataSource() + source.columns = ["t": [ + PluginColumnInfo(name: "a|b", dataType: "text", defaultValue: "'x`y'", comment: "first line\nsecond | *bold* ") + ]] + let (document, _) = try await readDocument([object("t")], from: source) + let markdown = DataDictionaryMarkdownRenderer.render(document) + + #expect(markdown.contains("| `a\\|b` | `text` | Yes | ``'x`y'`` | first line second \\| \\*bold\\* \\ |")) + } + + @Test("A composite foreign key is one row, and a key into another schema names that schema") + func compositeAndCrossSchemaKeys() async throws { + let source = CatalogOnlyDataSource() + source.columns = ["lines": [ + PluginColumnInfo(name: "order_id", dataType: "integer"), + PluginColumnInfo(name: "line_no", dataType: "integer") + ]] + source.foreignKeys = ["lines": [ + PluginForeignKeyInfo(name: "lines_fk", column: "order_id", referencedTable: "orders", + referencedColumn: "id", referencedDatabase: nil, referencedSchema: "sales"), + PluginForeignKeyInfo(name: "lines_fk", column: "line_no", referencedTable: "orders", + referencedColumn: "line", referencedDatabase: nil, referencedSchema: "sales") + ]] + let (document, _) = try await readDocument([object("lines")], from: source) + let rows = document.foreignKeyRows(at: 0) + + #expect(rows.count == 1) + #expect(rows.first?.columns == ["order_id", "line_no"]) + #expect(rows.first?.referencedColumns == ["id", "line"]) + #expect(rows.first?.target.text == "sales.orders") + #expect(rows.first?.target.anchor == nil) + } + + @Test("Objects from two schemas are named with their schema, and every anchor is unique") + func qualifiedNamesAndUniqueAnchors() async throws { + let source = CatalogOnlyDataSource() + source.columns = ["users": [PluginColumnInfo(name: "id", dataType: "int")]] + let (document, _) = try await readDocument( + [object("users", schema: "public"), object("users", schema: "audit")], from: source) + let markdown = DataDictionaryMarkdownRenderer.render(document) + + #expect(document.anchors == ["public-users", "audit-users"]) + #expect(markdown.contains("## `public.users`")) + #expect(markdown.contains("## `audit.users`")) + #expect(DataDictionaryAnchors.make(for: ["Order Items", "order-items", "contents", "!!"]) + == ["order-items", "order-items-2", "contents-2", "object"]) + } + + @Test("The HTML file is self-contained, escaped and prints with repeated table headers") + func htmlIsSelfContained() async throws { + let source = shopSource() + source.comments["users"] = " & co" + let (document, _) = try await readDocument([object("orgs"), object("users")], from: source) + let html = DataDictionaryHTMLRenderer.render(document) + + #expect(html.hasPrefix("")) + #expect(!html.contains("users")) + #expect(html.contains("
")) + #expect(html.contains("orgs.id")) + #expect(html.contains("@media print")) + #expect(html.contains("thead { display: table-header-group; }")) + } + + @Test("A read that fails is named in the file and in the warnings, and the export goes on") + func failedReadsAreReported() async throws { + let source = shopSource() + source.failingColumns = ["orgs"] + source.failingIndexes = ["users"] + let (document, failures) = try await readDocument([object("orgs"), object("users")], from: source) + let markdown = DataDictionaryMarkdownRenderer.render(document) + + #expect(markdown.contains("Columns could not be read: permission denied")) + #expect(failures.columns == ["orgs"]) + #expect(failures.indexes.count == 1) + #expect(failures.warnings.count == 2) + #expect(document.objects[1].columns.count == 2) + } + + @Test("Generated, identity and auto-increment columns say how their value is made") + func derivedDefaults() { + let generated = PluginColumnInfo( + name: "total", dataType: "numeric", generationExpression: "price * qty", generationKind: .stored) + let identity = PluginColumnInfo(name: "id", dataType: "bigint", identityKind: .always) + let autoIncrement = PluginColumnInfo(name: "id", dataType: "int", extra: "auto_increment") + + #expect(DataDictionaryDocument.defaultText(of: generated) == "GENERATED ALWAYS AS (price * qty) STORED") + #expect(DataDictionaryDocument.defaultText(of: identity) == "GENERATED ALWAYS AS IDENTITY") + #expect(DataDictionaryDocument.defaultText(of: autoIncrement) == "AUTO_INCREMENT") + } + + @Test("Stopping the export ends it between objects") + func cancellationStops() async throws { + let source = shopSource() + let progress = PluginExportProgress(progress: Progress(totalUnitCount: 0)) + source.onColumnRead = { table in + if table == "orgs" { progress.cancel() } + } + let plugin = DataDictionaryExportPlugin() + let destination = FileManager.default.temporaryDirectory + .appendingPathComponent("\(UUID().uuidString).md") + defer { try? FileManager.default.removeItem(at: destination) } + + await #expect(throws: PluginExportCancellationError.self) { + _ = try await plugin.export( + tables: [object("orgs"), object("users")], + dataSource: source, + destination: destination, + progress: progress + ) + } + #expect(!FileManager.default.fileExists(atPath: destination.path(percentEncoded: false))) + } +} diff --git a/apple/Plugins/FormatPluginTests/SQLExportReadsRowsTests.swift b/apple/Plugins/FormatPluginTests/SQLExportReadsRowsTests.swift new file mode 100644 index 0000000000..a5e3e29770 --- /dev/null +++ b/apple/Plugins/FormatPluginTests/SQLExportReadsRowsTests.swift @@ -0,0 +1,18 @@ +// +// SQLExportReadsRowsTests.swift +// FormatPluginTests +// + +import TableProPluginKit +import Testing + +struct SQLExportReadsRowsTests { + @Test("A SQL export reads a table's rows only with Data checked, and never a view's") + func readsRowsOnlyWithData() { + let plugin = SQLExportPlugin() + #expect(plugin.readsRows(optionValues: [true, true, true], kind: .table)) + #expect(!plugin.readsRows(optionValues: [true, true, false], kind: .table)) + #expect(!plugin.readsRows(optionValues: [true, true, true], kind: .view)) + #expect(plugin.readsRows(optionValues: [], kind: .table)) + } +} diff --git a/apple/Plugins/SQLExportPlugin/SQLExportPlugin.swift b/apple/Plugins/SQLExportPlugin/SQLExportPlugin.swift index cdf8f85f5d..832efd4521 100644 --- a/apple/Plugins/SQLExportPlugin/SQLExportPlugin.swift +++ b/apple/Plugins/SQLExportPlugin/SQLExportPlugin.swift @@ -100,6 +100,10 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi optionValues.contains(true) } + func readsRows(optionValues: [Bool], kind: PluginExportObjectKind) -> Bool { + kind.carriesRows && (optionValues.count <= 2 || optionValues[2]) + } + var currentFileExtension: String { settings.compressWithGzip ? "sql.gz" : "sql" } diff --git a/apple/Plugins/TableProPluginKit/ExportFormatPlugin.swift b/apple/Plugins/TableProPluginKit/ExportFormatPlugin.swift index 9e100646eb..557f67cc31 100644 --- a/apple/Plugins/TableProPluginKit/ExportFormatPlugin.swift +++ b/apple/Plugins/TableProPluginKit/ExportFormatPlugin.swift @@ -28,6 +28,9 @@ public protocol ExportFormatPlugin: TableProPlugin, Sendable { var currentFileExtension: String { get } + // The host counts rows before an export only for the objects whose rows the format writes. + func readsRows(optionValues: [Bool], kind: PluginExportObjectKind) -> Bool + func export( tables: [PluginExportTable], dataSource: any PluginExportDataSource, @@ -49,4 +52,5 @@ public extension ExportFormatPlugin { isTableExportable(optionValues: optionValues) } var currentFileExtension: String { Self.defaultFileExtension } + func readsRows(optionValues: [Bool], kind: PluginExportObjectKind) -> Bool { kind.carriesRows } } diff --git a/apple/Plugins/TableProPluginKit/PluginExportDataSource.swift b/apple/Plugins/TableProPluginKit/PluginExportDataSource.swift index ba42af17e9..52cfec4d06 100644 --- a/apple/Plugins/TableProPluginKit/PluginExportDataSource.swift +++ b/apple/Plugins/TableProPluginKit/PluginExportDataSource.swift @@ -72,6 +72,11 @@ public protocol PluginExportDataSource: AnyObject, Sendable { /// Mirrors the driver's creation scopes for a routine, trigger or event. Nil for every other kind. func fetchCreationScope(_ object: PluginExportTable) async throws -> PluginStatementScope? + + func fetchIndexes(table: String, databaseName: String) async throws -> [PluginIndexInfo] + + // Nil where the engine stores no comment on this kind of object, without asking the server. + func fetchObjectComment(_ object: PluginExportTable) async throws -> String? } public extension PluginExportDataSource { @@ -116,4 +121,8 @@ public extension PluginExportDataSource { func closeSessionScope(_ scope: PluginSessionScope) async throws {} func fetchCreationScope(_ object: PluginExportTable) async throws -> PluginStatementScope? { nil } + + func fetchIndexes(table: String, databaseName: String) async throws -> [PluginIndexInfo] { [] } + + func fetchObjectComment(_ object: PluginExportTable) async throws -> String? { nil } } diff --git a/apple/TablePro/Core/Database/DatabaseObjectToolEligibility.swift b/apple/TablePro/Core/Database/DatabaseObjectToolEligibility.swift index 07fca94080..4e91f18876 100644 --- a/apple/TablePro/Core/Database/DatabaseObjectToolEligibility.swift +++ b/apple/TablePro/Core/Database/DatabaseObjectToolEligibility.swift @@ -23,9 +23,6 @@ enum DatabaseObjectToolEligibility { @MainActor static func of(_ driver: DatabaseDriver?) -> Support { guard let driver else { return .none } - let commentable = TableInfo.TableType.allCases.filter { type in - driver.objectCommentStatement(name: "t", objectType: type.rawValue, schema: "s", comment: nil) != nil - } let droppable = UserDefinedTypeInfo.Kind.allCases.filter { kind in driver.generateDropTypeSQL(type: UserDefinedTypeInfo(name: "t", kind: kind, schema: "s")) != nil } @@ -33,10 +30,16 @@ enum DatabaseObjectToolEligibility { canRefreshMaterializedViews: driver.refreshMaterializedViewStatement( name: "t", schema: "s", concurrently: false ) != nil, - commentableTypes: Set(commentable), + commentableTypes: commentableTypes(of: driver), droppableTypeKinds: Set(droppable) ) } + + static func commentableTypes(of driver: DatabaseDriver) -> Set { + Set(TableInfo.TableType.allCases.filter { type in + driver.objectCommentStatement(name: "t", objectType: type.rawValue, schema: "s", comment: nil) != nil + }) + } } /// A view's source is its definition rather than its columns, which is what the DDL viewer diff --git a/apple/TablePro/Core/Menu/FileMenuBuilder.swift b/apple/TablePro/Core/Menu/FileMenuBuilder.swift index 3b1e84175a..7c01c4f0f6 100644 --- a/apple/TablePro/Core/Menu/FileMenuBuilder.swift +++ b/apple/TablePro/Core/Menu/FileMenuBuilder.swift @@ -270,6 +270,10 @@ enum FileMenuBuilder { String(localized: "Export Results…"), action: #selector(MainSplitViewController.exportQueryResults(_:)) ), + MenuItemFactory.item( + String(localized: "Export Data Dictionary…"), + action: #selector(MainSplitViewController.exportDataDictionary(_:)) + ), MenuItemFactory.separator, MenuItemFactory.item( String(localized: "Export Connections…"), diff --git a/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift b/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift index 4db3175243..0b8d20b6ab 100644 --- a/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift +++ b/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift @@ -25,6 +25,7 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable private let implicitSchemaName: String? private let pagination: PaginationCapability private let appliesRowScope: Bool + private let commentableKinds: Set private let cappedTables = OSAllocatedUnfairLock<[String]>(initialState: []) init(driver: DatabaseDriver, databaseType: DatabaseType) { @@ -34,6 +35,7 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable self.implicitSchemaName = snapshot?.schema.implicitSchemaName self.pagination = PaginationCapability.of(databaseType) self.appliesRowScope = Self.canApplyRowScope(on: databaseType) + self.commentableKinds = Self.commentableKinds(of: driver) self.driver = driver self.dbType = databaseType self.databaseTypeId = databaseType.rawValue @@ -43,6 +45,16 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable PluginMetadataRegistry.shared.snapshot(for: databaseType)?.editor.sqlDialect != nil } + // The same test the Structure tab's comment field uses, so the export never asks for a comment + // the app could not have shown or written. + static func commentableKinds(of driver: DatabaseDriver) -> Set { + let types = DatabaseObjectToolEligibility.Support.commentableTypes(of: driver) + let kinds: [(PluginExportObjectKind, TableInfo.TableType)] = [ + (.table, .table), (.view, .view), (.materializedView, .materializedView), (.foreignTable, .foreignTable) + ] + return Set(kinds.filter { types.contains($0.1) }.map(\.0)) + } + private var pluginDriver: (any PluginDatabaseDriver)? { (driver as? PluginDriverAdapter)?.schemaPluginDriver } @@ -253,6 +265,24 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable return try await pluginDriver.fetchCommentDDL(table: table, schema: exportSchema(for: databaseName)) } + func fetchIndexes(table: String, databaseName: String) async throws -> [PluginIndexInfo] { + guard let pluginDriver else { return [] } + return try await pluginDriver.fetchIndexes(table: table, schema: exportSchema(for: databaseName)) + } + + // Gated before the read: SQLite's table metadata counts rows, and it stores no comments. + func fetchObjectComment(_ object: PluginExportTable) async throws -> String? { + guard let pluginDriver, commentableKinds.contains(object.kind) else { return nil } + let metadata = try await pluginDriver.fetchTableMetadata( + table: object.name, + schema: exportSchema(for: object.databaseName) + ) + guard let comment = metadata.comment, + !comment.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + else { return nil } + return comment + } + // MARK: - Object DDL /// A driver addresses a routine, trigger or type through the info object it handed out, which diff --git a/apple/TablePro/Core/Services/Export/ExportFormatCatalog.swift b/apple/TablePro/Core/Services/Export/ExportFormatCatalog.swift index b43e89079d..ca1c45abdc 100644 --- a/apple/TablePro/Core/Services/Export/ExportFormatCatalog.swift +++ b/apple/TablePro/Core/Services/Export/ExportFormatCatalog.swift @@ -8,8 +8,10 @@ internal struct InstallableExportFormat: Equatable, Sendable { } internal enum ExportFormatCatalog { + internal static let dataDictionaryFormatId = "datadict" + private static let displayOrder = [ - "csv", "json", "sql", "xlsx", "md", "parquet", "mql" + "csv", "json", "sql", "xlsx", "md", "parquet", "mql", dataDictionaryFormatId ] internal static func installable(in manifest: RegistryManifest?) -> [InstallableExportFormat] { @@ -27,12 +29,19 @@ internal enum ExportFormatCatalog { installable(in: manifest).filter { !installedFormatIds.contains($0.formatId) } } + // A format that writes structure alone describes the catalog, so it needs objects to describe + // and an engine whose columns are declared rather than sampled from rows. internal static func available( _ plugins: [any ExportFormatPlugin], - forDatabaseTypeId databaseTypeId: String + forDatabaseTypeId databaseTypeId: String, + listsDatabaseObjects: Bool, + columnsAreSampled: Bool ) -> [any ExportFormatPlugin] { sorted(plugins.filter { plugin in let pluginType = type(of: plugin) + if writesStructureOnly(plugin), !listsDatabaseObjects || columnsAreSampled { + return false + } return accepts( databaseTypeId: databaseTypeId, supportedDatabaseTypeIds: pluginType.supportedDatabaseTypeIds, @@ -41,6 +50,10 @@ internal enum ExportFormatCatalog { }) } + internal static func writesStructureOnly(_ plugin: any ExportFormatPlugin) -> Bool { + !plugin.readsRows(optionValues: plugin.defaultTableOptionValues(), kind: .table) + } + internal static func accepts( databaseTypeId: String, supportedDatabaseTypeIds: [String], @@ -80,6 +93,8 @@ internal enum ExportFormatCatalog { case "md": String(localized: "Markdown tables. Paste into a README, an issue or a wiki.") case "parquet": String(localized: "Columnar format. Read by DuckDB, Spark, pandas and BigQuery.") case "mql": String(localized: "MongoDB query language. Use to import into MongoDB.") + case dataDictionaryFormatId: + String(localized: "Every column's type, key, default and comment, with indexes and foreign keys. Reads no rows.") default: nil } } diff --git a/apple/TablePro/Core/Services/Export/ExportService.swift b/apple/TablePro/Core/Services/Export/ExportService.swift index b1b6476c32..62f39fd5be 100644 --- a/apple/TablePro/Core/Services/Export/ExportService.swift +++ b/apple/TablePro/Core/Services/Export/ExportService.swift @@ -52,6 +52,9 @@ struct ExportState { var errorMessage: String? var warnings: [String] = [] + // False for an export that writes no rows, which then reports its progress in objects. + var readsRows = true + /// What the export wrote, as opposed to what went wrong with it. Kept apart from `warnings` /// because the success alert reads a non-empty `warnings` as a problem: it retitles itself, /// takes the caution icon, and drops its "Do not show this again" checkbox. @@ -71,7 +74,11 @@ final class ExportService: ObservableObject { private let effectiveQueryTimeoutSeconds: Int private let exportPlugin: @MainActor (String) -> (any ExportFormatPlugin)? - init(driver: DatabaseDriver, databaseType: DatabaseType) { + init( + driver: DatabaseDriver, + databaseType: DatabaseType, + exportPlugin: @escaping @MainActor (String) -> (any ExportFormatPlugin)? = ExportService.installedExportPlugin + ) { self.driver = driver self.databaseType = databaseType self.effectiveQueryTimeoutSeconds = (driver as? PluginDriverAdapter)?.effectiveQueryTimeoutSeconds @@ -79,7 +86,7 @@ final class ExportService: ObservableObject { configuredSeconds: driver.connection.queryTimeoutSeconds, globalSeconds: AppSettingsManager.shared.general.queryTimeoutSeconds ) - self.exportPlugin = Self.installedExportPlugin + self.exportPlugin = exportPlugin } /// Rows already in memory still need the engine that produced them: a SQL export has to quote @@ -188,8 +195,10 @@ final class ExportService: ObservableObject { let dataSource = ExportDataSourceAdapter(driver: driver, databaseType: databaseType) + let rowObjects = objects.filter { plugin.readsRows(optionValues: $0.optionValues, kind: $0.kind) } + state.readsRows = !rowObjects.isEmpty state.totalRows = await fetchTotalRowCount( - for: objects.filter { $0.kind.carriesRows }, + for: rowObjects, driver: driver, databaseType: databaseType, dataSource: dataSource diff --git a/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController+FileMenuActions.swift b/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController+FileMenuActions.swift index f0ed4d9466..88a81c2d4a 100644 --- a/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController+FileMenuActions.swift +++ b/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController+FileMenuActions.swift @@ -135,6 +135,10 @@ extension MainSplitViewController { commandActions?.exportTables() } + @objc func exportDataDictionary(_ sender: Any?) { + commandActions?.exportDataDictionary() + } + @objc func exportQueryResults(_ sender: Any?) { commandActions?.exportQueryResults() } diff --git a/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController+MenuValidation.swift b/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController+MenuValidation.swift index 57e9f6d788..04272a68cb 100644 --- a/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController+MenuValidation.swift +++ b/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController+MenuValidation.swift @@ -47,6 +47,7 @@ struct MenuValidationContext: Equatable { var isQueryTab = false /// Export Results exports the selected tab's rows, so an empty grid has nothing to offer. var hasResultRows = false + var canExportDataDictionary = false var isCurrentTabEditable = false /// Add Row and Duplicate Row stage `DEFAULT` for every column the server fills in, which only /// the table's own schema names. Until it lands, the result set's own metadata reports far less, @@ -215,6 +216,9 @@ extension MainSplitViewController: NSMenuItemValidation { case #selector(exportQueryResults(_:)): return context.isConnected && context.hasResultRows + case #selector(exportDataDictionary(_:)): + return context.isConnected && context.canExportDataDictionary + /// AppKit validated New Tab for free while it was its own selector. /// `NSWindow.validateUserInterfaceItem` only speaks to the native ones, so this is /// ours to enable and disable now. Close went back to `performClose:`, which every @@ -491,6 +495,7 @@ extension MainSplitViewController: NSMenuItemValidation { #selector(newEditorTab(_:)), #selector(openQuickSwitcher(_:)), #selector(exportTables(_:)), + #selector(exportDataDictionary(_:)), /// Both spellings of one command: the leaf that takes the driver's first format, and the /// row of the list that names another. One without the other would leave the list live over /// a leaf that is dim. @@ -629,6 +634,7 @@ extension MainSplitViewController: NSMenuItemValidation { canPresentHighlightRules: actions.canPresentHighlightRules, isQueryTab: actions.isQueryTab, hasResultRows: actions.hasResultRows, + canExportDataDictionary: actions.canExportDataDictionary, isCurrentTabEditable: actions.isCurrentTabEditable, isCurrentTabSchemaResolved: actions.isCurrentTabSchemaResolved, canRestorePreviousValues: actions.canRestorePreviousValues, diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index a24f0918ed..f75ffbd75c 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -51945,6 +51945,10 @@ } } }, + "Could not fetch foreign keys for: %@" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Could not fetch foreign keys, so foreign key constraints may be missing from the export." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -56550,6 +56554,10 @@ } } }, + "Data Dictionary" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Data File" : { "localizations" : { "fr" : { @@ -72669,6 +72677,9 @@ } } } + }, + "Every column's type, key, default and comment, with indexes and foreign keys. Reads no rows." : { + }, "Every connection using this profile asks for the password once each time TablePro runs." : { "localizations" : { @@ -75819,6 +75830,9 @@ } } } + }, + "Export Data Dictionary…" : { + }, "EXPORT DATABASE writes the whole database. There is no table filter." : { "localizations" : { @@ -104860,6 +104874,10 @@ } } }, + "Markdown has a table of contents for a README or a wiki. HTML is one file that opens in any browser and prints." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Markdown tables. Paste into a README, an issue or a wiki." : { "localizations" : { "fr" : { @@ -211423,6 +211441,10 @@ } } }, + "Write as" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Write capacity must be a whole number of at least 1" : { "extractionState" : "manual", "generatesSymbol" : false, diff --git a/apple/TablePro/Views/Export/ExportDialog.swift b/apple/TablePro/Views/Export/ExportDialog.swift index d060cd1bba..ee6dd298db 100644 --- a/apple/TablePro/Views/Export/ExportDialog.swift +++ b/apple/TablePro/Views/Export/ExportDialog.swift @@ -12,6 +12,7 @@ struct ExportDialog: View { @Binding var isPresented: Bool let mode: ExportMode var sidebarTables: [TableInfo] = [] + var initialFormatId: String? // MARK: - State @@ -138,8 +139,10 @@ struct ExportDialog: View { } .onAppear { let available = availableFormats - if let lastFormatId = TransferDialogStorage.shared.loadLastExportFormatId(), - available.contains(where: { type(of: $0).formatId == lastFormatId }) { + if let initialFormatId, available.contains(where: { type(of: $0).formatId == initialFormatId }) { + config.formatId = initialFormatId + } else if let lastFormatId = TransferDialogStorage.shared.loadLastExportFormatId(), + available.contains(where: { type(of: $0).formatId == lastFormatId }) { config.formatId = lastFormatId } else if !available.contains(where: { type(of: $0).formatId == config.formatId }), let first = available.first { @@ -206,7 +209,9 @@ struct ExportDialog: View { private var availableFormats: [any ExportFormatPlugin] { ExportFormatCatalog.available( pluginManager.allExportPlugins(), - forDatabaseTypeId: mode.formatDatabaseTypeId + forDatabaseTypeId: mode.formatDatabaseTypeId, + listsDatabaseObjects: mode.listsDatabaseObjects, + columnsAreSampled: databaseType.columnsAreSampled ) } @@ -1105,6 +1110,7 @@ private struct ExportProgressSheet: View { processedRows: service.state.processedRows, totalRows: service.state.totalRows, statusMessage: service.state.statusMessage, + readsRows: service.state.readsRows, onStop: onStop ) } diff --git a/apple/TablePro/Views/Export/ExportProgressView.swift b/apple/TablePro/Views/Export/ExportProgressView.swift index e63028ff82..17621cb479 100644 --- a/apple/TablePro/Views/Export/ExportProgressView.swift +++ b/apple/TablePro/Views/Export/ExportProgressView.swift @@ -19,6 +19,7 @@ struct ExportProgressView: View { let processedRows: Int let totalRows: Int let statusMessage: String + var readsRows = true let onStop: () -> Void @State private var showStopConfirmation = false @@ -58,14 +59,17 @@ struct ExportProgressView: View { Spacer() - if statusMessage.isEmpty { + if statusMessage.isEmpty, readsRows { Text(rowCountLabel) .font(.system(.body, design: .monospaced)) .foregroundStyle(.secondary) } } - if statusMessage.isEmpty, hasRowTotal { + if statusMessage.isEmpty, !readsRows, totalTables > 0 { + ProgressView(value: Double(min(tableIndex, totalTables)), total: Double(totalTables)) + .progressViewStyle(.linear) + } else if statusMessage.isEmpty, hasRowTotal { ProgressView(value: progressValue) .progressViewStyle(.linear) } else { diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+SidebarActions.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+SidebarActions.swift index 9b7eb35523..40bcac00ce 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+SidebarActions.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+SidebarActions.swift @@ -200,12 +200,20 @@ extension MainContentCoordinator { /// the dialog resolved `orders` against whichever container it considered current. func openExportDialog(preselectedTableNames: Set? = nil, scope: DatabaseContainerRef? = nil) { exportPreselection = preselectedTableNames.map { .tables(names: $0, scope: scope) } + exportFormatId = nil + activeSheet = .exportDialog + } + + func openExportDataDictionaryDialog() { + exportPreselection = nil + exportFormatId = ExportFormatCatalog.dataDictionaryFormatId activeSheet = .exportDialog } func openExportDialog(containers: [DatabaseContainerRef]) { guard !containers.isEmpty else { return } exportPreselection = .containers(containers) + exportFormatId = nil activeSheet = .exportDialog } diff --git a/apple/TablePro/Views/Main/Extensions/MainContentView+TransferSheets.swift b/apple/TablePro/Views/Main/Extensions/MainContentView+TransferSheets.swift index bc0e497108..438e264f9d 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentView+TransferSheets.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentView+TransferSheets.swift @@ -24,7 +24,8 @@ extension MainContentView { grouping: PluginManager.shared.databaseGroupingStrategy(for: connection.type) ) ), - sidebarTables: tables + sidebarTables: tables, + initialFormatId: coordinator.exportFormatId ) case .exportQueryResults: if let tab = coordinator.tabManager.selectedTab { diff --git a/apple/TablePro/Views/Main/MainContentCommandActions+DataDictionary.swift b/apple/TablePro/Views/Main/MainContentCommandActions+DataDictionary.swift new file mode 100644 index 0000000000..1a68ba6029 --- /dev/null +++ b/apple/TablePro/Views/Main/MainContentCommandActions+DataDictionary.swift @@ -0,0 +1,25 @@ +// +// MainContentCommandActions+DataDictionary.swift +// TablePro +// + +import Foundation + +extension MainContentCommandActions { + func exportDataDictionary() { + coordinator?.openExportDataDictionaryDialog() + } + + // Asks for the one plugin, since listing every format would load each lazily registered one. + var canExportDataDictionary: Bool { + guard let plugin = PluginManager.shared.exportPlugin(forFormat: ExportFormatCatalog.dataDictionaryFormatId) + else { return false } + let databaseType = currentDatabaseType + return !ExportFormatCatalog.available( + [plugin], + forDatabaseTypeId: databaseType.rawValue, + listsDatabaseObjects: true, + columnsAreSampled: databaseType.columnsAreSampled + ).isEmpty + } +} diff --git a/apple/TablePro/Views/Main/MainContentCoordinator.swift b/apple/TablePro/Views/Main/MainContentCoordinator.swift index a5df9ef068..13262dc2fd 100644 --- a/apple/TablePro/Views/Main/MainContentCoordinator.swift +++ b/apple/TablePro/Views/Main/MainContentCoordinator.swift @@ -309,6 +309,7 @@ final class MainContentCoordinator: ObservableObject { @Published var containerDropRequest: DatabaseDropRequest? @Published var importFile: ImportFileHandoff? @Published var exportPreselection: ExportPreselection? + @Published var exportFormatId: String? @Published var pendingLoadTrigger: TableLoadTrigger? var deferredRestoreLoadTabId: UUID? diff --git a/apple/TablePro/Views/Main/MainContentView.swift b/apple/TablePro/Views/Main/MainContentView.swift index 73ac1cd6a5..b54d2ae30a 100644 --- a/apple/TablePro/Views/Main/MainContentView.swift +++ b/apple/TablePro/Views/Main/MainContentView.swift @@ -182,6 +182,7 @@ struct MainContentView: View { if !$0 { coordinator.activeSheet = nil coordinator.exportPreselection = nil + coordinator.exportFormatId = nil /// Cleared on every dismissal, Cancel included. The request holds the closure /// that runs the rebuild, and that closure holds the structure view, which /// holds this coordinator; leaving it set after a cancel keeps the cycle alive diff --git a/apple/TableProTests/Core/Export/ExportFormatCatalogTests.swift b/apple/TableProTests/Core/Export/ExportFormatCatalogTests.swift index 42910274dd..594518434d 100644 --- a/apple/TableProTests/Core/Export/ExportFormatCatalogTests.swift +++ b/apple/TableProTests/Core/Export/ExportFormatCatalogTests.swift @@ -67,6 +67,17 @@ private final class StubAardvark: StubExportFormat, @unchecked Sendable { init() {} } +private final class StubDictionary: StubExportFormat, @unchecked Sendable { + static let pluginName = "Dictionary" + static let pluginVersion = "1.0.0" + static let pluginDescription = "Describes tables" + static let formatId = "datadict" + static let formatDisplayName = "Data Dictionary" + init() {} + + func readsRows(optionValues: [Bool], kind: PluginExportObjectKind) -> Bool { false } +} + struct ExportFormatCatalogTests { private func ids(_ plugins: [any ExportFormatPlugin]) -> [String] { plugins.map { type(of: $0).formatId } @@ -113,6 +124,24 @@ struct ExportFormatCatalogTests { #expect(forwards == ["csv", "sql", "parquet", "zebra"]) } + @Test("A format that writes structure alone is offered for database objects with declared columns") + func structureOnlyFormatNeedsDeclaredObjects() { + let plugins: [any ExportFormatPlugin] = [StubCSV(), StubDictionary()] + let objects = ExportFormatCatalog.available( + plugins, forDatabaseTypeId: "PostgreSQL", listsDatabaseObjects: true, columnsAreSampled: false) + let oneResult = ExportFormatCatalog.available( + plugins, forDatabaseTypeId: "PostgreSQL", listsDatabaseObjects: false, columnsAreSampled: false) + let sampled = ExportFormatCatalog.available( + plugins, forDatabaseTypeId: "MongoDB", listsDatabaseObjects: true, columnsAreSampled: true) + + #expect(ids(objects) == ["csv", "datadict"]) + #expect(ids(oneResult) == ["csv"]) + #expect(ids(sampled) == ["csv"]) + #expect(ExportFormatCatalog.writesStructureOnly(StubDictionary())) + #expect(!ExportFormatCatalog.writesStructureOnly(StubCSV())) + #expect(ExportFormatCatalog.description(for: StubDictionary()) != StubDictionary.pluginDescription) + } + private func manifest(_ plugins: [RegistryPlugin]) -> RegistryManifest { RegistryManifest(schemaVersion: RegistryManifest.supportedSchemaVersion, plugins: plugins) } diff --git a/apple/TableProTests/Core/Export/ExportServiceRowCountTests.swift b/apple/TableProTests/Core/Export/ExportServiceRowCountTests.swift new file mode 100644 index 0000000000..39e8d1a6ce --- /dev/null +++ b/apple/TableProTests/Core/Export/ExportServiceRowCountTests.swift @@ -0,0 +1,78 @@ +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private final class StructureAndDataFormat: ExportFormatPlugin, @unchecked Sendable { + static let pluginName = "Structure and Data" + static let pluginVersion = "1.0.0" + static let pluginDescription = "Writes structure, and rows when Data is checked" + static let formatId = "structure-and-data" + static let formatDisplayName = "Structure and Data" + static let defaultFileExtension = "txt" + static let iconName = "doc" + static let perTableOptionColumns = [ + PluginExportOptionColumn(id: "structure", label: "Structure", width: 40), + PluginExportOptionColumn(id: "data", label: "Data", width: 40) + ] + + required init() {} + + func defaultTableOptionValues() -> [Bool] { [true, true] } + + func readsRows(optionValues: [Bool], kind: PluginExportObjectKind) -> Bool { + kind.carriesRows && optionValues.count > 1 && optionValues[1] + } + + func export( + tables: [PluginExportTable], + dataSource: any PluginExportDataSource, + destination: URL, + progress: PluginExportProgress + ) async throws -> ExportFormatResult { + try Data().write(to: destination) + return ExportFormatResult() + } +} + +@MainActor +struct ExportServiceRowCountTests { + private func run(_ objects: [ExportObjectItem]) async throws -> (ExportService, [String]) { + let driver = MockDatabaseDriver(connection: TestFixtures.makeConnection(type: .postgresql)) + let format = StructureAndDataFormat() + let service = ExportService(driver: driver, databaseType: .postgresql) { formatId in + formatId == StructureAndDataFormat.formatId ? format : nil + } + var config = ExportConfiguration() + config.formatId = StructureAndDataFormat.formatId + let url = FileManager.default.temporaryDirectory.appendingPathComponent("\(UUID().uuidString).txt") + defer { try? FileManager.default.removeItem(at: url) } + try await service.export(objects: objects, config: config, to: url) + return (service, driver.executedQueries.filter { $0.contains("COUNT(") }) + } + + @Test("An export that writes no rows counts none, and reports its progress in objects") + func structureOnlyCountsNothing() async throws { + let (service, counts) = try await run([ + ExportObjectItem(name: "orders", kind: .table, optionValues: [true, false]), + ExportObjectItem(name: "customers", kind: .table, optionValues: [true, false]) + ]) + + #expect(counts.isEmpty) + #expect(!service.state.readsRows) + #expect(service.state.totalRows == 0) + } + + @Test("Only the tables whose rows are written are counted") + func onlyDataTablesAreCounted() async throws { + let (service, counts) = try await run([ + ExportObjectItem(name: "orders", kind: .table, optionValues: [true, true]), + ExportObjectItem(name: "customers", kind: .table, optionValues: [true, false]) + ]) + + #expect(counts.count == 1) + #expect(counts.first?.contains("orders") == true) + #expect(counts.first?.contains("customers") == false) + #expect(service.state.readsRows) + } +} diff --git a/apple/TableProTests/Core/Menu/ChromeSnapshot.json b/apple/TableProTests/Core/Menu/ChromeSnapshot.json index 687191d302..1d2c5cdd36 100644 --- a/apple/TableProTests/Core/Menu/ChromeSnapshot.json +++ b/apple/TableProTests/Core/Menu/ChromeSnapshot.json @@ -58,6 +58,7 @@ {"path": "File > Export"}, {"path": "File > Export > Export Tables…", "identifier": "shortcut.export", "action": "exportTables:", "key": "e", "modifiers": ["shift", "command"]}, {"path": "File > Export > Export Results…", "action": "exportQueryResults:"}, + {"path": "File > Export > Export Data Dictionary…", "action": "exportDataDictionary:"}, {"path": "File > Export", "separator": true}, {"path": "File > Export > Export Connections…", "action": "exportConnections:"}, {"path": "File", "separator": true}, diff --git a/apple/TableProTests/Core/Plugins/ExportDataSourceAdapterCatalogTests.swift b/apple/TableProTests/Core/Plugins/ExportDataSourceAdapterCatalogTests.swift new file mode 100644 index 0000000000..4ff9b7ea7e --- /dev/null +++ b/apple/TableProTests/Core/Plugins/ExportDataSourceAdapterCatalogTests.swift @@ -0,0 +1,68 @@ +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private final class CommentingStubDriver: PluginDriverStub, @unchecked Sendable { + let commentableTypes: Set + private(set) var metadataReads: [String] = [] + private(set) var indexReads: [String] = [] + + init(commentableTypes: Set) { + self.commentableTypes = commentableTypes + } + + func objectCommentStatement(name: String, objectType: String, schema: String?, comment: String?) -> String? { + commentableTypes.contains(objectType) ? "COMMENT ON \(objectType) \(name)" : nil + } + + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + metadataReads.append("\(schema ?? "").\(table)") + return PluginTableMetadata(tableName: table, comment: table == "blank" ? " " : "about \(table)") + } + + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { + indexReads.append("\(schema ?? "").\(table)") + return [PluginIndexInfo(name: "\(table)_pkey", columns: ["id"], isPrimary: true)] + } +} + +struct ExportDataSourceAdapterCatalogTests { + private func adapter(_ stub: CommentingStubDriver) -> ExportDataSourceAdapter { + let driver = PluginDriverAdapter(connection: TestFixtures.makeConnection(type: .postgresql), pluginDriver: stub) + return ExportDataSourceAdapter(driver: driver, databaseType: .postgresql) + } + + private func object(_ name: String, kind: PluginExportObjectKind) -> PluginExportTable { + PluginExportTable(name: name, databaseName: "sales", tableType: kind.rawValue, schema: "sales", kind: kind) + } + + @Test("A comment is read for the kinds the engine can comment on, in the object's own schema") + func commentIsReadWhereTheEngineStoresOne() async throws { + let stub = CommentingStubDriver(commentableTypes: ["TABLE"]) + let source = adapter(stub) + + #expect(try await source.fetchObjectComment(object("orders", kind: .table)) == "about orders") + #expect(try await source.fetchObjectComment(object("recent", kind: .view)) == nil) + #expect(try await source.fetchObjectComment(object("blank", kind: .table)) == nil) + #expect(stub.metadataReads == ["sales.orders", "sales.blank"]) + } + + @Test("An engine that stores no comments is never asked for table metadata, which may count rows") + func noCommentsMeansNoMetadataRead() async throws { + let stub = CommentingStubDriver(commentableTypes: []) + let source = adapter(stub) + + #expect(try await source.fetchObjectComment(object("orders", kind: .table)) == nil) + #expect(stub.metadataReads.isEmpty) + } + + @Test("Indexes are read from the driver in the container the export named") + func indexesComeFromTheNamedContainer() async throws { + let stub = CommentingStubDriver(commentableTypes: []) + let indexes = try await adapter(stub).fetchIndexes(table: "orders", databaseName: "sales") + + #expect(indexes.map { $0.name } == ["orders_pkey"]) + #expect(stub.indexReads == ["sales.orders"]) + } +} diff --git a/apple/TableProUITests/ExportUITests.swift b/apple/TableProUITests/ExportUITests.swift index a01b384b21..816fe2da1b 100644 --- a/apple/TableProUITests/ExportUITests.swift +++ b/apple/TableProUITests/ExportUITests.swift @@ -27,6 +27,59 @@ final class ExportUITests: UITestCase { } XCTAssertTrue(waitForPredicate(timeout: 10) { (format.value as? String) == "CSV" }, "The format must be CSV") + save(from: sheet, to: exported, in: app) + + XCTAssertTrue( + waitForPredicate(timeout: 30) { FileManager.default.fileExists(atPath: exported.path) }, + "Export must write \(exported.lastPathComponent) into the chosen folder" + ) + let lines = try String(contentsOf: exported, encoding: .utf8) + .split(separator: "\n") + .map(String.init) + XCTAssertEqual( + lines, + ["id,title,artist_id", "1,North,1", "2,South,2", "3,East,2", "4,West,3", "5,Up,4", "6,Down,5"], + "The file must hold album's header and every row" + ) + } + + func testExportingADataDictionaryWritesMarkdown() throws { + let launch = try launchWithFixture() + let window = launch.window + let app = launch.app + let destination = try XCTUnwrap(sandboxRoot) + let exported = destination.appendingPathComponent("album-dictionary.md") + + _ = openFromObjectBrowser("album", in: launch) + menuItem("Export Data Dictionary…", under: "File", in: app).click() + + let sheet = window.children(matching: .sheet).firstMatch + XCTAssertTrue( + sheet.staticTexts["1 table to export"].waitToExist(timeout: 30), + "Export Data Dictionary must open with the selected table, album, chosen" + ) + let format = element(sheet.popUpButtons, labelEndingWith: "Format") + XCTAssertTrue( + waitForPredicate(timeout: 10) { (format.value as? String) == "Data Dictionary" }, + "The sheet must open on the Data Dictionary format" + ) + save(from: sheet, to: exported, in: app) + + XCTAssertTrue( + waitForPredicate(timeout: 30) { FileManager.default.fileExists(atPath: exported.path) }, + "Export must write \(exported.lastPathComponent) into the chosen folder" + ) + let markdown = try String(contentsOf: exported, encoding: .utf8) + XCTAssertTrue(markdown.hasPrefix("# album-dictionary\n"), "The file must be titled after its name") + XCTAssertTrue(markdown.contains("- [`album`](#album)"), "The contents must link to album") + XCTAssertTrue(markdown.contains("## `album`"), "album must have its own section") + XCTAssertTrue(markdown.contains("| `title` | `TEXT` | No |"), "Each column must list its type and nullability") + XCTAssertFalse(markdown.contains("North"), "A data dictionary must hold no row") + } + + // Clicks Export…, names the file in the save panel, moves the panel to the file's folder and saves. + private func save(from sheet: XCUIElement, to exported: URL, in app: XCUIApplication) { + let destination = exported.deletingLastPathComponent() let export = sheet.buttons["Export…"].firstMatch XCTAssertTrue(waitForPredicate(timeout: 15) { export.exists && export.isEnabled }, "Export must be offered") export.click() @@ -56,22 +109,9 @@ final class ExportUITests: UITestCase { ) // With the extension hidden, the field shows the name without it. XCTAssertTrue( - (nameField.value as? String)?.hasPrefix("album-export") == true, + (nameField.value as? String)?.hasPrefix(exported.deletingPathExtension().lastPathComponent) == true, "Moving folder must keep the typed name, the field holds '\(nameField.value as? String ?? "nil")'" ) panel.buttons["OKButton"].firstMatch.click() - - XCTAssertTrue( - waitForPredicate(timeout: 30) { FileManager.default.fileExists(atPath: exported.path) }, - "Export must write \(exported.lastPathComponent) into the chosen folder" - ) - let lines = try String(contentsOf: exported, encoding: .utf8) - .split(separator: "\n") - .map(String.init) - XCTAssertEqual( - lines, - ["id,title,artist_id", "1,North,1", "2,South,2", "3,East,2", "4,West,3", "5,Up,4", "6,Down,5"], - "The file must hold album's header and every row" - ) } } diff --git a/apple/project.yml b/apple/project.yml index 497f6374ab..f347a3508e 100644 --- a/apple/project.yml +++ b/apple/project.yml @@ -368,6 +368,9 @@ targets: - target: MarkdownExport embed: true copy: { destination: plugins } + - target: DataDictionaryExport + embed: true + copy: { destination: plugins } - target: XLSXImport embed: true copy: { destination: plugins } @@ -637,6 +640,15 @@ targets: base: PRODUCT_BUNDLE_IDENTIFIER: com.TablePro.MarkdownExportPlugin + DataDictionaryExport: + templates: [DriverPlugin] + templateAttributes: + folder: DataDictionaryExportPlugin + principalClass: DataDictionaryExportPlugin + settings: + base: + PRODUCT_BUNDLE_IDENTIFIER: com.TablePro.DataDictionaryExportPlugin + XLSXImport: templates: [DriverPlugin] templateAttributes: @@ -782,7 +794,7 @@ targets: templateAttributes: folder: ClickHouseDriverPlugin - # The ten bundled format plugins in one bundle: none links a C library, and no two declare the + # The eleven bundled format plugins in one bundle: none links a C library, and no two declare the # same top-level name. SQLiteLinkage is for the round-trip tests, which need the SQLite the driver ships. FormatPluginTests: templates: [SQLiteLinkage] @@ -795,6 +807,9 @@ targets: - path: Plugins/CSVImportPlugin excludes: - Info.plist + - path: Plugins/DataDictionaryExportPlugin + excludes: + - Info.plist - path: Plugins/JSONExportPlugin excludes: - Info.plist @@ -1207,6 +1222,7 @@ aggregateTargets: - CSVExport - CSVImport - ClickHouseDriver + - DataDictionaryExport - JSONExport - JSONImport - MQLExport From 057fc7c8a07caf95b3dd119c73c4e89cbad983c0 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Wed, 21 Oct 2026 08:51:25 +0700 Subject: [PATCH 2/5] fix(export): read object comments from the catalog and keep failed reads in the dictionary --- .../DataDictionaryDocument.swift | 3 ++ .../DataDictionaryExportPlugin.swift | 18 +++++++----- .../DataDictionaryHTMLRenderer.swift | 10 +++++-- .../DataDictionaryMarkdownRenderer.swift | 10 +++++-- .../DuckDBDriverPlugin/DuckDBPlugin.swift | 9 ++++-- .../DataDictionaryExportTests.swift | 4 ++- .../PluginDatabaseDriver.swift | 7 +++++ .../Core/Database/DatabaseDriver.swift | 7 +++++ .../Core/Database/ObjectCommentEditing.swift | 5 ++-- .../Plugins/ExportDataSourceAdapter.swift | 8 ++---- .../Core/Plugins/PluginDriverAdapter.swift | 4 +++ .../Structure/TableStructureLoader.swift | 4 +-- .../ExportDataSourceAdapterCatalogTests.swift | 28 +++++++++++++++++++ 13 files changed, 93 insertions(+), 24 deletions(-) diff --git a/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryDocument.swift b/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryDocument.swift index 9e7bf4af49..0f9b784e74 100644 --- a/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryDocument.swift +++ b/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryDocument.swift @@ -15,6 +15,9 @@ struct DataDictionaryObject: Sendable { var indexes: [PluginIndexInfo] = [] var foreignKeys: [PluginForeignKeyInfo] = [] var columnsError: String? + var indexesError: String? + var foreignKeysError: String? + var commentError: String? var kindLabel: String { switch kind { diff --git a/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryExportPlugin.swift b/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryExportPlugin.swift index f86938b18e..43033f5ced 100644 --- a/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryExportPlugin.swift +++ b/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryExportPlugin.swift @@ -17,6 +17,8 @@ final class DataDictionaryExportPlugin: ObservableObject, ExportFormatPlugin, Se static let defaultFileExtension = "md" static let iconName = "book.closed" static let supportedObjectKinds: [PluginExportObjectKind] = [.table, .view, .materializedView, .foreignTable] + // Both read items to list a table's fields, though their snapshots do not mark the columns as sampled. + static let excludedDatabaseTypeIds = ["DynamoDB", "SurrealDB"] typealias Settings = DataDictionaryExportOptions static let settingsStorageId = "datadict" @@ -137,13 +139,13 @@ enum DataDictionaryReader { failures.columns.append(table.qualifiedName) } - if table.kind == .table || table.kind == .materializedView { - do { - object.indexes = try await dataSource.fetchIndexes(table: table.name, databaseName: table.databaseName) - } catch { - try rethrowIfCancelled(error, progress: progress) - failures.indexes.append(table.qualifiedName) - } + // Every kind, as the Structure tab reads them: SQL Server indexes a view. + do { + object.indexes = try await dataSource.fetchIndexes(table: table.name, databaseName: table.databaseName) + } catch { + try rethrowIfCancelled(error, progress: progress) + object.indexesError = error.localizedDescription + failures.indexes.append(table.qualifiedName) } if table.kind == .table { @@ -151,6 +153,7 @@ enum DataDictionaryReader { object.foreignKeys = try await dataSource.fetchForeignKeys(table: table.name, databaseName: table.databaseName) } catch { try rethrowIfCancelled(error, progress: progress) + object.foreignKeysError = error.localizedDescription failures.foreignKeys.append(table.qualifiedName) } } @@ -159,6 +162,7 @@ enum DataDictionaryReader { object.comment = try await dataSource.fetchObjectComment(table) } catch { try rethrowIfCancelled(error, progress: progress) + object.commentError = error.localizedDescription failures.comments.append(table.qualifiedName) } return object diff --git a/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryHTMLRenderer.swift b/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryHTMLRenderer.swift index bdff96a027..d90fcc4a2a 100644 --- a/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryHTMLRenderer.swift +++ b/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryHTMLRenderer.swift @@ -51,6 +51,8 @@ enum DataDictionaryHTMLRenderer { html += "

\(escape(object.kindLabel))." if let comment = object.comment { html += " \(escape(comment))" + } else if let error = object.commentError { + html += " Comment could not be read: \(escape(error))" } html += "

\n" @@ -63,7 +65,9 @@ enum DataDictionaryHTMLRenderer { } let indexes = document.indexRows(at: index) - if !indexes.isEmpty { + if let error = object.indexesError { + html += "

Indexes

\n

Indexes could not be read: \(escape(error))

\n" + } else if !indexes.isEmpty { html += "

Indexes

\n" html += table( header: ["Name", "Columns", "Kind", "Method"], @@ -72,7 +76,9 @@ enum DataDictionaryHTMLRenderer { } let keys = document.foreignKeyRows(at: index) - if !keys.isEmpty { + if let error = object.foreignKeysError { + html += "

Foreign keys

\n

Foreign keys could not be read: \(escape(error))

\n" + } else if !keys.isEmpty { html += "

Foreign keys

\n" html += table( header: ["Name", "Columns", "References", "On delete", "On update"], diff --git a/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryMarkdownRenderer.swift b/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryMarkdownRenderer.swift index 1133a1cb25..4e85c09867 100644 --- a/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryMarkdownRenderer.swift +++ b/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryMarkdownRenderer.swift @@ -29,6 +29,8 @@ enum DataDictionaryMarkdownRenderer { var lines = ["", "", "## \(code(document.displayName(at: index)))", ""] if let comment = object.comment { lines.append("**\(object.kindLabel).** \(text(comment))") + } else if let error = object.commentError { + lines.append("**\(object.kindLabel).** Comment could not be read: \(text(error))") } else { lines.append("**\(object.kindLabel).**") } @@ -43,7 +45,9 @@ enum DataDictionaryMarkdownRenderer { } let indexes = document.indexRows(at: index) - if !indexes.isEmpty { + if let error = object.indexesError { + lines += ["### Indexes", "", "Indexes could not be read: \(text(error))", ""] + } else if !indexes.isEmpty { lines += ["### Indexes", ""] lines += table( header: ["Name", "Columns", "Kind", "Method"], @@ -53,7 +57,9 @@ enum DataDictionaryMarkdownRenderer { } let keys = document.foreignKeyRows(at: index) - if !keys.isEmpty { + if let error = object.foreignKeysError { + lines += ["### Foreign keys", "", "Foreign keys could not be read: \(text(error))", ""] + } else if !keys.isEmpty { lines += ["### Foreign keys", ""] lines += table( header: ["Name", "Columns", "References", "On delete", "On update"], diff --git a/apple/Plugins/DuckDBDriverPlugin/DuckDBPlugin.swift b/apple/Plugins/DuckDBDriverPlugin/DuckDBPlugin.swift index 04f033a9d7..1ac473cc38 100644 --- a/apple/Plugins/DuckDBDriverPlugin/DuckDBPlugin.swift +++ b/apple/Plugins/DuckDBDriverPlugin/DuckDBPlugin.swift @@ -953,7 +953,7 @@ final class DuckDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { guard let row = countResult.rows.first, let firstCell = row.first else { return nil } return Int64(firstCell.asText ?? "0") }() - let comment = try await fetchObjectComment(table: table, schema: schemaName) + let comment = try await catalogComment(table: table, schema: schemaName) return PluginTableMetadata( tableName: table, @@ -963,7 +963,12 @@ final class DuckDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { ) } - private func fetchObjectComment(table: String, schema: String) async throws -> String? { + // fetchTableMetadata probes up to 100,001 rows; the comment alone is a catalog read. + func fetchObjectComment(table: String, schema: String?) async throws -> String? { + try await catalogComment(table: table, schema: resolveSchema(schema)) + } + + private func catalogComment(table: String, schema: String) async throws -> String? { let result = try await executeParameterized( query: DuckDBSchemaQueries.objectComment, parameters: [.text(try requireCatalog()), .text(schema), .text(table)] diff --git a/apple/Plugins/FormatPluginTests/DataDictionaryExportTests.swift b/apple/Plugins/FormatPluginTests/DataDictionaryExportTests.swift index 6b0633d4b1..62d33dadd9 100644 --- a/apple/Plugins/FormatPluginTests/DataDictionaryExportTests.swift +++ b/apple/Plugins/FormatPluginTests/DataDictionaryExportTests.swift @@ -136,6 +136,7 @@ struct DataDictionaryExportTests { for kind in DataDictionaryExportPlugin.supportedObjectKinds { #expect(!plugin.readsRows(optionValues: [], kind: kind)) } + #expect(Set(DataDictionaryExportPlugin.excludedDatabaseTypeIds) == ["DynamoDB", "SurrealDB"]) #expect(DataDictionaryFileType.markdown.fileExtension == "md") #expect(DataDictionaryFileType.html.fileExtension == "html") } @@ -148,7 +149,7 @@ struct DataDictionaryExportTests { #expect(source.rowReads == 0) #expect(failures.warnings.isEmpty) - #expect(source.indexReads == ["orgs", "users"]) + #expect(source.indexReads == ["orgs", "users", "active_users"]) #expect(source.foreignKeyReads == ["orgs", "users"]) #expect(document.objects.map { $0.columns.count } == [2, 2, 1]) #expect(document.objects[0].comment == "Companies that own accounts") @@ -276,6 +277,7 @@ struct DataDictionaryExportTests { let markdown = DataDictionaryMarkdownRenderer.render(document) #expect(markdown.contains("Columns could not be read: permission denied")) + #expect(markdown.contains("### Indexes\n\nIndexes could not be read: permission denied")) #expect(failures.columns == ["orgs"]) #expect(failures.indexes.count == 1) #expect(failures.warnings.count == 2) diff --git a/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift b/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift index aa870998d1..ec8f0c51a2 100644 --- a/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift +++ b/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift @@ -174,6 +174,9 @@ public protocol PluginDatabaseDriver: AnyObject, Sendable { /// inline. A driver that answers here must not also declare the same comments in /// `fetchTableDDL`, or the dump sets each one twice. func fetchCommentDDL(table: String, schema: String?) async throws -> [String] + + // A driver whose fetchTableMetadata reads rows answers this from the catalog alone. + func fetchObjectComment(table: String, schema: String?) async throws -> String? func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] func fetchTriggers(table: String, schema: String?) async throws -> [PluginTriggerInfo] func fetchCheckConstraints(table: String, schema: String?) async throws -> [PluginCheckConstraintInfo] @@ -968,6 +971,10 @@ public extension PluginDatabaseDriver { /// statements from `fetchTableDDL` in the same change. func fetchCommentDDL(table: String, schema: String?) async throws -> [String] { [] } + func fetchObjectComment(table: String, schema: String?) async throws -> String? { + try await fetchTableMetadata(table: table, schema: schema).comment + } + /// Answers whether `fetchAllIndexes` is a single query rather than the N+1 default below. var providesBulkIndexFetch: Bool { false } diff --git a/apple/TablePro/Core/Database/DatabaseDriver.swift b/apple/TablePro/Core/Database/DatabaseDriver.swift index 3e23301fcb..a4f391af4d 100644 --- a/apple/TablePro/Core/Database/DatabaseDriver.swift +++ b/apple/TablePro/Core/Database/DatabaseDriver.swift @@ -258,6 +258,9 @@ protocol DatabaseDriver: AnyObject, Sendable { /// Fetch table metadata (size, comment, engine, etc.) func fetchTableMetadata(tableName: String) async throws -> TableMetadata + // The comment alone, which some engines answer without the row probe their metadata runs. + func fetchObjectComment(tableName: String) async throws -> String? + /// Fetch list of all databases on the server func fetchDatabases() async throws -> [String] @@ -478,6 +481,10 @@ extension DatabaseDriver { func fetchCommentDDL(table: String) async throws -> [String] { [] } + func fetchObjectComment(tableName: String) async throws -> String? { + try await fetchTableMetadata(tableName: tableName).comment + } + func resolveQueryCompletionProfile( databaseTypeId: String, base: QueryCompletionProfile diff --git a/apple/TablePro/Core/Database/ObjectCommentEditing.swift b/apple/TablePro/Core/Database/ObjectCommentEditing.swift index a5dd7caabe..657c2ea247 100644 --- a/apple/TablePro/Core/Database/ObjectCommentEditing.swift +++ b/apple/TablePro/Core/Database/ObjectCommentEditing.swift @@ -29,10 +29,9 @@ enum ObjectCommentEditing { /// predate a change made in another window or another app. static func currentComment(of target: DatabaseObjectTarget) async throws -> String? { let name = target.name - let metadata = try await DatabaseManager.shared.withMetadataDriver(scope: target.scope) { driver in - try await driver.fetchTableMetadata(tableName: name) + return try await DatabaseManager.shared.withMetadataDriver(scope: target.scope) { driver in + try await driver.fetchObjectComment(tableName: name) } - return metadata.comment } static func statement(for comment: String?, on target: DatabaseObjectTarget, driver: DatabaseDriver) -> String? { diff --git a/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift b/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift index 0b8d20b6ab..0f0d3cc92f 100644 --- a/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift +++ b/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift @@ -270,16 +270,14 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable return try await pluginDriver.fetchIndexes(table: table, schema: exportSchema(for: databaseName)) } - // Gated before the read: SQLite's table metadata counts rows, and it stores no comments. + // Gated before the read: an engine that stores no comments may still count rows to answer. func fetchObjectComment(_ object: PluginExportTable) async throws -> String? { guard let pluginDriver, commentableKinds.contains(object.kind) else { return nil } - let metadata = try await pluginDriver.fetchTableMetadata( + let comment = try await pluginDriver.fetchObjectComment( table: object.name, schema: exportSchema(for: object.databaseName) ) - guard let comment = metadata.comment, - !comment.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty - else { return nil } + guard let comment, !comment.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return nil } return comment } diff --git a/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift b/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift index e20e9db75a..935a842191 100644 --- a/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift +++ b/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift @@ -686,6 +686,10 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor return TableMetadata(pluginMeta) } + func fetchObjectComment(tableName: String) async throws -> String? { + try await pluginDriver.fetchObjectComment(table: tableName, schema: pluginDriver.currentSchema) + } + func fetchDatabases() async throws -> [String] { try await pluginDriver.fetchDatabases() } diff --git a/apple/TablePro/Views/Structure/TableStructureLoader.swift b/apple/TablePro/Views/Structure/TableStructureLoader.swift index 452d2a9cc5..3c2c381aaa 100644 --- a/apple/TablePro/Views/Structure/TableStructureLoader.swift +++ b/apple/TablePro/Views/Structure/TableStructureLoader.swift @@ -63,7 +63,7 @@ struct TableStructureLoader { func tableComment() async throws -> String? { let table = tableName - return try await perform { try await $0.fetchTableMetadata(tableName: table).comment } + return try await perform { try await $0.fetchObjectComment(tableName: table) } } func concurrentRefreshAvailability() async throws -> PluginConcurrentRefreshAvailability? { @@ -83,7 +83,7 @@ struct TableStructureLoader { var comment: MetadataFetchOutcome? if includingTableComment { do { - comment = .fetched(try await driver.fetchTableMetadata(tableName: table).comment) + comment = .fetched(try await driver.fetchObjectComment(tableName: table)) } catch is CancellationError { comment = .cancelled } catch { diff --git a/apple/TableProTests/Core/Plugins/ExportDataSourceAdapterCatalogTests.swift b/apple/TableProTests/Core/Plugins/ExportDataSourceAdapterCatalogTests.swift index 4ff9b7ea7e..86e565e183 100644 --- a/apple/TableProTests/Core/Plugins/ExportDataSourceAdapterCatalogTests.swift +++ b/apple/TableProTests/Core/Plugins/ExportDataSourceAdapterCatalogTests.swift @@ -27,6 +27,23 @@ private final class CommentingStubDriver: PluginDriverStub, @unchecked Sendable } } +private final class CatalogCommentStubDriver: PluginDriverStub, @unchecked Sendable { + private(set) var metadataReads = 0 + + func objectCommentStatement(name: String, objectType: String, schema: String?, comment: String?) -> String? { + "COMMENT ON \(objectType) \(name)" + } + + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + metadataReads += 1 + return PluginTableMetadata(tableName: table, rowCount: 100_001, comment: "from metadata") + } + + func fetchObjectComment(table: String, schema: String?) async throws -> String? { + "from the catalog" + } +} + struct ExportDataSourceAdapterCatalogTests { private func adapter(_ stub: CommentingStubDriver) -> ExportDataSourceAdapter { let driver = PluginDriverAdapter(connection: TestFixtures.makeConnection(type: .postgresql), pluginDriver: stub) @@ -57,6 +74,17 @@ struct ExportDataSourceAdapterCatalogTests { #expect(stub.metadataReads.isEmpty) } + @Test("A driver that reads its comment from the catalog is never asked for table metadata") + func catalogCommentSkipsTheMetadataRead() async throws { + let stub = CatalogCommentStubDriver() + let driver = PluginDriverAdapter(connection: TestFixtures.makeConnection(type: .duckdb), pluginDriver: stub) + let source = ExportDataSourceAdapter(driver: driver, databaseType: .duckdb) + + #expect(try await source.fetchObjectComment(object("orders", kind: .table)) == "from the catalog") + #expect(try await driver.fetchObjectComment(tableName: "orders") == "from the catalog") + #expect(stub.metadataReads == 0) + } + @Test("Indexes are read from the driver in the container the export named") func indexesComeFromTheNamedContainer() async throws { let stub = CommentingStubDriver(commentableTypes: []) From 9176b87561c277ba81041c2920a2cf43d4025bf0 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Wed, 21 Oct 2026 08:57:23 +0700 Subject: [PATCH 3/5] fix(export): keep row progress for view exports and hide row scopes for the dictionary --- .../DataDictionaryDocument.swift | 3 +- .../DataDictionaryExportTests.swift | 2 +- .../ExportFormatPlugin.swift | 5 ++- .../Core/Services/Export/ExportService.swift | 2 +- .../Views/Export/ExportObjectTreeView.swift | 7 ++- .../Export/ExportServiceRowCountTests.swift | 44 ++++++++++++++++--- 6 files changed, 52 insertions(+), 11 deletions(-) diff --git a/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryDocument.swift b/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryDocument.swift index 0f9b784e74..cac64f681d 100644 --- a/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryDocument.swift +++ b/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryDocument.swift @@ -212,8 +212,9 @@ struct DataDictionaryDocument { let storage = column.generationKind.map { " \($0.rawValue)" } ?? "" return "GENERATED ALWAYS AS (\(expression))\(storage)" } - if let identity = column.identityKind { return "GENERATED \(identity.rawValue) AS IDENTITY" } + // MySQL reports an auto-increment column with an identity kind too, and calls it by its own name. if column.extra?.lowercased().contains("auto_increment") == true { return "AUTO_INCREMENT" } + if let identity = column.identityKind { return "GENERATED \(identity.rawValue) AS IDENTITY" } return nil } diff --git a/apple/Plugins/FormatPluginTests/DataDictionaryExportTests.swift b/apple/Plugins/FormatPluginTests/DataDictionaryExportTests.swift index 62d33dadd9..6e4f73e0e6 100644 --- a/apple/Plugins/FormatPluginTests/DataDictionaryExportTests.swift +++ b/apple/Plugins/FormatPluginTests/DataDictionaryExportTests.swift @@ -289,7 +289,7 @@ struct DataDictionaryExportTests { let generated = PluginColumnInfo( name: "total", dataType: "numeric", generationExpression: "price * qty", generationKind: .stored) let identity = PluginColumnInfo(name: "id", dataType: "bigint", identityKind: .always) - let autoIncrement = PluginColumnInfo(name: "id", dataType: "int", extra: "auto_increment") + let autoIncrement = PluginColumnInfo(name: "id", dataType: "int", extra: "auto_increment", identityKind: .byDefault) #expect(DataDictionaryDocument.defaultText(of: generated) == "GENERATED ALWAYS AS (price * qty) STORED") #expect(DataDictionaryDocument.defaultText(of: identity) == "GENERATED ALWAYS AS IDENTITY") diff --git a/apple/Plugins/TableProPluginKit/ExportFormatPlugin.swift b/apple/Plugins/TableProPluginKit/ExportFormatPlugin.swift index 557f67cc31..352a18ac7f 100644 --- a/apple/Plugins/TableProPluginKit/ExportFormatPlugin.swift +++ b/apple/Plugins/TableProPluginKit/ExportFormatPlugin.swift @@ -28,7 +28,8 @@ public protocol ExportFormatPlugin: TableProPlugin, Sendable { var currentFileExtension: String { get } - // The host counts rows before an export only for the objects whose rows the format writes. + // Whether the export streams this object's rows. The host counts rows up front only for these, + // and reports progress in objects when none is read. func readsRows(optionValues: [Bool], kind: PluginExportObjectKind) -> Bool func export( @@ -52,5 +53,5 @@ public extension ExportFormatPlugin { isTableExportable(optionValues: optionValues) } var currentFileExtension: String { Self.defaultFileExtension } - func readsRows(optionValues: [Bool], kind: PluginExportObjectKind) -> Bool { kind.carriesRows } + func readsRows(optionValues: [Bool], kind: PluginExportObjectKind) -> Bool { true } } diff --git a/apple/TablePro/Core/Services/Export/ExportService.swift b/apple/TablePro/Core/Services/Export/ExportService.swift index 62f39fd5be..2f605f40e0 100644 --- a/apple/TablePro/Core/Services/Export/ExportService.swift +++ b/apple/TablePro/Core/Services/Export/ExportService.swift @@ -198,7 +198,7 @@ final class ExportService: ObservableObject { let rowObjects = objects.filter { plugin.readsRows(optionValues: $0.optionValues, kind: $0.kind) } state.readsRows = !rowObjects.isEmpty state.totalRows = await fetchTotalRowCount( - for: rowObjects, + for: rowObjects.filter { $0.kind.carriesRows }, driver: driver, databaseType: databaseType, dataSource: dataSource diff --git a/apple/TablePro/Views/Export/ExportObjectTreeView.swift b/apple/TablePro/Views/Export/ExportObjectTreeView.swift index e9e3bd717a..31ac4afeaf 100644 --- a/apple/TablePro/Views/Export/ExportObjectTreeView.swift +++ b/apple/TablePro/Views/Export/ExportObjectTreeView.swift @@ -329,7 +329,7 @@ internal final class ExportObjectTreeCoordinator: NSObject, NSOutlineViewDataSou return AnyView( ExportTreeObjectRow( object: object, - offersRowScope: object.offersRowScope(on: owner.databaseType), + offersRowScope: object.offersRowScope(on: owner.databaseType) && formatReadsRows, optionColumns: optionColumns, supportsOption: supportsOption, setSelected: { [weak self] isSelected in @@ -364,6 +364,11 @@ internal final class ExportObjectTreeCoordinator: NSObject, NSOutlineViewDataSou PluginManager.shared.exportPlugin(forFormat: formatId) } + // A format that writes structure alone ignores a row scope, so it offers none. + private var formatReadsRows: Bool { + plugin.map { !ExportFormatCatalog.writesStructureOnly($0) } ?? true + } + private var optionColumns: [PluginExportOptionColumn] { guard let plugin else { return [] } return type(of: plugin).perTableOptionColumns diff --git a/apple/TableProTests/Core/Export/ExportServiceRowCountTests.swift b/apple/TableProTests/Core/Export/ExportServiceRowCountTests.swift index 39e8d1a6ce..e1eb618f07 100644 --- a/apple/TableProTests/Core/Export/ExportServiceRowCountTests.swift +++ b/apple/TableProTests/Core/Export/ExportServiceRowCountTests.swift @@ -35,16 +35,41 @@ private final class StructureAndDataFormat: ExportFormatPlugin, @unchecked Senda } } +private final class RowsOnlyFormat: ExportFormatPlugin, @unchecked Sendable { + static let pluginName = "Rows Only" + static let pluginVersion = "1.0.0" + static let pluginDescription = "Streams the rows of every object it is handed" + static let formatId = "rows-only" + static let formatDisplayName = "Rows Only" + static let defaultFileExtension = "txt" + static let iconName = "doc" + + required init() {} + + func export( + tables: [PluginExportTable], + dataSource: any PluginExportDataSource, + destination: URL, + progress: PluginExportProgress + ) async throws -> ExportFormatResult { + try Data().write(to: destination) + return ExportFormatResult() + } +} + @MainActor struct ExportServiceRowCountTests { - private func run(_ objects: [ExportObjectItem]) async throws -> (ExportService, [String]) { + private func run( + _ objects: [ExportObjectItem], + format: any ExportFormatPlugin = StructureAndDataFormat() + ) async throws -> (ExportService, [String]) { let driver = MockDatabaseDriver(connection: TestFixtures.makeConnection(type: .postgresql)) - let format = StructureAndDataFormat() - let service = ExportService(driver: driver, databaseType: .postgresql) { formatId in - formatId == StructureAndDataFormat.formatId ? format : nil + let formatId = type(of: format).formatId + let service = ExportService(driver: driver, databaseType: .postgresql) { requested in + requested == formatId ? format : nil } var config = ExportConfiguration() - config.formatId = StructureAndDataFormat.formatId + config.formatId = formatId let url = FileManager.default.temporaryDirectory.appendingPathComponent("\(UUID().uuidString).txt") defer { try? FileManager.default.removeItem(at: url) } try await service.export(objects: objects, config: config, to: url) @@ -63,6 +88,15 @@ struct ExportServiceRowCountTests { #expect(service.state.totalRows == 0) } + @Test("A row format exporting views reads rows, though a view is never counted up front") + func viewsAreReadButNotCounted() async throws { + let (service, counts) = try await run( + [ExportObjectItem(name: "recent_orders", kind: .view)], format: RowsOnlyFormat()) + + #expect(counts.isEmpty) + #expect(service.state.readsRows) + } + @Test("Only the tables whose rows are written are counted") func onlyDataTablesAreCounted() async throws { let (service, counts) = try await run([ From 1d68cf71520103e49293ed6cfd754ade04e03bc5 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Wed, 21 Oct 2026 09:07:58 +0700 Subject: [PATCH 4/5] fix(export): keep index prefix lengths and pass the new catalog arguments in tests --- .../DataDictionaryDocument.swift | 4 +++- .../DataDictionaryExportTests.swift | 12 ++++++++++++ .../Core/Export/DataSourceExportTests.swift | 7 ++++++- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryDocument.swift b/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryDocument.swift index cac64f681d..69522ea50b 100644 --- a/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryDocument.swift +++ b/apple/Plugins/DataDictionaryExportPlugin/DataDictionaryDocument.swift @@ -141,7 +141,9 @@ struct DataDictionaryDocument { func indexRows(at index: Int) -> [DataDictionaryIndexRow] { objects[index].indexes.map { info in - var keys = info.columns.joined(separator: ", ") + var keys = info.columns.map { column in + info.columnPrefixes?[column].map { "\(column)(\($0))" } ?? column + }.joined(separator: ", ") if let included = info.includedColumns, !included.isEmpty { keys += " INCLUDE (\(included.joined(separator: ", ")))" } diff --git a/apple/Plugins/FormatPluginTests/DataDictionaryExportTests.swift b/apple/Plugins/FormatPluginTests/DataDictionaryExportTests.swift index 6e4f73e0e6..ab4f5b43bc 100644 --- a/apple/Plugins/FormatPluginTests/DataDictionaryExportTests.swift +++ b/apple/Plugins/FormatPluginTests/DataDictionaryExportTests.swift @@ -284,6 +284,18 @@ struct DataDictionaryExportTests { #expect(document.objects[1].columns.count == 2) } + @Test("An index on a column prefix names the prefix length") + func indexPrefixLengths() async throws { + let source = CatalogOnlyDataSource(databaseTypeId: "MySQL") + source.columns = ["users": [PluginColumnInfo(name: "email", dataType: "varchar(255)")]] + source.indexes = ["users": [ + PluginIndexInfo(name: "uq_email", columns: ["email"], isUnique: true, columnPrefixes: ["email": 10]) + ]] + let (document, _) = try await readDocument([object("users", schema: "shop")], from: source) + + #expect(document.indexRows(at: 0).first?.columns == "email(10)") + } + @Test("Generated, identity and auto-increment columns say how their value is made") func derivedDefaults() { let generated = PluginColumnInfo( diff --git a/apple/TableProTests/Core/Export/DataSourceExportTests.swift b/apple/TableProTests/Core/Export/DataSourceExportTests.swift index 67a5937b6f..5f8f572dbf 100644 --- a/apple/TableProTests/Core/Export/DataSourceExportTests.swift +++ b/apple/TableProTests/Core/Export/DataSourceExportTests.swift @@ -73,7 +73,12 @@ struct DataSourceExportTests { private func formatIds(for databaseTypeId: String) -> [String] { ExportFormatCatalog - .available([StubMQLFormat(), StubSQLFormat(), StubCSVFormat()], forDatabaseTypeId: databaseTypeId) + .available( + [StubMQLFormat(), StubSQLFormat(), StubCSVFormat()], + forDatabaseTypeId: databaseTypeId, + listsDatabaseObjects: false, + columnsAreSampled: false + ) .map { type(of: $0).formatId } } From cce46c2a257f9bafe5f8e3ca86f617b0e6721fd5 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Wed, 21 Oct 2026 09:09:06 +0700 Subject: [PATCH 5/5] docs(changelog): add the data dictionary export fragments --- apple/changelog.d/3735.added.md | 1 + apple/changelog.d/3735.fixed.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 apple/changelog.d/3735.added.md create mode 100644 apple/changelog.d/3735.fixed.md diff --git a/apple/changelog.d/3735.added.md b/apple/changelog.d/3735.added.md new file mode 100644 index 0000000000..9d65a02b20 --- /dev/null +++ b/apple/changelog.d/3735.added.md @@ -0,0 +1 @@ +File > Export > Export Data Dictionary… writes every table's columns, keys, indexes and comments as Markdown or HTML. diff --git a/apple/changelog.d/3735.fixed.md b/apple/changelog.d/3735.fixed.md new file mode 100644 index 0000000000..31a3be0497 --- /dev/null +++ b/apple/changelog.d/3735.fixed.md @@ -0,0 +1 @@ +A SQL export with Structure only counted the rows of every table before it started.