feat(data-grid): mark invisible characters in cells and value fields

apple/Packages/TableProEditor/Sources/TableProTextEngine/SpecialCharacters/SpecialCharacter.swift

@@ -25,9 +25,22 @@ public extension ClassifiedSpecialCharacter {
 }
 
 public extension SpecialCharacter {
-    static func classify(in string: NSString, at index: Int) -> ClassifiedSpecialCharacter? {
+    // `marksLineBreaks` is for a view showing a value on one line, which draws no break for CR or LF.
+    static func classify(
+        in string: NSString,
+        at index: Int,
+        marksLineBreaks: Bool = false
+    ) -> ClassifiedSpecialCharacter? {
         guard index >= 0, index < string.length else { return nil }
-        guard mayBeSpecial(string.character(at: index)) else { return nil }
+        let unit = string.character(at: index)
+        if marksLineBreaks, lineBreakControls.contains(UInt32(unit)), let scalar = Unicode.Scalar(unit) {
+            return ClassifiedSpecialCharacter(
+                character: .marker(label: controlLabel(for: scalar.value)),
+                range: NSRange(location: index, length: 1),
+                scalar: scalar
+            )
+        }
+        guard mayBeSpecial(unit) else { return nil }
         guard let scalar = scalar(in: string, at: index) else { return nil }
         let range = NSRange(location: index, length: scalar.utf16.count)
         guard let character = classify(scalar, in: string, range: range) else { return nil }
@@ -59,6 +72,8 @@ public extension SpecialCharacter {
 private extension SpecialCharacter {
     static let preservedControls: Set<UInt32> = [0x09, 0x0A, 0x0D]
 
+    static let lineBreakControls: Set<UInt32> = [0x0A, 0x0D]
+
     static let combiningIgnorables: Set<UInt32> = [
         0x034F, 0x115F, 0x1160, 0x17B4, 0x17B5, 0x180B, 0x180C, 0x180D, 0x180F
     ]

apple/Packages/TableProEditor/Sources/TableProTextEngine/SpecialCharacters/SpecialCharacterDisplay.swift

@@ -46,7 +46,11 @@ private extension SpecialCharacterDisplay {
             var index = 0
             while index < text.length {
                 guard marks.count < style.maximumMarksPerLine else { return index }
-                guard let classified = SpecialCharacter.classify(in: text, at: index) else {
+                guard let classified = SpecialCharacter.classify(
+                    in: text,
+                    at: index,
+                    marksLineBreaks: style.marksLineBreaks
+                ) else {
                     index += 1
                     continue
                 }

apple/Packages/TableProEditor/Sources/TableProTextEngine/SpecialCharacters/SpecialCharacterLine.swift

@@ -0,0 +1,93 @@
+//
+//  SpecialCharacterLine.swift
+//  TableProTextEngine
+//
+
+import AppKit
+
+// The editor's marks for a view that draws one line itself, such as a grid cell.
+public struct SpecialCharacterLine {
+    public let line: CTLine
+    public let marks: [SpecialCharacterMark]
+
+    public init(_ string: NSAttributedString, style: SpecialCharacterStyle) {
+        let display = SpecialCharacterDisplay.make(from: string, style: style)
+        let line = CTLineCreateWithAttributedString(display.string)
+        self.line = line
+        marks = SpecialCharacterGeometry.positioned(display.marks, in: line)
+    }
+
+    // `drawn` is this line or an end truncation of it. A right-to-left line moves what it keeps, so
+    // the marks are placed again, and the ones the ellipsis replaced are dropped.
+    public func marks(drawnIn drawn: CTLine) -> [SpecialCharacterMark] {
+        guard drawn !== line, !marks.isEmpty else { return marks }
+        let length = CTLineGetStringRange(line).length
+        let runs = CTLineGetGlyphRuns(drawn) as? [CTRun] ?? []
+        let cut = runs.map(CTRunGetStringRange)
+            .filter { $0.location + $0.length == length }
+            .map(\.location)
+            .min() ?? length
+        let kept = marks.filter { $0.offset + $0.length <= cut }
+        return SpecialCharacterGeometry.positioned(kept, in: drawn)
+    }
+
+    public static func draw(
+        _ marks: [SpecialCharacterMark],
+        fromX x: CGFloat,
+        top: CGFloat,
+        height: CGFloat,
+        in context: CGContext
+    ) {
+        for mark in marks {
+            SpecialCharacterMarkRenderer.draw(
+                mark,
+                from: x + mark.minX,
+                to: x + mark.maxX,
+                top: top,
+                height: height,
+                color: mark.color,
+                in: context
+            )
+        }
+    }
+}
+
+// One mark as an inline attachment, for a text view that swaps it in for its character.
+public struct SpecialCharacterMarkImage {
+    public let image: NSImage
+    public let bounds: CGRect
+
+    public init(_ classified: ClassifiedSpecialCharacter, in text: NSString, font: NSFont, color: NSColor) {
+        let width: CGFloat
+        switch classified.character {
+        case .marker(let label):
+            width = SpecialCharacterMetrics.markerWidth(label: label, font: font)
+        case .blankSpace:
+            let space = NSAttributedString(string: text.substring(with: classified.range), attributes: [.font: font])
+            width = CGFloat(CTLineGetTypographicBounds(CTLineCreateWithAttributedString(space), nil, nil, nil))
+        }
+        let height = font.ascender - font.descender
+        bounds = CGRect(x: 0, y: font.descender, width: width, height: height)
+
+        let mark = SpecialCharacterMark(
+            offset: 0,
+            length: classified.range.length,
+            character: classified.character,
+            font: font,
+            color: color
+        )
+        image = NSImage(size: NSSize(width: width, height: height), flipped: true) { rect in
+            guard let context = NSGraphicsContext.current?.cgContext else { return false }
+            SpecialCharacterMarkRenderer.draw(
+                mark,
+                from: rect.minX,
+                to: rect.maxX,
+                top: rect.minY,
+                height: rect.height,
+                color: color,
+                in: context
+            )
+            return true
+        }
+    }
+}

apple/Packages/TableProEditor/Sources/TableProTextEngine/SpecialCharacters/SpecialCharacterStyle.swift

@@ -8,10 +8,12 @@ import AppKit
 public struct SpecialCharacterStyle: Equatable {
     public var color: NSColor
     public var maximumMarksPerLine: Int
+    public var marksLineBreaks: Bool
 
-    public init(color: NSColor = .systemOrange, maximumMarksPerLine: Int = 4_096) {
+    public init(color: NSColor = .systemOrange, maximumMarksPerLine: Int = 4_096, marksLineBreaks: Bool = false) {
         self.color = color
         self.maximumMarksPerLine = maximumMarksPerLine
+        self.marksLineBreaks = marksLineBreaks
     }
 }
 

apple/Packages/TableProEditor/Tests/TableProTextEngineTests/SpecialCharacterLineTests.swift

@@ -0,0 +1,133 @@
+import AppKit
+@testable import TableProTextEngine
+import Testing
+
+@MainActor
+struct SpecialCharacterLineTests {
+    private let font = NSFont.monospacedSystemFont(ofSize: 13, weight: .regular)
+
+    private func attributed(_ text: String) -> NSAttributedString {
+        NSAttributedString(string: text, attributes: [.font: font, .foregroundColor: NSColor.black])
+    }
+
+    private func width(of line: CTLine) -> CGFloat {
+        CGFloat(CTLineGetTypographicBounds(line, nil, nil, nil))
+    }
+
+    @Test("A zero-width space takes the room of its marker, right after the text before it")
+    func markerReservesItsWidth() throws {
+        let marked = SpecialCharacterLine(attributed("abc\u{200B}"), style: SpecialCharacterStyle())
+        let plain = CTLineCreateWithAttributedString(attributed("abc"))
+        let mark = try #require(marked.marks.first)
+
+        #expect(marked.marks.count == 1)
+        #expect(mark.character == .marker(label: "ZWSP"))
+        #expect(abs(mark.minX - width(of: plain)) < 0.5)
+        #expect(mark.maxX - mark.minX > 10)
+        #expect(abs(width(of: marked.line) - mark.maxX) < 0.5)
+    }
+
+    @Test("CR and LF are marked only on a single-line surface")
+    func lineBreaksMarkedOnRequest() {
+        let text = "a\rb\nc" as NSString
+
+        #expect(SpecialCharacter.classify(in: text, at: 1) == nil)
+        #expect(SpecialCharacter.classify(in: text, at: 3) == nil)
+        #expect(SpecialCharacter.classify(in: text, at: 1, marksLineBreaks: true)?.character == .marker(label: "CR"))
+        #expect(SpecialCharacter.classify(in: text, at: 3, marksLineBreaks: true)?.character == .marker(label: "LF"))
+        #expect(SpecialCharacter.classify(in: "\t" as NSString, at: 0, marksLineBreaks: true) == nil)
+
+        let line = SpecialCharacterLine(attributed("abc\r"), style: SpecialCharacterStyle(marksLineBreaks: true))
+        #expect(line.marks.map { $0.character } == [.marker(label: "CR")])
+    }
+
+    @Test("A right-to-left override no longer reorders the text after it")
+    func bidiOverrideIsNeutralised() throws {
+        let line = SpecialCharacterLine(attributed("a\u{202E}bc"), style: SpecialCharacterStyle())
+        var offsets: [Int: CGFloat] = [:]
+        CTLineEnumerateCaretOffsets(line.line) { offset, index, leading, _ in
+            if leading, offsets[index] == nil { offsets[index] = offset }
+        }
+
+        let b = try #require(offsets[2])
+        let c = try #require(offsets[3])
+        #expect(b < c)
+    }
+
+    @Test("A truncated line keeps the marks of what it shows and drops the ones the ellipsis replaced")
+    func truncationDropsHiddenMarks() throws {
+        let tail = SpecialCharacterLine(attributed(String(repeating: "x", count: 40) + "\u{200B}"), style: SpecialCharacterStyle())
+        let head = SpecialCharacterLine(attributed("\u{200B}" + String(repeating: "x", count: 40)), style: SpecialCharacterStyle())
+        let ellipsis = CTLineCreateWithAttributedString(attributed("\u{2026}"))
+        let tailCut = try #require(CTLineCreateTruncatedLine(tail.line, 120, .end, ellipsis))
+        let headCut = try #require(CTLineCreateTruncatedLine(head.line, 120, .end, ellipsis))
+
+        #expect(tail.marks(drawnIn: tailCut).isEmpty)
+        #expect(head.marks(drawnIn: headCut) == head.marks)
+        #expect(head.marks(drawnIn: head.line) == head.marks)
+    }
+
+    @Test("A truncated right-to-left line places its marks where the kept text moved to")
+    func truncatedRightToLeftMarksMove() throws {
+        let text = "\u{05E9}\u{05DC}\u{05D5}\u{05DD}\u{00A0}" + String(repeating: "\u{05E2}\u{05D5}\u{05DC}\u{05DD} ", count: 8)
+        let marked = SpecialCharacterLine(attributed(text), style: SpecialCharacterStyle())
+        let ellipsis = CTLineCreateWithAttributedString(attributed("\u{2026}"))
+        let cut = try #require(CTLineCreateTruncatedLine(marked.line, 120, .end, ellipsis))
+        let full = try #require(marked.marks.first)
+        let shown = try #require(marked.marks(drawnIn: cut).first)
+        var caret: CGFloat?
+        CTLineEnumerateCaretOffsets(cut) { offset, index, _, _ in
+            if index == 4 { caret = min(caret ?? offset, offset) }
+        }
+
+        #expect(abs(shown.minX - full.minX) > 50)
+        #expect(abs(shown.minX - (caret ?? -1)) < 0.5)
+        #expect(shown.maxX <= 120)
+    }
+
+    @Test("Drawing a line's marks inks its boxes")
+    func marksAreDrawn() {
+        let line = SpecialCharacterLine(attributed("\u{200B}"), style: SpecialCharacterStyle(color: .black))
+
+        #expect(inkedPixels(line.marks) > 0)
+        #expect(inkedPixels([]) == 0)
+    }
+
+    @Test("A marker image is as wide as the marker in a line, a special space as the space itself")
+    func markImageWidths() throws {
+        let text = "\u{200B}\u{00A0}" as NSString
+        let marker = try #require(SpecialCharacter.classify(in: text, at: 0))
+        let space = try #require(SpecialCharacter.classify(in: text, at: 1))
+        let line = SpecialCharacterLine(attributed("\u{200B}"), style: SpecialCharacterStyle())
+        let lineMark = try #require(line.marks.first)
+        let spaceWidth = width(of: CTLineCreateWithAttributedString(attributed("\u{00A0}")))
+
+        let markerImage = SpecialCharacterMarkImage(marker, in: text, font: font, color: .black)
+        let spaceImage = SpecialCharacterMarkImage(space, in: text, font: font, color: .black)
+
+        #expect(abs(markerImage.bounds.width - (lineMark.maxX - lineMark.minX)) < 0.5)
+        #expect(abs(spaceImage.bounds.width - spaceWidth) < 0.5)
+        #expect(markerImage.bounds.minY == font.descender)
+        #expect(markerImage.image.size == markerImage.bounds.size)
+    }
+
+    private func inkedPixels(_ marks: [SpecialCharacterMark]) -> Int {
+        let width = 80
+        let height = 24
+        guard let context = CGContext(
+            data: nil,
+            width: width,
+            height: height,
+            bitsPerComponent: 8,
+            bytesPerRow: width * 4,
+            space: CGColorSpaceCreateDeviceRGB(),
+            bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue
+        ) else { return 0 }
+        context.translateBy(x: 0, y: CGFloat(height))
+        context.scaleBy(x: 1, y: -1)
+        SpecialCharacterLine.draw(marks, fromX: 0, top: 2, height: 18, in: context)
+        guard let data = context.data else { return 0 }
+        let bytes = data.bindMemory(to: UInt8.self, capacity: width * height * 4)
+        return (0 ..< width * height).filter { bytes[$0 * 4 + 3] > 0 }.count
+    }
+}

apple/TablePro/Core/Services/Formatting/CellDisplayFormatter.swift

@@ -56,13 +56,13 @@ enum CellDisplayFormatter {
 
     /// A row is one line, so anything a format produces is capped and stripped of line breaks
     /// before it reaches a cell. Binary read as text is the case that needs it: a blob carries
-    /// newlines and runs to megabytes where hex never did.
+    /// newlines and runs to megabytes where hex never did. A lone CR stays for the cell to mark.
     private static func fitToCell(_ value: String) -> String {
         var displayValue = value
         let nsDisplay = displayValue as NSString
         if nsDisplay.length > maxDisplayLength {
             displayValue = nsDisplay.substring(to: maxDisplayLength) + "…"
         }
-        return displayValue.sanitizedForCellDisplay
+        return displayValue.flattenedForCell
     }
 }

apple/TablePro/Core/Utilities/Text/RevealedText.swift

@@ -24,8 +24,8 @@ internal struct RevealedText: Equatable, Sendable {
 
     internal let segments: [Segment]
 
-    internal init(_ text: String) {
-        segments = Self.segments(of: text as NSString)
+    internal init(_ text: String, marksLineBreaks: Bool = false) {
+        segments = Self.segments(of: text as NSString, marksLineBreaks: marksLineBreaks)
     }
 
     internal var revealsAnyCharacter: Bool {
@@ -53,12 +53,12 @@ internal struct RevealedText: Equatable, Sendable {
 }
 
 private extension RevealedText {
-    static func segments(of text: NSString) -> [Segment] {
+    static func segments(of text: NSString, marksLineBreaks: Bool) -> [Segment] {
         var segments: [Segment] = []
         var runStart = 0
         var index = 0
         while index < text.length {
-            guard let classified = SpecialCharacter.classify(in: text, at: index) else {
+            guard let classified = SpecialCharacter.classify(in: text, at: index, marksLineBreaks: marksLineBreaks) else {
                 index += 1
                 continue
             }

apple/TablePro/Models/Query/ColumnValueFilterSelection.swift

@@ -99,7 +99,7 @@ internal extension ColumnDistinctValue {
     /// What the popover lists and what its search matches, so "null" finds the NULL entry.
     var label: String {
         if isNull { return String(localized: "(NULL)") }
-        return display.isEmpty ? String(localized: "(Empty)") : display
+        return display.isEmpty ? String(localized: "(Empty)") : display.shownWithoutMarks
     }
 }
 

apple/TablePro/Models/Settings/AppSettings.swift

@@ -125,6 +125,7 @@ struct DataGridSettings: Codable, Equatable {
     var defaultSortBehavior: DefaultSortBehavior
     var defaultSortDirection: SortDirection
     var rowCopyFormat: RowCopyFormat
+    var showInvisibleCharacters: Bool
 
     static let `default` = DataGridSettings(
         rowHeight: .normal,
@@ -140,7 +141,8 @@ struct DataGridSettings: Codable, Equatable {
         truncateQueryResults: true,
         defaultSortBehavior: .none,
         defaultSortDirection: .ascending,
-        rowCopyFormat: .rows
+        rowCopyFormat: .rows,
+        showInvisibleCharacters: true
     )
 
     init(
@@ -157,7 +159,8 @@ struct DataGridSettings: Codable, Equatable {
         truncateQueryResults: Bool = true,
         defaultSortBehavior: DefaultSortBehavior = .none,
         defaultSortDirection: SortDirection = .ascending,
-        rowCopyFormat: RowCopyFormat = .rows
+        rowCopyFormat: RowCopyFormat = .rows,
+        showInvisibleCharacters: Bool = true
     ) {
         self.rowHeight = rowHeight
         self.dateFormat = dateFormat
@@ -173,6 +176,7 @@ struct DataGridSettings: Codable, Equatable {
         self.defaultSortBehavior = defaultSortBehavior
         self.defaultSortDirection = defaultSortDirection
         self.rowCopyFormat = rowCopyFormat
+        self.showInvisibleCharacters = showInvisibleCharacters
     }
 
     init(from decoder: Decoder) throws {
@@ -197,6 +201,9 @@ struct DataGridSettings: Codable, Equatable {
         defaultSortBehavior = container.decodeSetting(.defaultSortBehavior, default: fallback.defaultSortBehavior)
         defaultSortDirection = container.decodeSetting(.defaultSortDirection, default: fallback.defaultSortDirection)
         rowCopyFormat = container.decodeSetting(.rowCopyFormat, default: fallback.rowCopyFormat)
+        showInvisibleCharacters = container.decodeSetting(
+            .showInvisibleCharacters, default: fallback.showInvisibleCharacters
+        )
     }
 
     // MARK: - Validated Properties