From 32d2d8c95db665be64414f45f967d316d449ec60 Mon Sep 17 00:00:00 2001 From: ZhuchkaTriplesix Date: Sat, 17 Oct 2026 06:27:34 +0300 Subject: [PATCH] perf(export): hand large copies to the isolate in chunks (#1537) formatAsync ran one Isolate.run whose closure captured all rows. The captured object graph is copied on the calling isolate before the worker starts, so copying 50,000 x 20 cells to the clipboard stopped the UI for about 20-30 ms. Above about 20,000 cells (handOffCells) the rows now go over in chunks of that size, each formatted on its own isolate, with the event loop running between them; the parts are joined. formatChunk produces each part so that the join is exactly the format* output: the header (and JSON's '[') in the first chunk, ']' in the last, the JSON items split out of the indented array. The isolate calls live in separate functions so a closure captures only its chunk, not formatAsync's whole rows. Standalone Dart AOT, 50,000 x 20, Timer.periodic(1 ms) on the caller: longest gap 21 ms with one closure, 2-4 ms with 1,000-row chunks; total time ~85 ms -> ~55 ms. --- lib/shared/services/data_export_service.dart | 130 +++++++++++++++++-- test/shared/data_export_service_test.dart | 56 ++++++++ 2 files changed, 174 insertions(+), 12 deletions(-) diff --git a/lib/shared/services/data_export_service.dart b/lib/shared/services/data_export_service.dart index ac282c80..66f341f6 100644 --- a/lib/shared/services/data_export_service.dart +++ b/lib/shared/services/data_export_service.dart @@ -1,8 +1,10 @@ import 'dart:convert'; import 'dart:io'; import 'dart:isolate'; +import 'dart:math' as math; import 'package:file_selector/file_selector.dart'; +import 'package:flutter/foundation.dart' show visibleForTesting; import 'package:flutter/services.dart' show Clipboard, ClipboardData; import 'package:querya_desktop/core/csv/result_grid_csv.dart'; @@ -74,27 +76,37 @@ class DataExportService { ) { if (columns.isEmpty) return ''; final buf = StringBuffer(); + _writeMarkdownHeader(buf, columns); + _writeMarkdownRows(buf, columns, rows); + return buf.toString(); + } - String escapeMd(String s) { - if (s == 'NULL') return '`NULL`'; - return s - .replaceAll('|', '\\|') - .replaceAll('\r\n', ' ') - .replaceAll('\n', ' '); - } + static String _escapeMd(String s) { + if (s == 'NULL') return '`NULL`'; + return s + .replaceAll('|', '\\|') + .replaceAll('\r\n', ' ') + .replaceAll('\n', ' '); + } - buf.write('| ${columns.map(escapeMd).join(' | ')} |\n'); + static void _writeMarkdownHeader(StringBuffer buf, List columns) { + buf.write('| ${columns.map(_escapeMd).join(' | ')} |\n'); buf.write('| ${columns.map((_) => '---').join(' | ')} |\n'); + } + static void _writeMarkdownRows( + StringBuffer buf, + List columns, + List> rows, + ) { for (final row in rows) { final padded = List.generate( columns.length, - (i) => i < row.length ? escapeMd(row[i]) : '`NULL`', + (i) => i < row.length ? _escapeMd(row[i]) : '`NULL`', growable: false, ); buf.write('| ${padded.join(' | ')} |\n'); } - return buf.toString(); } /// Formats rows as SQL INSERT statements (`INSERT INTO table (...) VALUES (...);`). @@ -147,13 +159,52 @@ class DataExportService { await Clipboard.setData(ClipboardData(text: text)); } - /// Formats data on a background isolate to prevent UI stutter on large datasets. + /// About how many cells one isolate hand-off in [formatAsync] carries. + /// + /// The rows a closure captures are copied on the calling isolate before the + /// worker starts: 50,000 x 20 cells in one closure stopped the UI for + /// ~20-30 ms; 20,000 cells take a few ms, and the event loop runs between + /// chunks (#1537). + @visibleForTesting + static const int handOffCells = 20000; + + /// Formats data on background isolates to prevent UI stutter on large + /// datasets. Large inputs go over in chunks of about [handOffCells] cells; + /// the result is the same text as the `format*` functions produce. static Future formatAsync( DataExportFormat format, { required List columns, required List> rows, String tableName = 'export_table', - }) { + }) async { + final width = math.max(1, columns.length); + if (columns.isEmpty || rows.length * width <= handOffCells) { + return _formatInIsolate(format, columns, rows, tableName); + } + final chunkRows = math.max(1, handOffCells ~/ width); + final parts = []; + for (var start = 0; start < rows.length; start += chunkRows) { + final end = math.min(start + chunkRows, rows.length); + parts.add(await _formatChunkInIsolate( + format, + columns, + rows.sublist(start, end), + tableName, + first: start == 0, + last: end == rows.length, + )); + } + return parts.join(); + } + + // Separate functions so each closure captures only its own arguments: a + // closure created inside [formatAsync] would carry the whole `rows` along. + static Future _formatInIsolate( + DataExportFormat format, + List columns, + List> rows, + String tableName, + ) { return Isolate.run(() { switch (format) { case DataExportFormat.csv: @@ -168,6 +219,61 @@ class DataExportService { }); } + static Future _formatChunkInIsolate( + DataExportFormat format, + List columns, + List> rows, + String tableName, { + required bool first, + required bool last, + }) { + return Isolate.run(() => formatChunk( + format, + columns, + rows, + tableName, + first: first, + last: last, + )); + } + + /// The part of the whole document that the non-empty [rows] make, when + /// they are a chunk of a larger table: joined in order, the chunks give + /// exactly what the `format*` function gives for all the rows. [first] + /// adds the header (and JSON's `[`), [last] JSON's `]`. + @visibleForTesting + static String formatChunk( + DataExportFormat format, + List columns, + List> rows, + String tableName, { + required bool first, + required bool last, + }) { + switch (format) { + case DataExportFormat.csv: + if (first) return resultGridAsCsv(columns, rows); + final buf = StringBuffer(); + for (final row in rows) { + buf.write('\n'); + buf.write(formatCsvDataRow(row, columns.length)); + } + return buf.toString(); + case DataExportFormat.json: + // An indented array is `[\n`, the items joined by `,\n`, `\n]`. + final whole = formatJson(columns, rows); + final items = whole.substring(2, whole.length - 2); + return '${first ? '[\n' : ',\n'}$items${last ? '\n]' : ''}'; + case DataExportFormat.markdown: + final buf = StringBuffer(); + if (first) _writeMarkdownHeader(buf, columns); + _writeMarkdownRows(buf, columns, rows); + return buf.toString(); + case DataExportFormat.sqlDump: + return formatSqlInsertDump(tableName, columns, rows); + } + } + /// Streams formatted output to [sink] without assembling one giant [String]. static Future writeToSink( IOSink sink, diff --git a/test/shared/data_export_service_test.dart b/test/shared/data_export_service_test.dart index aaacc30e..f018f382 100644 --- a/test/shared/data_export_service_test.dart +++ b/test/shared/data_export_service_test.dart @@ -133,4 +133,60 @@ void main() { expect(md, DataExportService.formatMarkdownTable(columns, rows)); }); }); + + group('formatAsync in chunks (#1537)', () { + const columns = ['id', 'name | pipe', 'note']; + List> table(int n) => [ + for (var r = 0; r < n; r++) + [ + '$r', + r % 7 == 0 ? 'NULL' : 'n"a,me\n$r', + if (r % 5 != 0) "it's | $r", + ], + ]; + + String whole(DataExportFormat format, List> rows) => + switch (format) { + DataExportFormat.csv => DataExportService.formatCsv(columns, rows), + DataExportFormat.json => DataExportService.formatJson(columns, rows), + DataExportFormat.markdown => + DataExportService.formatMarkdownTable(columns, rows), + DataExportFormat.sqlDump => + DataExportService.formatSqlInsertDump('t', columns, rows), + }; + + for (final format in DataExportFormat.values) { + test('${format.name}: chunks joined give the whole document', () { + final rows = table(23); + for (final size in [1, 3, 7, 22, 23]) { + final parts = []; + for (var start = 0; start < rows.length; start += size) { + final end = + start + size < rows.length ? start + size : rows.length; + parts.add(DataExportService.formatChunk( + format, + columns, + rows.sublist(start, end), + 't', + first: start == 0, + last: end == rows.length, + )); + } + expect(parts.join(), whole(format, rows), reason: 'size $size'); + } + }); + + test('${format.name}: formatAsync over the hand-off size', () async { + final rows = + table(DataExportService.handOffCells ~/ columns.length * 2 + 5); + final text = await DataExportService.formatAsync( + format, + columns: columns, + rows: rows, + tableName: 't', + ); + expect(text, whole(format, rows)); + }); + } + }); }