diff --git a/cpp/README.md b/cpp/README.md index 9b5c79e2..86eb668f 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -526,13 +526,13 @@ on Windows 10 (22H2), 16 logical CPUs @ 3.39 GHz, MSVC 19.51 (Release), Google B | Benchmark | RealExcel.xlsb (100 rows) | 65K_Records_Data.xlsb (65,535 rows) | |---|---:|---:| -| `open` (from memory) | 23.4 µs | 1.17 ms | -| `parse` (4 or 6 bound columns) | 44.8 µs | 23.3 ms | -| `parse_batches`, batches of 1,000 / 10,000 rows | - | 22.4 ms / 22.9 ms | -| `infer_schema` (sample 100 / 1,000 rows) | 68.8 µs | 572.7 µs | +| `open` (from memory) | 22.7 µs | 1.23 ms | +| `parse` (4 or 6 bound columns) | 44.6 µs | 23.8 ms | +| `parse_batches`, batches of 1,000 / 10,000 rows | - | 22.6 ms / 23.1 ms | +| `infer_schema` (sample 100 / 1,000 rows) | 72.4 µs | 607.0 µs | `open` here is the byte-span overload, which copies the caller's buffer before returning (the caller may free -it afterwards), so it scales with file size: 1.17 ms is mostly copying the 3.7 MB fixture. Opening by +it afterwards), so it scales with file size: 1.23 ms is mostly copying the 3.7 MB fixture. Opening by path reads only the package directory and stays nearly flat — see the Rust suite below. `parse` scales linearly with rows × columns, at roughly 360 ns/row here. Draining the same sheet through the batched typed reader costs the same as one whole-sheet `parse` while holding @@ -549,12 +549,12 @@ sides decode every cell into an owned value (`std::string` for text, not the zer | Library | Mean | |---|---:| -| ExcelReader (`parse`) | 61.3 ms | -| DuckDB (`read_xlsx`, summing every column in SQL) | 413.8 ms | -| xlsxio (`xlsxioread_sheet_next_cell_*`) | 497.4 ms | -| xlnt (`worksheet::rows()` + `cell::to_string()`/`value()`) | 2,335.0 ms | +| ExcelReader (`parse`) | 58.0 ms | +| DuckDB (`read_xlsx`, summing every column in SQL) | 417.8 ms | +| xlsxio (`xlsxioread_sheet_next_cell_*`) | 515.5 ms | +| xlnt (`worksheet::rows()` + `cell::to_string()`/`value()`) | 2,495.0 ms | -ExcelReader is ~6.7x faster than DuckDB, ~8.1x faster than xlsxio and ~38x faster than xlnt on this +ExcelReader is ~7.2x faster than DuckDB, ~8.9x faster than xlsxio and ~43x faster than xlnt on this workload. xlsxio is a lean, purpose-built C streaming reader — the same abstraction level as ExcelReader's own native core — so it was the strongest of the two competitors tested, though still well behind. @@ -594,11 +594,11 @@ Measured on the same machine as above, GCC 16.2.0 (MSYS2 UCRT64, `-O3 -mavx2` fo | Workload | ExcelReader | zsv `compat` | zsv `fast` | |---|---:|---:|---:| -| Typed, 14 columns into `FullRow` | 27.4 ms | 22.6 ms | 20.0 ms | -| Cells, byte count | 11.6 ms | 5.8 ms | 3.5 ms | +| Typed, 14 columns into `FullRow` | 28.1 ms | 24.0 ms | 21.2 ms | +| Cells, byte count | 11.8 ms | 5.8 ms | 3.5 ms | -**zsv is faster on both.** Typed, it is ~1.2x (`compat`) and ~1.4x (`fast`) faster than ExcelReader. -Counting cells, it is ~2.0x and ~3.3x faster. +**zsv is faster on both.** Typed, it is ~1.2x (`compat`) and ~1.3x (`fast`) faster than ExcelReader. +Counting cells, it is ~2.0x and ~3.4x faster. The cells gap is the C ABI, not the CSV parser. `xl_rows_next_view` copies every cell's value into a native row the cursor owns, one call per row, and that alone costs ~10.6 ms here (calling it without @@ -635,11 +635,11 @@ is handed buffers that are already columnar, while `BM_WriteSheet` starts from a | Benchmark | Time | Rows/s | |---|---:|---:| -| `BM_WriteColumns` (pre-transposed) | 33.2 ms | 1.98 M/s | -| `BM_WriteSheet` (from `std::vector`) | 40.3 ms | 1.63 M/s | +| `BM_WriteColumns` (pre-transposed) | 34.0 ms | 1.94 M/s | +| `BM_WriteSheet` (from `std::vector`) | 39.9 ms | 1.64 M/s | -The transpose costs ~21% here. It is not free, but it is not the dominant cost of producing the -file — see the comparison below, where the same two cases over 14 columns land within ~14–16%. +The transpose costs ~17% here. It is not free, but it is not the dominant cost of producing the +file — see the comparison below, where the same two cases over 14 columns land within ~16–17%. `excelreader_cpp_write_compare_benchmarks` (under `-DEXCELREADER_BUILD_BENCHMARKS_COMPARE=ON`) puts that against xlnt, xlsxio and [DuckDB](https://github.com/duckdb/duckdb)'s `excel` extension, all @@ -656,12 +656,12 @@ competitor. | Library | Wall | CPU | |---|---:|---:| -| ExcelReader (`xl::write_columns`, pre-transposed) | 64.2–66.9 ms | 63.9–68.2 ms | -| ExcelReader (`xl::write_sheet`) | 74.8–76.0 ms | 74.7–76.4 ms | -| DuckDB (`COPY ... TO ... WITH (FORMAT xlsx)`) | 831.4–831.6 ms | 828.1 ms | -| libxlsxwriter (`worksheet_write_string`/`_number`) | 1,206.7 ms | 1,187.5 ms | -| xlsxio (`xlsxiowrite_add_cell_*`) | 2,405.1 ms | 1,125.0 ms | -| xlnt (`worksheet::cell().value()` + `save()`) | 5,525.3 ms | 5,515.6 ms | +| ExcelReader (`xl::write_columns`, pre-transposed) | 65.3–65.7 ms | 65.3–66.8 ms | +| ExcelReader (`xl::write_sheet`) | 76.2–76.5 ms | 76.4 ms | +| DuckDB (`COPY ... TO ... WITH (FORMAT xlsx)`) | 834.6–834.9 ms | 828.1–843.8 ms | +| libxlsxwriter (`worksheet_write_string`/`_number`) | 1,200.4 ms | 1,203.1 ms | +| xlsxio (`xlsxiowrite_add_cell_*`) | 2,441.4 ms | 859.4 ms | +| xlnt (`worksheet::cell().value()` + `save()`) | 5,554.0 ms | 5,546.9 ms | Same machine as above; Google Benchmark's own iteration counts, no `--benchmark_repetitions` (each iteration writes a whole 65,535-row file, so the slower cases run once or a handful of times). xlnt @@ -672,7 +672,7 @@ sampling within one run. `write_sheet` — the matched-work number, since it starts from the same `std::vector` every competitor is handed — is ~11x faster than DuckDB, ~16x faster than libxlsxwriter, ~32x -faster than xlsxio, and ~74x faster than xlnt on wall time. +faster than xlsxio, and ~73x faster than xlnt on wall time. ### Known issue: xlsxio + libxlsxwriter cannot share a binary @@ -709,10 +709,10 @@ Caveats, none of them optional when quoting these: - **xlsxio's CPU time is not stable across runs.** It read 843.8–906.3 ms before the fix (some runs missing the workbook.xml entry), 1,171.9 ms and 687.5 ms in the two runs after it, - 1,015.6 ms and 843.8 ms in the next two, 953.1 ms in the one after and 1,125.0 ms in the latest, while its wall time stayed at ~2.4 s throughout. Its wall time is dominated by I/O wait — + 1,015.6 ms and 843.8 ms in the next two, 953.1 ms in the one after, 1,125.0 ms in the next and 859.4 ms in the latest, while its wall time stayed at ~2.4 s throughout. Its wall time is dominated by I/O wait — CPU is well under half of wall, against every other case here being CPU-bound (wall ≈ CPU) — so quote the wall-time ratio, which is what a caller waits. Against this run's CPU time the gap from - `write_sheet` to xlsxio is ~15x, not ~32x. + `write_sheet` to xlsxio is ~11x, not ~32x. - **ExcelReader does slightly more work here**, not less: it attaches a number format to the two date columns so Excel shows a date, while every competitor case writes those as bare serial numbers. That difference favours the competitors. @@ -724,6 +724,6 @@ Caveats, none of them optional when quoting these: a full analytical query engine doing far more than any Excel-writing library here; this measures one narrow slice of it, not "DuckDB" as a whole. -The two ExcelReader cases land within ~14–16% of each other. The transpose's share grew as producing +The two ExcelReader cases land within ~16–17% of each other. The transpose's share grew as producing the file got cheaper, but it is still a minority of the cost. Reach for `write_columns` when your data is already columnar, but `write_sheet` is not the slow path. diff --git a/docs/guide/encryption.md b/docs/guide/encryption.md index fe4208b3..63ba6d5b 100644 --- a/docs/guide/encryption.md +++ b/docs/guide/encryption.md @@ -74,14 +74,15 @@ using ExcelReader.Core.Writer; using ExcelReader.Core.Writer.Xlsx; using var plain = new MemoryStream(); -await using (var workbook = XlsxWorkbookWriter.Create(plain, leaveOpen: true)) +var workbook = XlsxWorkbookWriter.Create(plain, leaveOpen: true); +await using (workbook.ConfigureAwait(false)) { // write sheets and rows as usual } plain.Position = 0; using var destination = File.Create("secret.xlsx"); -await Excel.EncryptPackageAsync(plain, destination, "hunter2"); +await Excel.EncryptPackageAsync(plain, destination, "hunter2").ConfigureAwait(false); ``` The package stream must be seekable (it is read twice, so the container never has to be buffered in diff --git a/docs/guide/untrusted-input.md b/docs/guide/untrusted-input.md index 5922d20d..2ddb7c1a 100644 --- a/docs/guide/untrusted-input.md +++ b/docs/guide/untrusted-input.md @@ -94,11 +94,13 @@ catch (Exception ex) when (IsRejectedFile(ex)) Console.WriteLine($"Rejected: {ex.Message}"); } -static bool IsRejectedFile(Exception ex) => - ex is InvalidDataException or EndOfStreamException or NotSupportedException +static bool IsRejectedFile(Exception ex) +{ + return ex is InvalidDataException or EndOfStreamException or NotSupportedException or ExcelEncryptionException or ExcelParseException || ex.GetType() == typeof(ArgumentException) || ex.GetType() == typeof(InvalidOperationException); +} ``` `LimitName` is the option's name (or `Columns`, `ArrayMaxLength`); `Limit` and `Actual` are in that @@ -140,8 +142,10 @@ file in a spreadsheet program, neutralize those fields yourself: ```csharp -static string Neutralize(string text) => - text.Length > 0 && "=+-@\t\r".Contains(text[0]) ? "'" + text : text; +static string Neutralize(string text) +{ + return text.Length > 0 && "=+-@\t\r".Contains(text[0]) ? "'" + text : text; +} row.Write(Neutralize(comment)); ``` diff --git a/docs/performance/benchmarks.md b/docs/performance/benchmarks.md index 42a0a6f9..1c289013 100644 --- a/docs/performance/benchmarks.md +++ b/docs/performance/benchmarks.md @@ -447,23 +447,23 @@ zero-copy advantage the other can't take: | Comparison | Format | ExcelReader | Competitor | Ratio | |---|---|---:|---:|---:| -| vs. [calamine](https://github.com/tafia/calamine) (Rust) | XLSX | 72.3 ms | 271.2 ms | ~3.8x faster | -| vs. calamine (Rust) | XLSB | 46.3 ms | 82.9 ms | ~1.8x faster | -| vs. [DuckDB](https://github.com/duckdb/duckdb) `read_xlsx` (C++) | XLSX | 61.3 ms | 413.8 ms | ~6.7x faster | -| vs. [xlsxio](https://github.com/brechtsanders/xlsxio) (C) | XLSX | 61.3 ms | 497.4 ms | ~8.1x faster | -| vs. [xlnt](https://github.com/tfussell/xlnt) (C++) | XLSX | 61.3 ms | 2,335.0 ms | ~38x faster | -| vs. [zsv](https://github.com/liquidaty/zsv) `fast` engine (C) | CSV | 27.4 ms | 20.0 ms | ~1.4x **slower** | +| vs. [calamine](https://github.com/tafia/calamine) (Rust) | XLSX | 69.5 ms | 276.4 ms | ~4.0x faster | +| vs. calamine (Rust) | XLSB | 47.8 ms | 87.0 ms | ~1.8x faster | +| vs. [DuckDB](https://github.com/duckdb/duckdb) `read_xlsx` (C++) | XLSX | 58.0 ms | 417.8 ms | ~7.2x faster | +| vs. [xlsxio](https://github.com/brechtsanders/xlsxio) (C) | XLSX | 58.0 ms | 515.5 ms | ~8.9x faster | +| vs. [xlnt](https://github.com/tfussell/xlnt) (C++) | XLSX | 58.0 ms | 2,495.0 ms | ~43x faster | +| vs. [zsv](https://github.com/liquidaty/zsv) `fast` engine (C) | CSV | 28.1 ms | 21.2 ms | ~1.3x **slower** | Writing the same 14 columns × 65,535 rows from row-shaped data, the C++ suite measures -`write_sheet` at 74.8–76.0 ms against DuckDB's 831 ms (~11x), libxlsxwriter's 1,207 ms (~16x), -xlsxio's 2,405 ms (~32x) and xlnt's 5,525 ms (~74x). The Rust suite measures 40.7 ms against -rust_xlsxwriter's 322.0 ms (~7.9x) over 7 columns. Caveats for both are in the binding READMEs. +`write_sheet` at 76.2–76.5 ms against DuckDB's 835 ms (~11x), libxlsxwriter's 1,200 ms (~16x), +xlsxio's 2,441 ms (~32x) and xlnt's 5,554 ms (~73x). The Rust suite measures 40.8 ms against +rust_xlsxwriter's 338.5 ms (~8.3x) over 7 columns. Caveats for both are in the binding READMEs. DuckDB, xlsxio and xlnt do not read `.xlsb`, so those comparisons are XLSX-only. calamine is a fast, well-optimized reader in its own right — the gap there is real but not the order of magnitude seen against the C/C++ competitors. zsv is the one competitor ahead. The CSV row is the typed workload, where both sides build the same -14-column rows with owned strings. Just counting cell bytes, zsv is ~3.3x faster, and that gap is +14-column rows with owned strings. Just counting cell bytes, zsv is ~3.4x faster, and that gap is ExcelReader's C ABI, which serializes every row into a buffer, not its CSV parser. That suite is built with GCC (zsv does not compile under MSVC). Its caveats are in the C++ README. diff --git a/docs/snippets/Encryption.cs b/docs/snippets/Encryption.cs index 8d7c2fb8..2000a921 100644 --- a/docs/snippets/Encryption.cs +++ b/docs/snippets/Encryption.cs @@ -64,14 +64,15 @@ internal static async Task Run() { #region encryption-write using var plain = new MemoryStream(); - await using (var workbook = XlsxWorkbookWriter.Create(plain, leaveOpen: true)) + var workbook = XlsxWorkbookWriter.Create(plain, leaveOpen: true); + await using (workbook.ConfigureAwait(false)) { // write sheets and rows as usual } plain.Position = 0; using var destination = File.Create("secret.xlsx"); - await Excel.EncryptPackageAsync(plain, destination, "hunter2"); + await Excel.EncryptPackageAsync(plain, destination, "hunter2").ConfigureAwait(false); #endregion } } diff --git a/docs/snippets/UntrustedInput.cs b/docs/snippets/UntrustedInput.cs index c894b537..41f42d00 100644 --- a/docs/snippets/UntrustedInput.cs +++ b/docs/snippets/UntrustedInput.cs @@ -42,11 +42,13 @@ internal static void Run(Stream upload, ExcelReaderOptions uploadOptions) Console.WriteLine($"Rejected: {ex.Message}"); } - static bool IsRejectedFile(Exception ex) => - ex is InvalidDataException or EndOfStreamException or NotSupportedException + static bool IsRejectedFile(Exception ex) + { + return ex is InvalidDataException or EndOfStreamException or NotSupportedException or ExcelEncryptionException or ExcelParseException || ex.GetType() == typeof(ArgumentException) || ex.GetType() == typeof(InvalidOperationException); + } #endregion } } @@ -59,8 +61,10 @@ internal static class Snippet internal static void Run(CsvRowWriter row, string comment) { #region untrusted-input-csv-injection - static string Neutralize(string text) => - text.Length > 0 && "=+-@\t\r".Contains(text[0]) ? "'" + text : text; + static string Neutralize(string text) + { + return text.Length > 0 && "=+-@\t\r".Contains(text[0]) ? "'" + text : text; + } row.Write(Neutralize(comment)); #endregion diff --git a/python/README.md b/python/README.md index e4e231fe..fdf2424b 100644 --- a/python/README.md +++ b/python/README.md @@ -129,7 +129,7 @@ A sheet reads in one of these shapes: `rows()` hands back cell *text*, which means the library formats every value to a string on the way out and builds one Python object per cell. `parse()` skips that entirely: you give it a schema, and the conversion happens natively, straight into typed column buffers. On a 65,536 × 14 sheet it is -~30× faster than `rows()`, and faster than `polars.read_excel()` — see [Reading](#reading) below for +~20× faster than `rows()`, and faster than `polars.read_excel()` — see [Reading](#reading) below for the measured numbers. ```python @@ -581,45 +581,46 @@ encrypt_package(plain, secret, "hunter2") `tests/ExcelReader.Benchmarks/Data/65K_Records_Data.xlsb` (65,535 data rows, 14 columns), the same fixture the .NET, C++ and Rust suites use. Measured on Windows 10 (22H2), AMD Ryzen 7 5700X (16 logical CPUs @ 3.39 GHz), CPython 3.14.7, 10 runs each (medians shown; `min` is in the scripts' own -output), against a library published from commit 449ca144. Competitors: polars 1.44.1 (with -fastexcel 0.21.0), pandas 3.0.5 (with openpyxl 3.1.5), xlsxwriter 3.2.9. +output), against a library published from commit ef831b19. Competitors: polars 2.0.0 (with +fastexcel 0.21.0), pandas 3.1.0 (with openpyxl 3.1.5), xlsxwriter 3.2.9. ### Reading | API | Median | What it produces | |---|---:|---| -| `to_arrow_stream()`, read by pyarrow | 31.2 ms | Arrow record batches, streamed | -| `to_arrow()` | 30.1 ms | the same parse, handed to pyarrow zero-copy | -| `parse()` | 32.5 ms | typed columnar buffers, converted natively | -| `parse_batches()` | 33.2 ms | the same typed buffers, in batches | -| `to_polars()` | 33.6 ms | typed columnar DataFrame, schema inferred | -| `polars.read_excel()` | 120.6 ms | typed columnar DataFrame, types inferred | -| `rows()` | 975.7 ms | one `Cell` object per cell, streamed per row | +| `to_arrow()` | 31.2 ms | the same parse, handed to pyarrow zero-copy | +| `to_arrow_stream()`, read by pyarrow | 32.0 ms | Arrow record batches, streamed | +| `parse()` | 33.8 ms | typed columnar buffers, converted natively | +| `parse_batches()` | 34.9 ms | the same typed buffers, in batches | +| `to_polars()` | 35.4 ms | typed columnar DataFrame, schema inferred | +| `polars.read_excel()` | 135.0 ms | typed columnar DataFrame, types inferred | +| `rows()` | 693.8 ms | one `Cell` object per cell, streamed per row | The batched readers cost the same as their whole-sheet counterparts and lower the peak memory: -`to_arrow_stream(batch_size=10000)` peaks at 11.4 MiB against 19.5 MiB for one whole-sheet +`to_arrow_stream(batch_size=10000)` peaks at 10.9 MiB against 19.4 MiB for one whole-sheet record batch on this file. -Only the `to_polars()` / `polars.read_excel()` pair is a like-for-like comparison (~3.6x), and even +Only the `to_polars()` / `polars.read_excel()` pair is a like-for-like comparison (~3.8x), and even that one is loose: both produce a typed columnar DataFrame with inferred types, but the inference rules are not identical. The rows above it produce different things and are listed to show what each API -costs, not to rank them — `rows()` is ~30x slower than `parse()` because it materializes +costs, not to rank them — `rows()` is ~20x slower than `parse()` because it materializes 917,504 Python objects, which is the price of that shape, not a slow parser. CSV, from `65K_Records_Data.csv` (8.2 MB), same 14 typed columns: | API | Median | |---|---:| -| `parse(parallelism=1)` | 23.3 ms | -| `parse(parallelism=0)` | 11.3 ms | -| `to_polars(parallelism=1)` | 24.2 ms | -| `to_polars(parallelism=0)` | 9.7 ms | -| `polars.read_csv(try_parse_dates=True)` | 6.9 ms | -| `pandas.read_csv`, pyarrow engine | 116.0 ms | -| `pandas.read_csv`, C engine | 132.3 ms | - -polars' CSV reader is ahead (multithreaded, and it infers types from the text without our schema -step); ExcelReader is ~10x faster than pandas on either engine. These include opening the file, so +| `parse(parallelism=1)` | 25.8 ms | +| `parse(parallelism=0)` | 13.7 ms | +| `to_polars(parallelism=1)` | 25.3 ms | +| `to_polars(parallelism=0)` | 11.3 ms | +| `polars.read_csv(try_parse_dates=True)` | 12.9 ms | +| `pandas.read_csv`, pyarrow engine | 114.5 ms | +| `pandas.read_csv`, C engine | 73.0 ms | + +At `parallelism=0` ExcelReader and polars' multithreaded CSV reader are level (`to_polars()` 11.3 ms +against 12.9 ms; polars infers types from the text, we convert against a given schema). ExcelReader is +~5x faster than pandas' C engine and ~8x faster than its pyarrow engine, measured with `parse(parallelism=0)`. These include opening the file, so they are above the in-memory `xl_parse_typed` figures in [Reading a large sheet on several threads](#reading-a-large-sheet-on-several-threads). @@ -627,28 +628,28 @@ threads](#reading-a-large-sheet-on-several-threads). | API | Median | Output | |---|---:|---| -| `write_workbook()` → csv | 33.3 ms | 8.2 MB | -| `write_workbook()` → xlsb | 51.7 ms | 4.5 MB | -| `write_workbook()` → xls | 59.2 ms | 13.4 MB | -| `write_workbook()` → xlsx | 68.5 ms | 5.1 MB | -| `write_workbook(polars_df)` → xlsx | 76.6 ms | 5.1 MB | -| `write_workbook(pandas_df)` → xlsx | 82.5 ms | 5.1 MB | -| `polars.DataFrame.write_excel()` | 4,714.9 ms | 5.6 MB | -| `pandas.DataFrame.to_excel()` | 7,371.4 ms | 5.5 MB | +| `write_workbook()` → csv | 35.9 ms | 8.2 MB | +| `write_workbook()` → xls | 53.0 ms | 13.4 MB | +| `write_workbook()` → xlsb | 53.7 ms | 4.5 MB | +| `write_workbook()` → xlsx | 68.8 ms | 5.1 MB | +| `write_workbook(polars_df)` → xlsx | 76.1 ms | 5.1 MB | +| `write_workbook(pandas_df)` → xlsx | 82.4 ms | 5.1 MB | +| `polars.DataFrame.write_excel()` | 4,754.8 ms | 5.6 MB | +| `pandas.DataFrame.to_excel()` | 7,696.4 ms | 5.5 MB | The two DataFrame comparisons are matched work — same DataFrame in, xlsx out both times: `write_workbook(polars_df)` is ~62x faster than polars' own `write_excel()`, and -`write_workbook(pandas_df)` ~89x faster than `to_excel()`. Both of ours pay a conversion the raw path does not: the DataFrame goes +`write_workbook(pandas_df)` ~93x faster than `to_excel()`. Both of ours pay a conversion the raw path does not: the DataFrame goes through Arrow, and each column is cast to the layout the native side reads and copied out of Arrow's -buffers, which is the gap between those rows and the 68.5 ms one. Handing `write_workbook()` buffers +buffers, which is the gap between those rows and the 68.8 ms one. Handing `write_workbook()` buffers that are already columnar — what `parse()` returns — skips it. -`write_workbook(xlsx)` at 68.5 ms lands within ~4 ms of the C++ binding's -`xl::write_columns` on the same 14 columns (64.2 ms), which is the expected result: both are thin wrappers +`write_workbook(xlsx)` at 68.8 ms lands within ~3 ms of the C++ binding's +`xl::write_columns` on the same 14 columns (65.7 ms), which is the expected result: both are thin wrappers over the same `xl_write_typed` call, and neither adds work per cell. `xls` being the largest file and still faster than xlsx is not a paradox — BIFF8 writes -fixed-width records with no compression, so it trades ~2.6x the bytes for less work per cell. xlsb is faster still, at 51.7 ms. +fixed-width records with no compression, so it trades ~2.6x the bytes for less work per cell. xlsb costs about the same, at 53.7 ms. ## Notes diff --git a/python/src/excelreader/_native.py b/python/src/excelreader/_native.py index f30c5f64..3e2614bd 100644 --- a/python/src/excelreader/_native.py +++ b/python/src/excelreader/_native.py @@ -614,7 +614,7 @@ def _bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.xl_start_row.argtypes = [ctypes.c_void_p] lib.xl_start_row.restype = ctypes.c_int32 - lib.xl_write_string.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_int32] + lib.xl_write_string.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int32] lib.xl_write_string.restype = ctypes.c_int32 lib.xl_write_int64.argtypes = [ctypes.c_void_p, ctypes.c_int64] diff --git a/python/src/excelreader/reader.py b/python/src/excelreader/reader.py index 8b50b178..b6960794 100644 --- a/python/src/excelreader/reader.py +++ b/python/src/excelreader/reader.py @@ -42,6 +42,7 @@ _FORMATS = _native.FORMATS _CELL_HEADER = struct.Struct(" Iterator[list[Cell]]: written = ctypes.c_int32() capacity = _INITIAL_ROW_BUFFER buffer = ctypes.create_string_buffer(capacity) + view = memoryview(buffer) while True: status = self._lib.xl_rows_next(cursor, buffer, capacity, ctypes.byref(written)) if status == _native.XL_EOF: @@ -253,9 +255,10 @@ def rows(self) -> Iterator[list[Cell]]: if status == _native.XL_BUFFER_TOO_SMALL: capacity = written.value buffer = ctypes.create_string_buffer(capacity) + view = memoryview(buffer) continue _check(status) - yield _decode_row(buffer[: written.value], written.value, date1904) + yield _decode_row(view, written.value, date1904) finally: self._close_rows(cursor) @@ -522,16 +525,16 @@ def _info(self) -> _native.NativeTableInfo: return info -def _decode_row(blob: bytes, length: int, date1904: bool) -> list[Cell]: +def _decode_row(blob: memoryview, length: int, date1904: bool) -> list[Cell]: count = struct.unpack_from(" None: def _write_cell(self, handle: ctypes.c_void_p, value: Any) -> int: lib = self._lib + kind = type(value) + if kind is str: + raw = value.encode("utf-8") + return lib.xl_write_string(handle, raw, _native.int32_len(raw)) + if kind is int: + return lib.xl_write_int64(handle, value) + if kind is float: + return lib.xl_write_float64(handle, value) + if kind is bool: + return lib.xl_write_bool(handle, 1 if value else 0) if value is None: return lib.xl_write_string(handle, None, 0) - if isinstance(value, tuple): + if kind is tuple: return lib.xl_write_null(handle, int(value[1])) - if isinstance(value, bool): - return lib.xl_write_bool(handle, 1 if value else 0) - if isinstance(value, int): - return lib.xl_write_int64(handle, value) - if isinstance(value, float): - return lib.xl_write_float64(handle, value) if isinstance(value, datetime.datetime): if value.tzinfo is not None: value = value.astimezone(datetime.timezone.utc).replace(tzinfo=None) @@ -143,8 +147,14 @@ def _write_cell(self, handle: ctypes.c_void_p, value: Any) -> int: return lib.xl_write_time( handle, (value.hour * 3600 + value.minute * 60 + value.second) * 1_000_000 + value.microsecond ) - pointer, length = _encode(value) - return lib.xl_write_string(handle, pointer, length) + if isinstance(value, bool): + return lib.xl_write_bool(handle, 1 if value else 0) + if isinstance(value, int): + return lib.xl_write_int64(handle, value) + if isinstance(value, float): + return lib.xl_write_float64(handle, value) + raw = value.encode("utf-8") + return lib.xl_write_string(handle, raw, _native.int32_len(raw)) def _finished_bytes(self, handle: ctypes.c_void_p) -> bytes: buffer = _native.NativeBuffer() diff --git a/rust/excelreader/README.md b/rust/excelreader/README.md index 2ecd4d47..90ad71c0 100644 --- a/rust/excelreader/README.md +++ b/rust/excelreader/README.md @@ -617,12 +617,12 @@ takes 20 samples instead — each of its iterations writes a whole 65,535-row fi | Benchmark | RealExcel.xlsb (100 rows) | 65K_Records_Data.xlsb (65,535 rows) | |---|---:|---:| -| `open` | 85.8 µs | 99.4 µs | -| `parse_sheet` (2 or 6 bound columns) | 54.7 µs | 24.0 ms | -| typed chunks of 1,000 / 10,000 rows | - | 22.8 ms / 22.9 ms | -| `infer_schema` (sample 100 / 1,000 rows) | 83.9 µs | 571.0 µs | +| `open` | 90.5 µs | 102.1 µs | +| `parse_sheet` (2 or 6 bound columns) | 57.9 µs | 24.9 ms | +| typed chunks of 1,000 / 10,000 rows | - | 23.4 ms / 23.4 ms | +| `infer_schema` (sample 100 / 1,000 rows) | 89.7 µs | 611.1 µs | -`open` stays nearly flat across the 655x row-count jump (+16%) - XLSB's header carries its +`open` stays nearly flat across the 655x row-count jump (+13%) - XLSB's header carries its dimensions/index, so opening costs metadata, not row data. `parse_sheet` scales linearly with rows × columns. Reading the same sheet in typed chunks costs no more than one whole-sheet `parse_sheet`, while holding only one chunk at a time. `infer_schema` scales with its sample size, @@ -635,10 +635,10 @@ zero-copy advantage the other can't take: | Format | ExcelReader (`Sheet::parse::`) | calamine (`worksheet_range` + `Data` match) | |---|---:|---:| -| XLSX | 72.3 ms | 271.2 ms | -| XLSB | 46.3 ms | 82.9 ms | +| XLSX | 69.5 ms | 276.4 ms | +| XLSB | 47.8 ms | 87.0 ms | -ExcelReader is ~3.8x faster than calamine for XLSX and ~1.8x faster for XLSB on this workload - +ExcelReader is ~4.0x faster than calamine for XLSX and ~1.8x faster for XLSB on this workload - calamine is a fast, well-optimized reader in its own right, so the gap is real but not the order of magnitude seen against slower libraries. @@ -648,15 +648,15 @@ rows to `.xlsx`, all three starting from the same in-memory `Vec`: | Benchmark | Median | |---|---:| -| `columns` (`write_columns`, pre-transposed) | 33.8 ms | -| `sheet` (`write_sheet`, from `Vec`) | 40.7 ms | -| `rust_xlsxwriter` (cell-at-a-time) | 322.0 ms | +| `columns` (`write_columns`, pre-transposed) | 35.0 ms | +| `sheet` (`write_sheet`, from `Vec`) | 40.8 ms | +| `rust_xlsxwriter` (cell-at-a-time) | 338.5 ms | `sheet` is the matched-work number — it starts from the same shape `rust_xlsxwriter` is handed and -pays the row-to-column transpose itself — and is ~7.9x faster. `columns` is a ceiling no +pays the row-to-column transpose itself — and is ~8.3x faster. `columns` is a ceiling no cell-at-a-time API can reach, since it is handed buffers that are already columnar; read it only against `sheet`, as the cost of having row-shaped data in the first place. That cost turns out to be -about 21%: not free, but a minority of the cost of producing the file. +about 17%: not free, but a minority of the cost of producing the file. Two caveats, both running against the headline number rather than for it. ExcelReader does slightly *more* work here: it attaches a number format to the date column so Excel shows a date, and writes a diff --git a/src/ExcelReader.Core/Parser/Internal/ColumnParserFactory.cs b/src/ExcelReader.Core/Parser/Internal/ColumnParserFactory.cs index e9867871..b1186ca5 100644 --- a/src/ExcelReader.Core/Parser/Internal/ColumnParserFactory.cs +++ b/src/ExcelReader.Core/Parser/Internal/ColumnParserFactory.cs @@ -480,6 +480,7 @@ private static class EnumCache private static readonly FrozenDictionary _nameMap = BuildNameMap(); private static readonly FrozenDictionary.AlternateLookup> _alternateLookup = _nameMap.GetAlternateLookup>(); private static readonly FrozenDictionary _valueMap = BuildValueMap(); + private static readonly FrozenDictionary _longMap = _valueMap.Where(p => p.Key >= long.MinValue && p.Key <= long.MaxValue).ToFrozenDictionary(p => (long)p.Key, p => p.Value); private static FrozenDictionary BuildNameMap() { @@ -513,6 +514,29 @@ private static Int128 Numeric(TEnum value) return Convert.ToInt64(value, CultureInfo.InvariantCulture); } + private static bool TryParseLong(ReadOnlySpan text, out long result) + { + result = 0; + bool negative = !text.IsEmpty && text[0] == (byte)'-'; + ReadOnlySpan digits = negative ? text[1..] : text; + if (digits.IsEmpty || digits.Length > 18) + { + return false; + } + long acc = 0; + foreach (byte b in digits) + { + uint digit = (uint)(b - (byte)'0'); + if (digit > 9) + { + return false; + } + acc = acc * 10 + digit; + } + result = negative ? -acc : acc; + return true; + } + // Integer text is parsed exactly; a double cannot tell apart integers above 2^53. private static bool TryGetIntegral(in Cell cell, double number, out Int128 integral) { @@ -534,6 +558,14 @@ public static bool TryParse(in Cell cell, out TEnum value) { if (cell.Type is CellType.Number or CellType.Date && cell.TryGetDouble(out double d)) { + if (TryParseLong(cell.Value, out long small)) + { + return _longMap.TryGetValue(small, out value); + } + if (cell.Value.IsEmpty && d == Math.Truncate(d) && d is >= -9223372036854775808.0 and < 9223372036854775808.0) + { + return _longMap.TryGetValue((long)d, out value); + } if (!TryGetIntegral(in cell, d, out Int128 integral)) { value = default; diff --git a/src/ExcelReader.Core/Parser/SheetPipeline/ChunkProducer.cs b/src/ExcelReader.Core/Parser/SheetPipeline/ChunkProducer.cs index b045a087..4fd34c7a 100644 --- a/src/ExcelReader.Core/Parser/SheetPipeline/ChunkProducer.cs +++ b/src/ExcelReader.Core/Parser/SheetPipeline/ChunkProducer.cs @@ -1,6 +1,7 @@ using System.Buffers; using ExcelReader.Core.Reader; using ExcelReader.Core.Reader.Internal; +using ExcelReader.Core.Reader.Zip; namespace ExcelReader.Core.Parser.SheetPipeline { @@ -33,6 +34,10 @@ internal static async Task RunAsync( int cut = cutter.Cut(buffer.AsSpan(0, filled), out bool ended); if (ended || eof) { + if (source is LimitedReadStream entry) + { + await entry.ReadRemainderForChecksumAsync(ct).ConfigureAwait(false); + } int length = ended ? cut : filled; if (length > 0) { diff --git a/src/ExcelReader.Core/Reader/PooledStreamRowEnumerator.cs b/src/ExcelReader.Core/Reader/PooledStreamRowEnumerator.cs index 278e654e..a5e35186 100644 --- a/src/ExcelReader.Core/Reader/PooledStreamRowEnumerator.cs +++ b/src/ExcelReader.Core/Reader/PooledStreamRowEnumerator.cs @@ -1,6 +1,7 @@ using System.ComponentModel; using System.Diagnostics; using ExcelReader.Core.Reader.Internal; +using ExcelReader.Core.Reader.Zip; namespace ExcelReader.Core.Reader { @@ -117,6 +118,22 @@ private async ValueTask OpenThenFillAsync() await _io.FillAsync(_source, _ct).ConfigureAwait(false); } + // The last row ends before the entry does; reading the rest compares the entry's checksum. + private protected bool Finish() + { + (_source as LimitedReadStream)?.ReadRemainderForChecksum(); + return false; + } + + private protected async ValueTask FinishAsync() + { + if (_source is LimitedReadStream entry) + { + await entry.ReadRemainderForChecksumAsync(_ct).ConfigureAwait(false); + } + return false; + } + private protected void Ensure(int count) { if (_io.Len - _io.Pos < count && !_io.Eof) diff --git a/src/ExcelReader.Core/Reader/SheetDataReader.cs b/src/ExcelReader.Core/Reader/SheetDataReader.cs index daea60d4..e0ae6672 100644 --- a/src/ExcelReader.Core/Reader/SheetDataReader.cs +++ b/src/ExcelReader.Core/Reader/SheetDataReader.cs @@ -220,6 +220,14 @@ private static Type TypeOf(Cell cell) }; } + private static readonly object BoxedTrue = true; + private static readonly object BoxedFalse = false; + + private static object Boxed(bool value) + { + return value ? BoxedTrue : BoxedFalse; + } + /// public object GetValue(int i) { @@ -229,7 +237,7 @@ public object GetValue(int i) CellType.Empty => DBNull.Value, CellType.Number => cell.TryGetDouble(out double d) ? d : cell.GetString(), CellType.Date => cell.TryGetDateTime(_isDate1904, out DateTime dt) ? dt : cell.GetString(), - CellType.Boolean => ExcelCellReaders.Bool(in cell, default, out bool b) ? b : cell.GetString(), + CellType.Boolean => ExcelCellReaders.Bool(in cell, default, out bool b) ? Boxed(b) : cell.GetString(), _ => cell.GetString(), }; } diff --git a/src/ExcelReader.Core/Reader/Xlsb/XlsbWorkbook.Enumerator.cs b/src/ExcelReader.Core/Reader/Xlsb/XlsbWorkbook.Enumerator.cs index 4884a29b..3e677a2e 100644 --- a/src/ExcelReader.Core/Reader/Xlsb/XlsbWorkbook.Enumerator.cs +++ b/src/ExcelReader.Core/Reader/Xlsb/XlsbWorkbook.Enumerator.cs @@ -100,7 +100,7 @@ void IRowIndexedEnumerator.EnableRowIndex() public bool MoveNext() { _ct.ThrowIfCancellationRequested(); - return MoveNextCore(); + return MoveNextCore() || Finish(); } /// @@ -109,7 +109,7 @@ public ValueTask MoveNextAsync() _ct.ThrowIfCancellationRequested(); if (_ended) { - return new ValueTask(false); + return FinishAsync(); } ResetRow(); if (!_pendingRowHdr) @@ -117,7 +117,7 @@ public ValueTask MoveNextAsync() int seek = SeekRowHdrFromBuffer(); if (seek == 0) { - return new ValueTask(false); + return FinishAsync(); } if (seek == 2) { @@ -137,7 +137,7 @@ private async ValueTask MoveNextRowAsync(bool seekDone) { if (!seekDone && !await SeekRowHdrAsync().ConfigureAwait(false)) { - return false; + return await FinishAsync().ConfigureAwait(false); } _pendingRowHdr = false; _rowIndex = _nextRowIndex; diff --git a/src/ExcelReader.Core/Reader/Xlsx/XlsxWorkbook.Enumerator.cs b/src/ExcelReader.Core/Reader/Xlsx/XlsxWorkbook.Enumerator.cs index cbb17044..125a0866 100644 --- a/src/ExcelReader.Core/Reader/Xlsx/XlsxWorkbook.Enumerator.cs +++ b/src/ExcelReader.Core/Reader/Xlsx/XlsxWorkbook.Enumerator.cs @@ -137,7 +137,7 @@ public bool MoveNext() { case HeadKind.End: _reachedEnd = true; - return false; + return Finish(); case HeadKind.Row: if (!BeginRow()) { @@ -193,7 +193,7 @@ public ValueTask MoveNextAsync() switch (ClassifyHead()) { case HeadKind.End: - return new ValueTask(false); + return FinishAsync(); case HeadKind.Row: return ReadRowAsync(); default: @@ -477,7 +477,7 @@ private int ParseCellSpan(byte[] buf, int len, int p) } else if (kind == Kind.Inline) { - int end = TryEmitPlainInline(buf, len, p, col, style); + int end = TryEmitPlainInline(buf.AsSpan(0, len), p, col, style); if (end >= 0) { return end; @@ -537,9 +537,14 @@ private int TryParseCanonicalCell(ReadOnlySpan buf, int p, ref int nextCol nextCol = col + 1; return i + 2; } - if (kind == Kind.Inline || !buf.Slice(i, 4).SequenceEqual(">"u8)) + if (!buf.Slice(i, 4).SequenceEqual(">"u8)) { - return -1; + int end = TryParseCanonicalCellBody(buf, i, col, style, kind); + if (end >= 0) + { + nextCol = col + 1; + } + return end; } int start = i + 4; int v = ScanCanonicalValue(buf, start, kind, out int sharedIndex); @@ -552,6 +557,58 @@ private int TryParseCanonicalCell(ReadOnlySpan buf, int p, ref int nextCol return v + 8; } + [MethodImpl(MethodImplOptions.NoInlining)] + private int TryParseCanonicalCellBody(ReadOnlySpan buf, int i, int col, int style, Kind kind) + { + if (buf[i] != (byte)'>') + { + return -1; + } + if (kind == Kind.Inline) + { + return TryEmitPlainInline(buf, i + 1, col, style); + } + int start = SkipFormula(buf, i + 1); + if (start < 0 || !buf[start..].StartsWith(""u8)) + { + return -1; + } + start += 3; + int v = ScanCanonicalValue(buf, start, kind, out int sharedIndex); + if (v < 0 || !buf[v..].StartsWith(""u8)) + { + return -1; + } + EmitCanonicalCell(kind, buf, start, v, col, style, sharedIndex); + return v + 8; + } + + // Formula text and attributes hold no raw '<', so the next '<' after the start tag is the end tag. + private static int SkipFormula(ReadOnlySpan buf, int p) + { + if (buf.Length - p < 3 || buf[p] != (byte)'<' || buf[p + 1] != (byte)'f' || !IsBoundary(buf[p + 2])) + { + return -1; + } + int gt = buf[(p + 2)..].IndexOfAny((byte)'<', (byte)'>'); + if (gt < 0 || buf[p + 2 + gt] != (byte)'>') + { + return -1; + } + gt += p + 2; + if (buf[gt - 1] == (byte)'/') + { + return gt + 1; + } + int lt = buf[(gt + 1)..].IndexOf((byte)'<'); + if (lt < 0) + { + return -1; + } + lt += gt + 1; + return buf[lt..].StartsWith(""u8) ? lt + 4 : -1; + } + // A shared string's value must be its index in at most 9 digits; any other value runs to the next markup. [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int ScanCanonicalValue(ReadOnlySpan buf, int start, Kind kind, out int sharedIndex) @@ -691,20 +748,20 @@ private int TryEmitSharedIndex(byte[] buf, int len, int valueStart, int col, int return i + 8; } - private int TryEmitPlainInline(byte[] buf, int len, int p, int col, int style) + private int TryEmitPlainInline(ReadOnlySpan buf, int p, int col, int style) { - if (!buf.AsSpan(p, Math.Min(7, len - p)).StartsWith(""u8)) + if (!buf[p..].StartsWith(""u8)) { return -1; } int textStart = p + 7; - int rel = XlsxXml.IndexOfMarkup(buf.AsSpan(textStart, len - textStart), out bool mayEscape); + int rel = XlsxXml.IndexOfMarkup(buf[textStart..], out bool mayEscape); if (rel < 0) { return -1; } int lt = textStart + rel; - if (!buf.AsSpan(lt, Math.Min(13, len - lt)).StartsWith(""u8) || (mayEscape && buf.AsSpan(textStart, rel).IndexOf("_x"u8) >= 0)) + if (!buf[lt..].StartsWith(""u8) || (mayEscape && buf.Slice(textStart, rel).IndexOf("_x"u8) >= 0)) { return -1; } diff --git a/src/ExcelReader.Core/Reader/Zip/LimitedReadStream.cs b/src/ExcelReader.Core/Reader/Zip/LimitedReadStream.cs index fa843a6b..00a85c3d 100644 --- a/src/ExcelReader.Core/Reader/Zip/LimitedReadStream.cs +++ b/src/ExcelReader.Core/Reader/Zip/LimitedReadStream.cs @@ -1,3 +1,4 @@ +using System.Buffers; using ExcelReader.Core.Reader.Internal; using ExcelReader.Core.Writer.Zip; @@ -95,6 +96,47 @@ public override Task ReadAsync(byte[] buffer, int offset, int count, Cancel return ReadAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask(); } + private bool VerifiesChecksum => _expectedCrc is not null || _inner is PrefetchStream { VerifiesChecksum: true }; + + // A reader stops at the end of the sheet's rows; reading what follows still compares the entry's checksum. + internal void ReadRemainderForChecksum() + { + if (!VerifiesChecksum || _produced >= _declaredLength) + { + return; + } + byte[] scratch = ArrayPool.Shared.Rent(16 * 1024); + try + { + while (Read(scratch.AsSpan()) > 0) + { + } + } + finally + { + ArrayPool.Shared.Return(scratch); + } + } + + internal async ValueTask ReadRemainderForChecksumAsync(CancellationToken ct) + { + if (!VerifiesChecksum || _produced >= _declaredLength) + { + return; + } + byte[] scratch = ArrayPool.Shared.Rent(16 * 1024); + try + { + while (await ReadAsync(scratch.AsMemory(), ct).ConfigureAwait(false) > 0) + { + } + } + finally + { + ArrayPool.Shared.Return(scratch); + } + } + protected override void Dispose(bool disposing) { if (disposing) diff --git a/src/ExcelReader.Core/Reader/Zip/PrefetchStream.cs b/src/ExcelReader.Core/Reader/Zip/PrefetchStream.cs index 5017b3bc..f2ae2cc5 100644 --- a/src/ExcelReader.Core/Reader/Zip/PrefetchStream.cs +++ b/src/ExcelReader.Core/Reader/Zip/PrefetchStream.cs @@ -41,6 +41,8 @@ internal PrefetchStream(Stream inner, long declaredLength = 0, uint? expectedCrc _producer = Task.Run(() => ProduceAsync(_cts.Token), _cts.Token); } + internal bool VerifiesChecksum => _expectedCrc is not null; + public override int Read(byte[] buffer, int offset, int count) { return Read(buffer.AsSpan(offset, count)); diff --git a/src/ExcelReader.Core/Writer/Internal/RecordColumns.cs b/src/ExcelReader.Core/Writer/Internal/RecordColumns.cs index 330217eb..4fe63f07 100644 --- a/src/ExcelReader.Core/Writer/Internal/RecordColumns.cs +++ b/src/ExcelReader.Core/Writer/Internal/RecordColumns.cs @@ -2,6 +2,7 @@ using System.Globalization; using System.Linq.Expressions; using System.Reflection; +using System.Runtime.CompilerServices; using ExcelReader.Core.Parser; namespace ExcelReader.Core.Writer.Internal @@ -63,6 +64,22 @@ private static Expression ToStringExpression(Expression value, Type pt) Expression.Call(boxed, toString), Expression.Constant(null, typeof(string))); } + // Runtime value types (Guid, DateTimeOffset, TimeSpan...) format to the same text as UTF-8, without the string. + private static MethodCallExpression? TryBuildUtf8Write(ParameterExpression rowParam, Expression value, Type pt) + { + Type? underlying = Nullable.GetUnderlyingType(pt); + Type core = underlying ?? pt; + // char is left out: its UTF-8 TryFormat throws on a lone surrogate. + if (!core.IsValueType || core.Assembly != typeof(object).Assembly || core == typeof(char) + || !typeof(IUtf8SpanFormattable).IsAssignableFrom(core) || !typeof(IFormattable).IsAssignableFrom(core)) + { + return null; + } + string helper = underlying is null ? nameof(InvariantText.WriteValue) : nameof(InvariantText.WriteNullable); + MethodInfo method = typeof(InvariantText).GetMethod(helper, BindingFlags.NonPublic | BindingFlags.Static)!.MakeGenericMethod(typeof(TRow), core); + return Expression.Call(method, rowParam, value); + } + private static MethodCallExpression? TryBuildConverterWrite( PropertyInfo prop, ParameterExpression rowParam, Expression value, Dictionary instances) { @@ -102,6 +119,11 @@ private static Action Build() continue; } MethodInfo write = RowWriteMethods.Select(prop.PropertyType, out bool asString); + if (asString && TryBuildUtf8Write(rowParam, value, prop.PropertyType) is { } utf8Write) + { + body.Add(utf8Write); + continue; + } if (asString) { value = ToStringExpression(value, prop.PropertyType); @@ -197,6 +219,32 @@ internal static string FormatValue(TValue value) return value.HasValue ? FormatValue(value.GetValueOrDefault()) : null; } + [SkipLocalsInit] + internal static void WriteValue(TRow row, TValue value) + where TRow : IRowWriter + where TValue : struct, IFormattable, IUtf8SpanFormattable + { + Span utf8 = stackalloc byte[128]; + if (value.TryFormat(utf8, out int written, typeof(TValue) == typeof(DateTimeOffset) ? "O" : default, CultureInfo.InvariantCulture)) + { + row.WriteUtf8(utf8[..written]); + return; + } + row.Write(FormatValue(value)); + } + + internal static void WriteNullable(TRow row, TValue? value) + where TRow : IRowWriter + where TValue : struct, IFormattable, IUtf8SpanFormattable + { + if (!value.HasValue) + { + row.Write((string?)null); + return; + } + WriteValue(row, value.GetValueOrDefault()); + } + internal static string? FormatReference(TValue? value) where TValue : class, IFormattable { diff --git a/src/ExcelReader.Core/Writer/Internal/SharedStringTable.cs b/src/ExcelReader.Core/Writer/Internal/SharedStringTable.cs index c790aee7..81071581 100644 --- a/src/ExcelReader.Core/Writer/Internal/SharedStringTable.cs +++ b/src/ExcelReader.Core/Writer/Internal/SharedStringTable.cs @@ -1,4 +1,5 @@ using System.Globalization; +using System.Runtime.InteropServices; using ExcelReader.Core.Reader.Xlsb; using ExcelReader.Core.Writer.Xlsb; @@ -19,6 +20,7 @@ internal sealed class SharedStringTable private readonly Dictionary.AlternateLookup> _serialSpan; private int _count; private int _uniqueCount; + private bool _sharded; internal SharedStringTable() { @@ -36,35 +38,33 @@ internal SharedStringTable() internal int GetOrAdd(string value) { _count++; - if (_serial.TryGetValue(value, out int index)) + ref int index = ref CollectionsMarshal.GetValueRefOrAddDefault(_serial, value, out bool exists); + if (!exists) { - return index; + index = SerialMiss(value); } - index = SerialMiss(value); - _serial.Add(value, index); return index; } internal int GetOrAdd(ReadOnlySpan value) { _count++; - if (_serialSpan.TryGetValue(value, out int index)) + ref int index = ref CollectionsMarshal.GetValueRefOrAddDefault(_serialSpan, value, out bool exists); + if (!exists) { - return index; + index = SerialMiss(value); } - index = SerialMiss(value); - _serialSpan.TryAdd(value, index); return index; } // Workers may have added it to a shard; a text new to both takes the next index. private int SerialMiss(ReadOnlySpan value) { - return ShardFor(value).SpanIndexes.TryGetValue(value, out int index) ? index : _uniqueCount++; + return _sharded && ShardFor(value).SpanIndexes.TryGetValue(value, out int index) ? index : _uniqueCount++; } /// Resolves a text from a parallel worker without counting the reference; see . - internal int GetOrAddConcurrent(ReadOnlySpan value) + internal int GetOrAddConcurrent(ReadOnlySpan value, string? owned) { if (_serialSpan.TryGetValue(value, out int serial)) { @@ -73,12 +73,12 @@ internal int GetOrAddConcurrent(ReadOnlySpan value) Shard shard = ShardFor(value); lock (shard.Gate) { - if (shard.SpanIndexes.TryGetValue(value, out int index)) + ref int index = ref shard.Slot(value, owned, out bool exists); + if (!exists) { - return index; + index = Interlocked.Increment(ref _uniqueCount) - 1; + _sharded = true; } - index = Interlocked.Increment(ref _uniqueCount) - 1; - shard.SpanIndexes.TryAdd(value, index); return index; } } @@ -220,6 +220,15 @@ internal Shard() internal Dictionary.AlternateLookup> SpanIndexes { get; } internal Lock Gate { get; } = new(); + + internal ref int Slot(ReadOnlySpan value, string? owned, out bool exists) + { + if (owned is null) + { + return ref CollectionsMarshal.GetValueRefOrAddDefault(SpanIndexes, value, out exists); + } + return ref CollectionsMarshal.GetValueRefOrAddDefault(Indexes, owned, out exists); + } } } } diff --git a/src/ExcelReader.Core/Writer/Internal/StyleTable.cs b/src/ExcelReader.Core/Writer/Internal/StyleTable.cs index 940a73c1..5aaf4cbe 100644 --- a/src/ExcelReader.Core/Writer/Internal/StyleTable.cs +++ b/src/ExcelReader.Core/Writer/Internal/StyleTable.cs @@ -15,7 +15,8 @@ internal sealed class StyleTable private readonly Dictionary _indices = []; private readonly List _styles = []; private readonly HashSet _customFormats = new(StringComparer.Ordinal); - private readonly Dictionary _dateVariants = []; + private readonly Lock _dateVariantGate = new(); + private int[] _dateVariants = []; internal StyleTable() { @@ -51,18 +52,33 @@ internal int Add(CellStyle style) return index; } - // Locked because parallel record writers call this from worker threads. + // Parallel record writers resolve this per date cell on every worker, so a known variant is read + // without the lock; entries hold variant + 1, leaving 0 for "not resolved yet". internal int DateVariant(int styleId) { - lock (_dateVariants) + int[] known = Volatile.Read(ref _dateVariants); + if ((uint)styleId < (uint)known.Length && known[styleId] != 0) { - if (_dateVariants.TryGetValue(styleId, out int variant)) + return known[styleId] - 1; + } + return AddDateVariant(styleId); + } + + private int AddDateVariant(int styleId) + { + lock (_dateVariantGate) + { + int[] known = _dateVariants; + if ((uint)styleId < (uint)known.Length && known[styleId] != 0) { - return variant; + return known[styleId] - 1; } CellStyle style = _styles[styleId]; - variant = style.NumberFormat is null ? Add(style with { NumberFormat = DateStyle.NumberFormat }) : styleId; - _dateVariants[styleId] = variant; + int variant = style.NumberFormat is null ? Add(style with { NumberFormat = DateStyle.NumberFormat }) : styleId; + int[] next = new int[Math.Max(known.Length, styleId + 1)]; + known.CopyTo(next, 0); + next[styleId] = variant + 1; + Volatile.Write(ref _dateVariants, next); return variant; } } diff --git a/src/ExcelReader.Core/Writer/Parallel/WorkerStringCache.cs b/src/ExcelReader.Core/Writer/Parallel/WorkerStringCache.cs index f4e91bbf..49bf97df 100644 --- a/src/ExcelReader.Core/Writer/Parallel/WorkerStringCache.cs +++ b/src/ExcelReader.Core/Writer/Parallel/WorkerStringCache.cs @@ -23,21 +23,33 @@ internal WorkerStringCache(SharedStringTable table) internal int GetOrAdd(string value) { - return GetOrAdd(value.AsSpan()); + return GetOrAdd(value, value); } internal int GetOrAdd(ReadOnlySpan value) + { + return GetOrAdd(value, null); + } + + // An owned string is stored as is; a span is copied into a new string only when it is kept. + private int GetOrAdd(ReadOnlySpan value, string? owned) { _references++; if (_spanIndexes.TryGetValue(value, out int index)) { return index; } - index = _table.GetOrAddConcurrent(value); - if (_indexes.Count < MaxEntries) + index = _table.GetOrAddConcurrent(value, owned); + if (_indexes.Count >= MaxEntries) { - _spanIndexes.TryAdd(value, index); + return index; + } + if (owned is not null) + { + _indexes.TryAdd(owned, index); + return index; } + _spanIndexes.TryAdd(value, index); return index; } diff --git a/src/ExcelReader.Core/Writer/Xls/XlsWorkbookWriter.cs b/src/ExcelReader.Core/Writer/Xls/XlsWorkbookWriter.cs index 73c39d45..a8fdd19e 100644 --- a/src/ExcelReader.Core/Writer/Xls/XlsWorkbookWriter.cs +++ b/src/ExcelReader.Core/Writer/Xls/XlsWorkbookWriter.cs @@ -85,7 +85,10 @@ public int AddStyle(CellStyle style) internal int StyleCount => _styles.Count; - internal int DateStyle(int styleId) => _styles.DateVariant(styleId); + internal int DateStyle(int styleId) + { + return _styles.DateVariant(styleId); + } internal int AddSharedString(ReadOnlySpan value) { diff --git a/src/ExcelReader.Core/Writer/Xlsb/XlsbWorkbookWriter.cs b/src/ExcelReader.Core/Writer/Xlsb/XlsbWorkbookWriter.cs index 460b78c7..f32a9721 100644 --- a/src/ExcelReader.Core/Writer/Xlsb/XlsbWorkbookWriter.cs +++ b/src/ExcelReader.Core/Writer/Xlsb/XlsbWorkbookWriter.cs @@ -99,7 +99,10 @@ public int AddStyle(CellStyle style) internal int StyleCount => _styles.Count; - internal int DateStyle(int styleId) => _styles.DateVariant(styleId); + internal int DateStyle(int styleId) + { + return _styles.DateVariant(styleId); + } /// /// Synchronous counterpart to , for native/unmanaged callers whose ABI is diff --git a/src/ExcelReader.Core/Writer/Xlsx/XlsxSheetWriter.cs b/src/ExcelReader.Core/Writer/Xlsx/XlsxSheetWriter.cs index cfb19fbd..ccaf67e3 100644 --- a/src/ExcelReader.Core/Writer/Xlsx/XlsxSheetWriter.cs +++ b/src/ExcelReader.Core/Writer/Xlsx/XlsxSheetWriter.cs @@ -80,7 +80,11 @@ private static int DetachedCapacity(int batchRows) internal bool UseSharedStrings => _owner.UseSharedStrings; internal bool Date1904 => _owner.Date1904; - internal int DateStyle(int styleId) => _owner.DateStyle(styleId); + internal int DateStyle(int styleId) + { + return _owner.DateStyle(styleId); + } + internal bool ResourcesReleased => _stream is null && _rowBuffer.IsReleased; internal int GetSharedStringIndex(string value) diff --git a/src/ExcelReader.Core/Writer/Xlsx/XlsxWorkbookWriter.cs b/src/ExcelReader.Core/Writer/Xlsx/XlsxWorkbookWriter.cs index 602de252..ccb7387a 100644 --- a/src/ExcelReader.Core/Writer/Xlsx/XlsxWorkbookWriter.cs +++ b/src/ExcelReader.Core/Writer/Xlsx/XlsxWorkbookWriter.cs @@ -121,7 +121,10 @@ public int AddStyle(CellStyle style) internal int StyleCount => _styles.Count; - internal int DateStyle(int styleId) => _styles.DateVariant(styleId); + internal int DateStyle(int styleId) + { + return _styles.DateVariant(styleId); + } /// /// Synchronous counterpart to , for native/unmanaged callers whose ABI is diff --git a/src/ExcelReader.Generator/ExcelRowMapGenerator.cs b/src/ExcelReader.Generator/ExcelRowMapGenerator.cs index e4881983..3ad7386e 100644 --- a/src/ExcelReader.Generator/ExcelRowMapGenerator.cs +++ b/src/ExcelReader.Generator/ExcelRowMapGenerator.cs @@ -365,7 +365,8 @@ private static PropertyPlan BuildPlan(List diagnostics, INamedTy { bool needsNullConditional = isNullable || !underlying.IsValueType; string valueExpr = WriteValueExpression(writeKind, needsNullConditional, member, underlying); - writeEmit = $" .Column({Literal(names[0])}, static (row, m) => row.Write({valueExpr}))"; + string writeMethod = writeKind == WriteKind.Utf8Text ? "WriteUtf8" : "Write"; + writeEmit = $" .Column({Literal(names[0])}, static (row, m) => row.{writeMethod}({valueExpr}))"; } } @@ -409,11 +410,7 @@ private static ReadKind SelectReadKind(bool isGuid, bool isNullable) private static string WriteValueExpression(WriteKind kind, bool needsNullConditional, string propertyName, ITypeSymbol underlying) { - if (kind == WriteKind.Utf8Text) - { - return $"global::System.Text.Encoding.UTF8.GetString(m.{propertyName})"; - } - if (kind == WriteKind.Direct) + if (kind is WriteKind.Utf8Text or WriteKind.Direct) { return $"m.{propertyName}"; } diff --git a/src/ExcelReader.Native/Exports.Writer.cs b/src/ExcelReader.Native/Exports.Writer.cs index ea8186c4..aec0ac74 100644 --- a/src/ExcelReader.Native/Exports.Writer.cs +++ b/src/ExcelReader.Native/Exports.Writer.cs @@ -212,43 +212,43 @@ public static int WriteString(nint handle, byte* value, int valueLength) [UnmanagedCallersOnly(EntryPoint = "xl_write_int64")] public static int WriteInt64(nint handle, long value) { - return RunWriterOp(handle, writerHandle => writerHandle.WriteInt64(value)); + return RunWriterOp(handle, value, static (writerHandle, state) => writerHandle.WriteInt64(state)); } [UnmanagedCallersOnly(EntryPoint = "xl_write_float64")] public static int WriteFloat64(nint handle, double value) { - return RunWriterOp(handle, writerHandle => writerHandle.WriteFloat64(value)); + return RunWriterOp(handle, value, static (writerHandle, state) => writerHandle.WriteFloat64(state)); } [UnmanagedCallersOnly(EntryPoint = "xl_write_bool")] public static int WriteBool(nint handle, int value) { - return RunWriterOp(handle, writerHandle => writerHandle.WriteBool(value != 0)); + return RunWriterOp(handle, value != 0, static (writerHandle, state) => writerHandle.WriteBool(state)); } [UnmanagedCallersOnly(EntryPoint = "xl_write_date")] public static int WriteDate(nint handle, int daysSinceEpoch) { - return RunWriterOp(handle, writerHandle => writerHandle.WriteDate(daysSinceEpoch)); + return RunWriterOp(handle, daysSinceEpoch, static (writerHandle, state) => writerHandle.WriteDate(state)); } [UnmanagedCallersOnly(EntryPoint = "xl_write_time")] public static int WriteTime(nint handle, long microsecondsSinceMidnight) { - return RunWriterOp(handle, writerHandle => writerHandle.WriteTime(microsecondsSinceMidnight)); + return RunWriterOp(handle, microsecondsSinceMidnight, static (writerHandle, state) => writerHandle.WriteTime(state)); } [UnmanagedCallersOnly(EntryPoint = "xl_write_timestamp")] public static int WriteTimestamp(nint handle, long microsecondsSinceEpoch) { - return RunWriterOp(handle, writerHandle => writerHandle.WriteTimestamp(microsecondsSinceEpoch)); + return RunWriterOp(handle, microsecondsSinceEpoch, static (writerHandle, state) => writerHandle.WriteTimestamp(state)); } [UnmanagedCallersOnly(EntryPoint = "xl_write_null")] public static int WriteNull(nint handle, int type) { - return RunWriterOp(handle, writerHandle => writerHandle.WriteNull(type)); + return RunWriterOp(handle, type, static (writerHandle, state) => writerHandle.WriteNull(state)); } [UnmanagedCallersOnly(EntryPoint = "xl_end_row")] @@ -355,6 +355,25 @@ private static int RunWriterOp(nint handle, Action operation } } + private static int RunWriterOp(nint handle, TState state, Action operation) + { + NativeApi.ClearLastError(); + if (!TryResolveWriter(handle, out NativeWriterHandle? writerHandle)) + { + return NativeStatus.InvalidHandle; + } + try + { + operation(writerHandle, state); + return NativeStatus.Ok; + } + catch (Exception exception) + { + NativeApi.SetLastError(exception.Message); + return NativeStatus.Error; + } + } + private static bool TryResolveWriter(nint handle, [NotNullWhen(true)] out NativeWriterHandle? writerHandle) { writerHandle = NativeHandleTable.Resolve(handle); diff --git a/src/ExcelReader.Native/pgo/excelreader.mibc b/src/ExcelReader.Native/pgo/excelreader.mibc index d9c38b97..b07c17ed 100644 Binary files a/src/ExcelReader.Native/pgo/excelreader.mibc and b/src/ExcelReader.Native/pgo/excelreader.mibc differ diff --git a/tests/ExcelReader.Tests/Generator/GeneratorTests.cs b/tests/ExcelReader.Tests/Generator/GeneratorTests.cs index 25a955f2..03f39e81 100644 --- a/tests/ExcelReader.Tests/Generator/GeneratorTests.cs +++ b/tests/ExcelReader.Tests/Generator/GeneratorTests.cs @@ -1031,7 +1031,7 @@ public ref partial struct Reading Assert.Contains("ref partial struct Reading : global::ExcelReader.Core.Parser.IExcelRowMap<", generated, StringComparison.Ordinal); Assert.Contains("global::ExcelReader.Core.Writer.IExcelRecordMap<", generated, StringComparison.Ordinal); Assert.Contains("global::ExcelReader.Core.Parser.ExcelCellReaders.Utf8(in c, in x, out global::System.ReadOnlySpan v)", generated, StringComparison.Ordinal); - Assert.Contains("row.Write(global::System.Text.Encoding.UTF8.GetString(m.Station))", generated, StringComparison.Ordinal); + Assert.Contains("row.WriteUtf8(m.Station)", generated, StringComparison.Ordinal); } } } diff --git a/tests/ExcelReader.Tests/Parser/EnumParsingTests.cs b/tests/ExcelReader.Tests/Parser/EnumParsingTests.cs index f46a6157..dc2a24ec 100644 --- a/tests/ExcelReader.Tests/Parser/EnumParsingTests.cs +++ b/tests/ExcelReader.Tests/Parser/EnumParsingTests.cs @@ -76,6 +76,95 @@ public void Should_ParseAnUnsignedEnum_When_AMemberExceedsInt64(string text, ulo } } + private enum Signed : long + { + Min = long.MinValue, + Negative = -5, + Zero = 0, + Max = long.MaxValue, + } + + private enum Wide : ulong + { + AboveLong = 9223372036854775808, + Max = ulong.MaxValue, + } + + [Theory] + [InlineData("-9223372036854775808", long.MinValue)] + [InlineData("-5", -5L)] + [InlineData("-0", 0L)] + [InlineData("0", 0L)] + [InlineData("9223372036854775807", long.MaxValue)] + public void Should_ResolveSignedMembers_When_TheyReachTheInt64Boundaries(string text, long expected) + { + Assert.True(TryParse(CellType.Number, text, out Signed value)); + Assert.Equal((Signed)expected, value); + } + + [Theory] + [InlineData("9223372036854775808")] + [InlineData("-9223372036854775809")] + [InlineData("-4")] + [InlineData("-")] + [InlineData("5.5")] + [InlineData("1E3")] + public void Should_RejectUndeclaredSignedValues(string text) + { + Assert.False(TryParse(CellType.Number, text, out Signed _)); + } + + [Theory] + [InlineData("9223372036854775808", 9223372036854775808UL)] + [InlineData("18446744073709551615", ulong.MaxValue)] + public void Should_ResolveMembersOnlyRepresentableAsUnsigned(string text, ulong expected) + { + Assert.True(TryParse(CellType.Number, text, out Wide value)); + Assert.Equal((Wide)expected, value); + Assert.False(TryParse(CellType.Number, "9223372036854775807", out Wide _)); + } + + [Theory] + [InlineData(-5.0, -5L)] + [InlineData(0.0, 0L)] + [InlineData(-9223372036854775808.0, long.MinValue)] + public void Should_ResolveBinaryNumbers_When_TheyAreIntegral(double number, long expected) + { + Cell cell = new(CellType.Number, default, number, hasNumber: true, styleIndex: 0); + + Assert.True(ExcelCellReaders.Enum(in cell, default, out Signed value)); + Assert.Equal((Signed)expected, value); + } + + [Theory] + [InlineData(-4.0)] + [InlineData(0.5)] + [InlineData(9223372036854775808.0)] + [InlineData(double.NaN)] + [InlineData(double.PositiveInfinity)] + public void Should_RejectBinaryNumbers_When_NotADeclaredIntegral(double number) + { + Cell cell = new(CellType.Number, default, number, hasNumber: true, styleIndex: 0); + + Assert.False(ExcelCellReaders.Enum(in cell, default, out Signed _)); + } + + [Fact] + public void Should_ResolveAUnsignedMemberFromABinaryNumber() + { + Cell cell = new(CellType.Number, default, 9223372036854775808.0, hasNumber: true, styleIndex: 0); + + Assert.True(ExcelCellReaders.Enum(in cell, default, out Wide value)); + Assert.Equal(Wide.AboveLong, value); + } + + [Fact] + public void Should_ResolveAnAliasFromNumericText() + { + Assert.True(TryParse(CellType.Number, "1", out Aliased value)); + Assert.Equal(Aliased.First, value); + } + [Fact] public void Should_ReadABinaryNumberAsItsStoredDouble() { diff --git a/tests/ExcelReader.Tests/Parser/ParallelCsv/AggregateCsvModelTests.cs b/tests/ExcelReader.Tests/Parser/ParallelCsv/AggregateCsvModelTests.cs index a476e06b..e2b72f5c 100644 --- a/tests/ExcelReader.Tests/Parser/ParallelCsv/AggregateCsvModelTests.cs +++ b/tests/ExcelReader.Tests/Parser/ParallelCsv/AggregateCsvModelTests.cs @@ -375,8 +375,15 @@ public override readonly int GetHashCode() return Last.GetHashCode(); } - public static bool operator ==(WideRow left, WideRow right) => left.Equals(right); - public static bool operator !=(WideRow left, WideRow right) => !left.Equals(right); + public static bool operator ==(WideRow left, WideRow right) + { + return left.Equals(right); + } + + public static bool operator !=(WideRow left, WideRow right) + { + return !left.Equals(right); + } } private sealed class WideLog : IRowAccumulator diff --git a/tests/ExcelReader.Tests/Reader/Xlsx/XlsxCanonicalCellTests.cs b/tests/ExcelReader.Tests/Reader/Xlsx/XlsxCanonicalCellTests.cs index 0323414f..73a0f817 100644 --- a/tests/ExcelReader.Tests/Reader/Xlsx/XlsxCanonicalCellTests.cs +++ b/tests/ExcelReader.Tests/Reader/Xlsx/XlsxCanonicalCellTests.cs @@ -24,6 +24,174 @@ public void CellsInExcelsShapeReadTheSameAsThroughTheGeneralPath() Assert.Equal(expected, actual); } + private static readonly string[] FormulaCells = + [ + """A{r}+13""", + """A1*24.5""", + """6""", + """{1;2}45000""", + """IF(A1>2,"a<b",'q"')a<b""", + """A1&"x"A_x0041_""", + """"x" x """, + """TRUE()1""", + """1/0#DIV/0!""", + """"s"2""", + """NOW()""", + """A1""", + """A1""", + """A1>20""", + """7""", + """RAND()0.25""", + """8""", + """X2024-01-02T03:04:05""", + """A19""", + """"y"no ref""", + """"s"12345678901""", + """""", + ]; + + [Fact] + public void FormulaCellsReadTheSameOnTheCanonicalPathAsOnTheGeneralPath() + { + AssertPathsAgree(FormulaCells, prefixedAgrees: true); + } + + [Fact] + public void FormulaCellsReadTheirCachedValues() + { + using XlsxWorkbook reader = Excel.FromXlsx(new MemoryStream(Workbook(Rows(FormulaCells, 1)))); + using XlsxWorkbook.Enumerator rows = reader.FirstSheet.GetEnumerator(); + Assert.True(rows.MoveNext()); + Row row = rows.Current; + + Assert.True(row[0].TryGetDouble(out double a)); + Assert.Equal(3, a); + Assert.Equal("6", row[2].GetString()); + Assert.Equal(CellType.Date, row[3].Type); + Assert.Equal("aplain {r}""", + """rich text""", + """ spaced """, + """A_x0042_C""", + """under_score""", + """""", + """""", + """a&b""", + """xph""", + """styled""", + """no ref""", + """""", + """""", + """a_x000D_b""", + ]; + + [Fact] + public void InlineCellsReadTheSameOnTheCanonicalPathAsOnTheGeneralPath() + { + AssertPathsAgree(InlineCells, prefixedAgrees: true); + } + + [Fact] + public void AnInlineCellWithAValueReadsTheSameOnTheCanonicalPath() + { + AssertPathsAgree(["""5""", """xy"""], prefixedAgrees: false); + } + + [Fact] + public void InlineCellsReadTheirText() + { + using XlsxWorkbook reader = Excel.FromXlsx(new MemoryStream(Workbook(Rows(InlineCells, 1)))); + using XlsxWorkbook.Enumerator rows = reader.FirstSheet.GetEnumerator(); + Assert.True(rows.MoveNext()); + Row row = rows.Current; + + Assert.Equal("plain 1", row[0].GetString()); + Assert.Equal("rich text", row[1].GetString()); + Assert.Equal(" spaced ", row[2].GetString()); + Assert.Equal("ABC", row[3].GetString()); + Assert.Equal("under_score", row[4].GetString()); + Assert.Equal("a&b", row[7].GetString()); + Assert.Equal("x", row[8].GetString()); + Assert.Equal(1, row[9].StyleIndex); + Assert.Equal("no ref", row[10].GetString()); + } + + private static void AssertPathsAgree(string[] cells, bool prefixedAgrees) + { + const int RowCount = 300; + string sheet = Rows(cells, RowCount); + byte[] plain = Workbook(sheet); + using MemoryStream general = WorkbookBuilder.Build(sheet.Replace("{StylesInner("")}"); + + List expected = Snapshot(general); + + Assert.Equal(RowCount, expected.Count); + Assert.Equal(expected, Snapshot(new MemoryStream(plain))); + Assert.Equal(expected, TrickleSnapshot(plain)); + if (!prefixedAgrees) + { + return; + } + string prefixedSheet = sheet.Replace("<", "{StylesInner("")}"); + return ms.ToArray(); + } + + private static string Rows(string[] cells, int rows) + { + var sb = new StringBuilder(); + for (int r = 1; r <= rows; r++) + { + string n = r.ToString(CultureInfo.InvariantCulture); + sb.Append(CultureInfo.InvariantCulture, $""""""); + foreach (string cell in cells) + { + sb.Append(cell.Replace("{r}", n, StringComparison.Ordinal)); + } + sb.Append(""); + } + return sb.ToString(); + } + + // Seven bytes per read, so every row is cut and re-parsed at every few bytes of its cells. + private static List TrickleSnapshot(byte[] workbook) + { + using XlsxWorkbook reader = Excel.FromXlsx(new MemoryStream(workbook)); + reader.OpenSheetWithoutPrefetch(0).Dispose(); + using var sheet = new MemoryStream(); + using (Stream source = reader.OpenSheetStream(0)) + { + source.CopyTo(sheet); + } + reader.Lifetime.Acquire(reader); + using var rows = new XlsxWorkbook.Enumerator(reader, new TrickleStream(sheet.ToArray())); + var snapshot = new List(); + while (rows.MoveNext()) + { + snapshot.Add(Render(rows.Current)); + } + return snapshot; + } + private static string SharedStrings(string p) { return $"<{p}si><{p}t>alpha<{p}si><{p}t>beta<{p}si><{p}t>gamma"; @@ -66,20 +234,26 @@ private static List Snapshot(Stream workbook) var rows = new List(); foreach (Row row in reader.FirstSheet) { - var sb = new StringBuilder(); - foreach (RowCell rowCell in row.Cells) + rows.Add(Render(row)); + } + return rows; + } + + private static string Render(Row row) + { + var sb = new StringBuilder(); + foreach (RowCell rowCell in row.Cells) + { + Cell cell = rowCell.Value; + string number = "-"; + if (cell.TryGetDouble(out double value)) { - Cell cell = rowCell.Value; - string number = "-"; - if (cell.TryGetDouble(out double value)) - { - number = value.ToString("R", CultureInfo.InvariantCulture); - } - sb.Append(CultureInfo.InvariantCulture, $"{rowCell.ColumnIndex}:{cell.Type}:{cell.StyleIndex}:{cell.GetString()}:{number}|"); + number = value.ToString("R", CultureInfo.InvariantCulture); } - rows.Add(sb.ToString()); + string date = cell.TryGetDateTime(false, out DateTime dt) ? dt.ToString("O", CultureInfo.InvariantCulture) : "-"; + sb.Append(CultureInfo.InvariantCulture, $"{rowCell.ColumnIndex}:{cell.Type}:{cell.StyleIndex}:{cell.GetString()}:{number}:{date}|"); } - return rows; + return sb.ToString(); } } } diff --git a/tests/ExcelReader.Tests/Reader/Zip/ZipChecksumTests.cs b/tests/ExcelReader.Tests/Reader/Zip/ZipChecksumTests.cs index 0e104524..d6176c40 100644 --- a/tests/ExcelReader.Tests/Reader/Zip/ZipChecksumTests.cs +++ b/tests/ExcelReader.Tests/Reader/Zip/ZipChecksumTests.cs @@ -59,6 +59,35 @@ public void AnIntactPackageStillReads(bool xlsb, bool fromStream, bool prefetch) Assert.Equal((long)Rows * (Rows + 1) / 2, sum); } + [Theory] + [MemberData(nameof(Cases))] + public void ADamagedSheetWithALongTailAfterSheetDataIsRefused(bool xlsb, bool fromStream, bool prefetch) + { + string sheet = xlsb ? "xl/worksheets/sheet1.bin" : "xl/worksheets/sheet1.xml"; + byte[] package = Damage(WithLongTail(Write(xlsb), sheet, xlsb), xlsb, sheet); + var options = new ExcelReaderOptions { PrefetchDecompression = prefetch }; + + InvalidDataException error = Assert.Throws(() => Sum(package, fromStream, options)); + Assert.Contains("CRC-32", error.Message, StringComparison.Ordinal); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ADamagedSheetWithALongTailIsRefusedByAParallelRead(bool xlsb) + { + string sheet = xlsb ? "xl/worksheets/sheet1.bin" : "xl/worksheets/sheet1.xml"; + byte[] package = Damage(WithLongTail(Write(xlsb), sheet, xlsb), xlsb, sheet); + using IExcelWorkbook workbook = Excel.Open(package); + + await Assert.ThrowsAsync(async () => + await SheetParallel.AggregateAsync( + workbook.FirstSheet, + new RowAggregation { Seed = static () => 0, Accumulate = static (ref int s, Row r) => s++, Combine = static (a, b) => a + b }, + new SheetParallelOptions { DegreeOfParallelism = 4 }, + TestContext.Current.CancellationToken)); + } + [Theory] [InlineData(false)] [InlineData(true)] @@ -118,6 +147,52 @@ private static byte[] Write(bool xlsb, bool sharedStrings = false) return ms.ToArray(); } + // Content after the sheet data that a reader has no reason to parse, long enough to span many buffer fills: + // an XML comment after , or one large record after BrtEndSheetData. + private static byte[] WithLongTail(byte[] package, string entryName, bool xlsb) + { + using var output = new MemoryStream(); + using (var source = new ZipArchive(new MemoryStream(package), ZipArchiveMode.Read)) + using (var target = new ZipArchive(output, ZipArchiveMode.Create, leaveOpen: true)) + { + foreach (ZipArchiveEntry entry in source.Entries) + { + using var copy = new MemoryStream(); + using (Stream data = entry.Open()) + { + data.CopyTo(copy); + } + byte[] content = copy.ToArray(); + if (string.Equals(entry.FullName, entryName, StringComparison.Ordinal)) + { + content = xlsb ? WithTailRecord(content) : WithTailComment(content); + } + using Stream written = target.CreateEntry(entry.FullName, CompressionLevel.NoCompression).Open(); + written.Write(content); + } + } + return output.ToArray(); + } + + private static byte[] WithTailComment(byte[] xml) + { + string text = Encoding.UTF8.GetString(xml); + return Encoding.UTF8.GetBytes(text.Replace("", "", StringComparison.Ordinal)); + } + + // BrtEndSheetData is record 146 with no payload: 0x92 0x01 0x00. The tail record is id 0x7F, length 1 MiB. + private static byte[] WithTailRecord(byte[] bin) + { + int end = bin.AsSpan().LastIndexOf(new byte[] { 0x92, 0x01, 0x00 }) + 3; + Assert.True(end > 3, "BrtEndSheetData not found"); + byte[] tail = new byte[4 + (1 << 20)]; + tail[0] = 0x7F; + tail[1] = 0x80; + tail[2] = 0x80; + tail[3] = 0x40; + return [.. bin.AsSpan(0, end), .. tail, .. bin.AsSpan(end)]; + } + // The package is stored, not deflated, so one changed letter of the cell text "text" stays valid // XML or BIFF12 and only the checksum can tell. private static byte[] Damage(byte[] package, bool xlsb, string entryName) diff --git a/tests/ExcelReader.Tests/Writer/Parallel/WorkerStringCacheTests.cs b/tests/ExcelReader.Tests/Writer/Parallel/WorkerStringCacheTests.cs index 8aacce49..dd593d64 100644 --- a/tests/ExcelReader.Tests/Writer/Parallel/WorkerStringCacheTests.cs +++ b/tests/ExcelReader.Tests/Writer/Parallel/WorkerStringCacheTests.cs @@ -50,8 +50,8 @@ public void A_Text_Added_By_One_Path_Should_Keep_Its_Index_On_The_Other() var table = new SharedStringTable(); int serialA = table.GetOrAdd("a"); - int concurrentA = table.GetOrAddConcurrent("a"); - int concurrentB = table.GetOrAddConcurrent("b"); + int concurrentA = table.GetOrAddConcurrent("a", "a"); + int concurrentB = table.GetOrAddConcurrent("b", owned: null); int serialB = table.GetOrAdd("b".AsSpan()); int serialC = table.GetOrAdd("c"); @@ -77,7 +77,7 @@ public void GetOrAddConcurrent_Should_Give_Every_Thread_The_Same_Dense_Indexes() for (int step = 0; step < texts; step++) { int i = (step * 7919 + thread * 104_729) % texts; - indexes[i] = table.GetOrAddConcurrent(values[i]); + indexes[i] = table.GetOrAddConcurrent(values[i], thread % 2 == 0 ? values[i] : null); } seen[thread] = indexes; });