Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 78 additions & 0 deletions .github/workflows/nuget-benchmark.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
name: NuGet Benchmark

on:
workflow_dispatch:
inputs:
rows:
description: Workbook row count
required: true
default: '100000'
type: string
columns:
description: Workbook column count
required: true
default: '10'
type: string
iterations:
description: Fresh processes per runtime and scenario
required: true
default: '5'
type: string
schedule:
- cron: '17 4 * * 1'

permissions:
contents: read

env:
CARGO_TERM_COLOR: always
BENCHMARK_ROWS: ${{ inputs.rows || '100000' }}
BENCHMARK_COLUMNS: ${{ inputs.columns || '10' }}
BENCHMARK_ITERATIONS: ${{ inputs.iterations || '5' }}
PACKAGE_VERSION: 0.1.0-benchmark.${{ github.run_number }}

jobs:
benchmark:
name: Benchmark ${{ matrix.rid }}
strategy:
fail-fast: false
matrix:
include:
- runner: windows-latest
rid: win-x64
- runner: windows-11-arm
rid: win-arm64
- runner: ubuntu-latest
rid: linux-x64
- runner: ubuntu-24.04-arm
rid: linux-arm64
- runner: macos-15-intel
rid: osx-x64
- runner: macos-latest
rid: osx-arm64
runs-on: ${{ matrix.runner }}
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@1.85.0
- uses: actions/setup-dotnet@v4
with:
dotnet-version: 8.0.x
- uses: Swatinem/rust-cache@v2
with:
key: nuget-benchmark-${{ matrix.rid }}
- name: Run NuGet stress benchmark
shell: pwsh
run: >-
./scripts/compare-nuget-v1-rust.ps1
-Rid '${{ matrix.rid }}'
-Rows $env:BENCHMARK_ROWS
-Columns $env:BENCHMARK_COLUMNS
-Iterations $env:BENCHMARK_ITERATIONS
-MiniExcelRustVersion $env:PACKAGE_VERSION
- uses: actions/upload-artifact@v4
with:
name: nuget-benchmark-${{ matrix.rid }}
path: |
target/benchmarks/nuget-v1/benchmark-${{ matrix.rid }}.json
target/benchmarks/nuget-v1/benchmark-${{ matrix.rid }}.md
if-no-files-found: error
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
/target/
/benchmarks/dotnet-v1-query/bin/
/benchmarks/dotnet-v1-query/obj/
/benchmarks/nuget-v1-query/bin/
/benchmarks/nuget-v1-query/obj/
/dotnet/**/bin/
/dotnet/**/obj/
/web-demo/dist/
Expand Down
17 changes: 17 additions & 0 deletions benchmarks/nuget-v1-query/NuGetV1Query.csproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<MiniExcelRustPackageVersion Condition="'$(MiniExcelRustPackageVersion)' == ''">0.1.0-benchmark</MiniExcelRustPackageVersion>
<MiniExcelVersion Condition="'$(MiniExcelVersion)' == ''">1.46.0</MiniExcelVersion>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="MiniExcel" Version="[$(MiniExcelVersion)]" />
<PackageReference Include="MiniExcel.Rust" Version="$(MiniExcelRustPackageVersion)" />
</ItemGroup>

</Project>
238 changes: 238 additions & 0 deletions benchmarks/nuget-v1-query/Program.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,238 @@
using System.Diagnostics;
using System.Globalization;
using System.IO.Compression;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using MiniExcelLibs;
using ManagedMiniExcel = MiniExcelLibs.MiniExcel;

if (args.Length == 0)
return Usage();

return args[0].ToLowerInvariant() switch
{
"generate" => Generate(args),
"verify" => Verify(args),
"managed" => Benchmark(args, useRust: false),
"rust" => Benchmark(args, useRust: true),
_ => Usage()
};

static int Generate(string[] arguments)
{
if (arguments.Length != 4 ||
!int.TryParse(arguments[2], out var rowCount) || rowCount < 1 ||
!int.TryParse(arguments[3], out var columnCount) || columnCount is < 1 or > 26)
return Usage();

CreateWorkbook(Path.GetFullPath(arguments[1]), rowCount, columnCount);
return 0;
}

static int Verify(string[] arguments)
{
if (arguments.Length != 2)
return Usage();

var path = Path.GetFullPath(arguments[1]);
using var managed = Query(path, useRust: false).GetEnumerator();
using var rust = Query(path, useRust: true).GetEnumerator();
long rowIndex = 0;
while (true)
{
var hasManaged = managed.MoveNext();
var hasRust = rust.MoveNext();
Require(hasManaged == hasRust, $"Row count differs after row {rowIndex}.");
if (!hasManaged)
break;
CompareRows(managed.Current, rust.Current, rowIndex);
rowIndex++;
}

Console.WriteLine($"Verified {rowIndex} rows against MiniExcel 1.46.0.");
return 0;
}

static int Benchmark(string[] arguments, bool useRust)
{
if (arguments.Length is < 2 or > 4 ||
arguments.Length >= 3 && (!int.TryParse(arguments[2], out var passes) || passes < 1) ||
arguments.Length >= 4 && (!int.TryParse(arguments[3], out var warmups) || warmups < 0))
return Usage();

var path = Path.GetFullPath(arguments[1]);
var measuredPasses = arguments.Length >= 3 ? int.Parse(arguments[2], CultureInfo.InvariantCulture) : 1;
var warmupPasses = arguments.Length >= 4 ? int.Parse(arguments[3], CultureInfo.InvariantCulture) : 0;
for (var pass = 0; pass < warmupPasses; pass++)
Consume(path, useRust);

GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
var allocatedBefore = GC.GetTotalAllocatedBytes(precise: true);
var stopwatch = Stopwatch.StartNew();
var firstRowMilliseconds = 0d;
long rows = 0;
long cells = 0;
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
for (var pass = 0; pass < measuredPasses; pass++)
{
foreach (var row in Query(path, useRust))
{
if (rows == 0)
firstRowMilliseconds = stopwatch.Elapsed.TotalMilliseconds;
rows++;
cells += row.Count;
AppendRow(hash, row);
}
}
stopwatch.Stop();

Console.WriteLine(JsonSerializer.Serialize(new BenchmarkResult(
useRust ? "MiniExcel.Rust" : "MiniExcel",
Environment.Version.ToString(),
measuredPasses,
rows,
cells,
Convert.ToHexString(hash.GetHashAndReset()),
stopwatch.Elapsed.TotalMilliseconds,
firstRowMilliseconds,
GC.GetTotalAllocatedBytes(precise: true) - allocatedBefore)));
return 0;
}

static IEnumerable<IDictionary<string, object?>> Query(string path, bool useRust)
{
if (useRust)
return MiniExcelRust.Query(path, useHeaderRow: false);
return ManagedMiniExcel.Query(path, useHeaderRow: false)
.Cast<IDictionary<string, object?>>();
}

static void Consume(string path, bool useRust)
{
foreach (var row in Query(path, useRust))
_ = row.Count;
}

static void CompareRows(
IDictionary<string, object?> managed,
IDictionary<string, object?> rust,
long rowIndex)
{
Require(managed.Keys.SequenceEqual(rust.Keys, StringComparer.Ordinal),
$"Column order differs at row {rowIndex}.");
foreach (var key in managed.Keys)
{
Require(Normalize(managed[key]) == Normalize(rust[key]),
$"Value differs at row {rowIndex}, column {key}: managed={managed[key]}, rust={rust[key]}.");
}
}

static void AppendRow(IncrementalHash hash, IDictionary<string, object?> row)
{
foreach (var cell in row)
{
AppendText(hash, cell.Key);
AppendText(hash, Normalize(cell.Value));
}
}

static void AppendText(IncrementalHash hash, string value)
{
var bytes = Encoding.UTF8.GetBytes(value);
hash.AppendData(BitConverter.GetBytes(bytes.Length));
hash.AppendData(bytes);
}

static string Normalize(object? value) => value switch
{
null or DBNull => "null",
IFormattable formattable => formattable.ToString(null, CultureInfo.InvariantCulture) ?? string.Empty,
_ => value.ToString() ?? string.Empty
};

static void CreateWorkbook(string path, int rows, int columns)
{
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
if (File.Exists(path))
File.Delete(path);
using var archive = ZipFile.Open(path, ZipArchiveMode.Create);
AddEntry(archive, "[Content_Types].xml", """
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">
<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>
<Default Extension="xml" ContentType="application/xml"/>
<Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/>
<Override PartName="/xl/worksheets/sheet1.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/>
</Types>
""");
AddEntry(archive, "_rels/.rels", """
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/>
</Relationships>
""");
AddEntry(archive, "xl/workbook.xml", """
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">
<sheets><sheet name="Sheet1" sheetId="1" r:id="rId1"/></sheets>
</workbook>
""");
AddEntry(archive, "xl/_rels/workbook.xml.rels", """
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/>
</Relationships>
""");

var entry = archive.CreateEntry("xl/worksheets/sheet1.xml", CompressionLevel.Fastest);
using var writer = new StreamWriter(entry.Open(), new UTF8Encoding(false));
writer.Write("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?><worksheet xmlns=\"http://schemas.openxmlformats.org/spreadsheetml/2006/main\"><sheetData>");
for (var row = 1; row <= rows; row++)
{
writer.Write($"<row r=\"{row}\">");
for (var column = 1; column <= columns; column++)
{
var reference = $"{(char)('A' + column - 1)}{row}";
var value = (long)(row - 1) * columns + column;
writer.Write($"<c r=\"{reference}\"><v>{value}</v></c>");
}
writer.Write("</row>");
}
writer.Write("</sheetData></worksheet>");
}

static void AddEntry(ZipArchive archive, string name, string contents)
{
var entry = archive.CreateEntry(name, CompressionLevel.Fastest);
using var writer = new StreamWriter(entry.Open(), new UTF8Encoding(false));
writer.Write(contents);
}

static void Require(bool condition, string message)
{
if (!condition)
throw new InvalidOperationException(message);
}

static int Usage()
{
Console.Error.WriteLine("Usage:");
Console.Error.WriteLine(" NuGetV1Query generate <xlsx-path> <rows> <columns>");
Console.Error.WriteLine(" NuGetV1Query verify <xlsx-path>");
Console.Error.WriteLine(" NuGetV1Query <managed|rust> <xlsx-path> [passes] [warmup-passes]");
return 2;
}

internal sealed record BenchmarkResult(
string Runtime,
string DotNetRuntime,
int Passes,
long Rows,
long Cells,
string ContentHash,
double ElapsedMilliseconds,
double FirstRowMilliseconds,
long AllocatedBytes);
19 changes: 19 additions & 0 deletions docs/dotnet-v1-query-benchmark.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,25 @@ This benchmark compares dynamic, headerless XLSX streaming over the same workboo

Both runners enumerate every returned row without retaining the complete worksheet. Save performance, typed mapping, formulas, and other APIs are outside this comparison.

### NuGet-To-NuGet Stress Harness

To benchmark the distributable package rather than the Rust CLI, run:

```powershell
pwsh ./scripts/compare-nuget-v1-rust.ps1
```

This harness builds a local `MiniExcel.Rust` package, resolves the latest stable public MiniExcel v1
package, restores both into an isolated `net8.0` consumer, and compares `MiniExcel.Query` with
`MiniExcelRust.Query`. Before timing, it verifies every row, column, and normalized value. Cold and
steady scenarios run in alternating fresh processes and report query time, first-row latency,
managed allocation, peak working set, and peak private memory to
`target/benchmarks/nuget-v1/benchmark-<rid>.{json,md}`.

Use `-Rows`, `-Columns`, `-Iterations`, `-Passes`, and `-WarmupPasses` to change the load. The
`NuGet Benchmark` GitHub workflow runs the same harness on Windows, Linux, and macOS for x64 and
Arm64. Pass `-MiniExcelVersion 1.46.0` to pin a historical baseline for reproducible comparisons.

## Fairness Controls

- Both runners use Release builds, the same workbook, and equivalent public dynamic Query APIs.
Expand Down
17 changes: 17 additions & 0 deletions docs/dotnet-v1-query-benchmark.zh-CN.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,23 @@

两个 runner 都会遍历所有返回行,但不会把完整 worksheet 保留在内存中。本测试不包含 Save、类型映射、公式及其他 API。

### NuGet 对 NuGet 压力测试

如需比较实际发布包而不是 Rust CLI,请运行:

```powershell
pwsh ./scripts/compare-nuget-v1-rust.ps1
```

该脚本先构建本地 `MiniExcel.Rust` 包,自动解析 NuGet 上最新的稳定 MiniExcel v1,再创建隔离的
`net8.0` 消费者并对比 `MiniExcel.Query` 与 `MiniExcelRust.Query`。计时前会逐行、逐列、
逐值验证结果;Cold 与 Steady 场景使用交替的新进程执行,并将 Query 耗时、首行延迟、托管分配、
峰值工作集与峰值私有内存写入 `target/benchmarks/nuget-v1/benchmark-<rid>.{json,md}`。

可通过 `-Rows`、`-Columns`、`-Iterations`、`-Passes` 和 `-WarmupPasses` 调整压力。
GitHub 的 `NuGet Benchmark` workflow 会在 Windows、Linux、macOS 的 x64 与 Arm64 环境运行同一套测试。
如需复现历史结果,可传入 `-MiniExcelVersion 1.46.0` 固定基线版本。

## 公平性控制

- 两个 runner 都使用 Release 构建、同一份工作簿和语义等价的公开动态 Query API。
Expand Down
Loading
Loading