From 55af1ce403f51108b31168da44208b6c7f01d4ae Mon Sep 17 00:00:00 2001 From: Brad Wilson Date: Sun, 20 Jul 2025 14:28:42 -0700 Subject: [PATCH 1/4] Update tests to generate CTRF reports --- test/test.v3/Tests.cs | 10 ++++------ tools/builder/targets/TestCore.cs | 14 +++++++++++--- tools/builder/targets/TestFx.cs | 12 ++++++++++-- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/test/test.v3/Tests.cs b/test/test.v3/Tests.cs index 4282b24..ec94f7d 100644 --- a/test/test.v3/Tests.cs +++ b/test/test.v3/Tests.cs @@ -5,11 +5,9 @@ public class Tests [Fact] public void Passing() { } - [Fact(Explicit = true)] - public void Failing() => Assert.Fail("This is a failing test"); + [Fact(Skip = "Unconditionally skipped")] + public void Skipped() => Assert.Fail("This does not run"); - [Theory] - [InlineData(true, Explicit = true)] - [InlineData(false)] - public void ConditionalSkip(bool value) => Assert.SkipWhen(value, "Conditionally skipped"); + [Fact(Explicit = true)] + public void Explicit() => Assert.Fail("This only runs explicitly."); } diff --git a/tools/builder/targets/TestCore.cs b/tools/builder/targets/TestCore.cs index c003211..7f29c1f 100644 --- a/tools/builder/targets/TestCore.cs +++ b/tools/builder/targets/TestCore.cs @@ -12,12 +12,20 @@ public class TestCore { public static async Task OnExecute(BuildContext context) { - context.BuildStep("Running .NET Core tests"); + context.BuildStep("Running .NET tests"); Directory.CreateDirectory(context.TestOutputFolder); - File.Delete(Path.Combine(context.TestOutputFolder, "test.xunit.runner.visualstudio-netcore.trx")); - await context.Exec("dotnet", $"test test/test.xunit.runner.visualstudio -tl:off --configuration {context.Configuration} --no-build --framework net8.0 --logger trx;LogFileName=test.xunit.runner.visualstudio-netcore.trx --results-directory \"{context.TestOutputFolder}\" --verbosity {context.Verbosity}"); + var testFolder = Path.Combine(context.BaseFolder, "test", "test.xunit.runner.visualstudio", "bin", context.ConfigurationText, "net8.0"); + var testPath = Path.Combine(testFolder, "test.xunit.runner.visualstudio.dll"); + var reportPath = Path.Combine(context.TestOutputFolder, "test.xunit.runner.visualstudio-netcore.ctrf"); + File.Delete(reportPath); + + await context.Exec("dotnet", $"exec {testPath} -ctrf {reportPath}"); + + context.BuildStep("Running .NET VSTest integration tests"); + + await context.Exec("dotnet", $"test test/test.v3 -tl:off --configuration {context.Configuration} --no-build --framework net8.0 --verbosity {context.Verbosity}"); await context.Exec("dotnet", $"test test/test.v2 -tl:off --configuration {context.Configuration} --no-build --framework net8.0 --verbosity {context.Verbosity}"); } } diff --git a/tools/builder/targets/TestFx.cs b/tools/builder/targets/TestFx.cs index f6204ee..58dd984 100644 --- a/tools/builder/targets/TestFx.cs +++ b/tools/builder/targets/TestFx.cs @@ -15,9 +15,17 @@ public static async Task OnExecute(BuildContext context) context.BuildStep("Running .NET Framework tests"); Directory.CreateDirectory(context.TestOutputFolder); - File.Delete(Path.Combine(context.TestOutputFolder, "test.xunit.runner.visualstudio-netfx.trx")); - await context.Exec("dotnet", $"test test/test.xunit.runner.visualstudio -tl:off --configuration {context.Configuration} --no-build --framework net472 --logger trx;LogFileName=test.xunit.runner.visualstudio-netfx.trx --results-directory \"{context.TestOutputFolder}\" --verbosity {context.Verbosity}"); + var testFolder = Path.Combine(context.BaseFolder, "test", "test.xunit.runner.visualstudio", "bin", context.ConfigurationText, "net472"); + var testPath = Path.Combine(testFolder, "test.xunit.runner.visualstudio.exe"); + var reportPath = Path.Combine(context.TestOutputFolder, "test.xunit.runner.visualstudio-netfx.ctrf"); + File.Delete(reportPath); + + await context.Exec(testPath, $"-ctrf {reportPath}", testFolder); + + context.BuildStep("Running .NET Framework VSTest integration tests"); + + await context.Exec("dotnet", $"test test/test.v3 -tl:off --configuration {context.Configuration} --no-build --framework net472 --verbosity {context.Verbosity}"); await context.Exec("dotnet", $"test test/test.v2 -tl:off --configuration {context.Configuration} --no-build --framework net472 --verbosity {context.Verbosity}"); await context.Exec("dotnet", $"test test/test.v1 -tl:off --configuration {context.Configuration} --no-build --framework net472 --verbosity {context.Verbosity}"); } From 6156713989784288572387d4651e88af5e135d4b Mon Sep 17 00:00:00 2001 From: Brad Wilson Date: Sun, 20 Jul 2025 14:30:39 -0700 Subject: [PATCH 2/4] Add GitHub CTRF test reporting --- .github/workflows/pull-request.yaml | 12 ++++++++++++ .github/workflows/push-main.yaml | 8 ++++++++ 2 files changed, 20 insertions(+) diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index aa0748a..cea4c2f 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -40,3 +40,15 @@ jobs: path: artifacts/test compression-level: 9 if: always() + + - name: Publish Test Report + uses: ctrf-io/github-test-reporter@v1 + with: + report-path: $/artifacts/test/*.ctrf + summary-report: true + github-report: true + pull-request: true + update-comment: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + if: always() diff --git a/.github/workflows/push-main.yaml b/.github/workflows/push-main.yaml index 434999d..b96b6e6 100644 --- a/.github/workflows/push-main.yaml +++ b/.github/workflows/push-main.yaml @@ -60,3 +60,11 @@ jobs: path: artifacts/packages compression-level: 0 if: always() + + - name: Publish Test Report + uses: ctrf-io/github-test-reporter@v1 + with: + report-path: $/artifacts/test/*.ctrf + summary-report: true + github-report: true + if: always() From 1d21473bb6f46cdd4c7dad898054968a4ae9a48d Mon Sep 17 00:00:00 2001 From: Brad Wilson Date: Sun, 20 Jul 2025 14:39:54 -0700 Subject: [PATCH 3/4] Skip .NET Framework integration tests on Mono --- tools/builder/targets/TestFx.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/builder/targets/TestFx.cs b/tools/builder/targets/TestFx.cs index 58dd984..e04ee5d 100644 --- a/tools/builder/targets/TestFx.cs +++ b/tools/builder/targets/TestFx.cs @@ -23,6 +23,9 @@ public static async Task OnExecute(BuildContext context) await context.Exec(testPath, $"-ctrf {reportPath}", testFolder); + if (context.NeedMono) + return; + context.BuildStep("Running .NET Framework VSTest integration tests"); await context.Exec("dotnet", $"test test/test.v3 -tl:off --configuration {context.Configuration} --no-build --framework net472 --verbosity {context.Verbosity}"); From c03207df10fa7e2eaac22d2b1a5fb67158c5eda2 Mon Sep 17 00:00:00 2001 From: Brad Wilson Date: Sun, 20 Jul 2025 14:40:04 -0700 Subject: [PATCH 4/4] Incorrect artifact path --- .github/workflows/pull-request.yaml | 2 +- .github/workflows/push-main.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index cea4c2f..ad246a4 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -44,7 +44,7 @@ jobs: - name: Publish Test Report uses: ctrf-io/github-test-reporter@v1 with: - report-path: $/artifacts/test/*.ctrf + report-path: './artifacts/test/*.ctrf' summary-report: true github-report: true pull-request: true diff --git a/.github/workflows/push-main.yaml b/.github/workflows/push-main.yaml index b96b6e6..ba6a5df 100644 --- a/.github/workflows/push-main.yaml +++ b/.github/workflows/push-main.yaml @@ -64,7 +64,7 @@ jobs: - name: Publish Test Report uses: ctrf-io/github-test-reporter@v1 with: - report-path: $/artifacts/test/*.ctrf + report-path: './artifacts/test/*.ctrf' summary-report: true github-report: true if: always()