Feature/add testrun retries to output

CHANGELOG.md

@@ -1,5 +1,10 @@
 # test-runner - Changelog
 
+## 3.3.0 - 2026-02-25
+
+* Add `numberOfResets` to `TestRunnerResult`, tracking how many times a test run was cancelled and restarted due to lack of progress.
+  * Included in JSON output and JUnit XML report properties.
+
 ## 3.2.0 - 2024-12-10
 
 * Add `QueryHelper.setGlobalRetryOptions()` static method.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@apexdevtools/test-runner",
-  "version": "3.2.0",
+  "version": "3.3.0",
   "description": "Apex parallel test runner with reliability goodness",
   "author": {
     "name": "Apex Dev Tools Team",

src/command/Testall.ts

@@ -214,6 +214,8 @@ export class Testall {
         Promise.resolve(missingTests),
         store
       );
+
+      store.numberOfResets = result.numberOfResets;
     }
   }
 

src/results/OutputGenerator.ts

@@ -19,6 +19,7 @@ export interface TestRunSummary {
   runIds: string[];
   reruns: TestRerun[];
   coverageResult?: CoverageReport;
+  numberOfResets: number;
 }
 
 export interface OutputGenerator {

src/results/ReportGenerator.ts

@@ -41,9 +41,16 @@ export class ReportGenerator implements OutputGenerator {
     fileName: string,
     runSummary: TestRunSummary
   ): void {
-    const { startTime, testResults, runResult, reruns } = runSummary;
+    const { startTime, testResults, runResult, reruns, numberOfResets } =
+      runSummary;
     const results = testResults as ExtendedApexTestResult[];
-    const summary = this.summary(startTime, results, runResult, reruns);
+    const summary = this.summary(
+      startTime,
+      results,
+      runResult,
+      reruns,
+      numberOfResets
+    );
     logger.logOutputFile(
       path.join(outputDirBase, fileName + '.xml'),
       this.generateJunit(summary, results)
@@ -58,7 +65,8 @@ export class ReportGenerator implements OutputGenerator {
     startTime: Date,
     testResults: ExtendedApexTestResult[],
     runResults: ApexTestRunResult,
-    reruns: TestRerun[]
+    reruns: TestRerun[],
+    numberOfResets: number
   ): SummaryData {
     // combine test and method names for fullname
     testResults.forEach(test => {
@@ -130,6 +138,7 @@ export class ReportGenerator implements OutputGenerator {
       username: this.username,
       testRunId: runResults.AsyncApexJobId,
       userId: runResults.UserId,
+      numberOfResets,
     };
   }
 
@@ -180,6 +189,7 @@ export class ReportGenerator implements OutputGenerator {
     junit += `            <property name="username" value="${summary.username}"/>\n`;
     junit += `            <property name="testRunId" value="${summary.testRunId}"/>\n`;
     junit += `            <property name="userId" value="${summary.userId}"/>\n`;
+    junit += `            <property name="numberOfResets" value="${summary.numberOfResets}"/>\n`;
     junit += '        </properties>\n';
     testResults.forEach(test => {
       const success = test.Outcome === 'Pass';
@@ -229,7 +239,8 @@ export class ReportGenerator implements OutputGenerator {
     json += `    "orgId": "${summary.orgId}",\n`;
     json += `    "username": "${summary.username}",\n`;
     json += `    "testRunId": "${summary.testRunId}",\n`;
-    json += `    "userId": "${summary.userId}"\n`;
+    json += `    "userId": "${summary.userId}",\n`;
+    json += `    "numberOfResets": ${summary.numberOfResets}\n`;
     json += '  },\n';
     json += '  "tests": [\n';
 
@@ -298,4 +309,5 @@ interface SummaryData {
   username: string;
   testRunId: string;
   userId: string;
+  numberOfResets: number;
 }

src/results/TestResultStore.ts

@@ -21,6 +21,7 @@ export class TestResultStore {
   asyncError?: TestError;
   reruns: TestRerun[];
   coverage?: CoverageReport;
+  numberOfResets: number;
 
   get resultsArray() {
     return Array.from(this.tests.values());
@@ -31,6 +32,7 @@ export class TestResultStore {
     this.runIds = [];
     this.tests = new Map<string, ApexTestResult>();
     this.reruns = [];
+    this.numberOfResets = 0;
   }
 
   public saveAsyncResult(res: TestRunnerResult): void {
@@ -43,6 +45,7 @@ export class TestResultStore {
     });
 
     this.asyncError = res.error;
+    this.numberOfResets = res.numberOfResets;
   }
 
   public saveSyncResult(reruns: TestRerun[]): void {
@@ -88,6 +91,7 @@ export class TestResultStore {
       runIds: this.runIds,
       reruns: this.reruns,
       coverageResult: this.coverage,
+      numberOfResets: this.numberOfResets,
     };
   }
 

src/runner/TestRunner.ts

@@ -44,6 +44,7 @@ export interface TestRunnerResult {
   run: ApexTestRunResult;
   tests: ApexTestResult[];
   error?: TestError;
+  numberOfResets: number; // Track the number of times the test run has been reset due to hanging or cancellation
 }
 
 export interface TestRunner {
@@ -66,7 +67,7 @@ export class AsyncTestRunner implements TestRunner {
   private readonly _testItems: TestItem[];
   private readonly _options: TestRunnerOptions;
   private readonly _testService: TestService;
-  private _stats;
+  private _stats: TestStats;
 
   static forClasses(
     logger: Logger,
@@ -141,6 +142,9 @@ export class AsyncTestRunner implements TestRunner {
       token
     );
 
+    // Add numberOfResets to the result object
+    result.numberOfResets = this._stats.getNumberOfTimesReset();
+
     // Ensure result for partial reporting
     try {
       if (token?.isCancellationRequested) {
@@ -220,6 +224,7 @@ export class AsyncTestRunner implements TestRunner {
         return (lastResult = {
           run,
           tests,
+          numberOfResets: this._stats.getNumberOfTimesReset(),
         });
       },
 

test/command/TestDebugLogs.spec.ts

@@ -74,6 +74,7 @@ describe('TestDebugLogs', () => {
     const runner = new MockTestRunner({
       run: runnerResult,
       tests: mockTestResults,
+      numberOfResets: 0,
     });
     const methodCollector = mockDefaultCollector(logger, mockConnection);
     qhStub.query.onCall(0).resolves([]);
@@ -110,6 +111,7 @@ describe('TestDebugLogs', () => {
     const runner = new MockTestRunner({
       run: runnerResult,
       tests: mockTestResults,
+      numberOfResets: 0,
     });
     const methodCollector = mockDefaultCollector(logger, mockConnection);
     qhStub.query.onCall(0).resolves([{ Id: 'AnId' }]); // User Id
@@ -154,6 +156,7 @@ describe('TestDebugLogs', () => {
     const runner = new MockTestRunner({
       run: runnerResult,
       tests: mockTestResults,
+      numberOfResets: 0,
     });
     const methodCollector = mockDefaultCollector(logger, mockConnection);
     qhStub.query.onCall(0).resolves([{ Id: 'AnId' }]); // User Id
@@ -198,6 +201,7 @@ describe('TestDebugLogs', () => {
     const runner = new MockTestRunner({
       run: runnerResult,
       tests: mockTestResults,
+      numberOfResets: 0,
     });
     const { classId, className, methodName } = defaultTestInfo;
     const methodCollector = new MockTestMethodCollector(

test/command/Testall.spec.ts

@@ -176,6 +176,7 @@ describe('TestAll', () => {
     const runner = new MockTestRunner({
       run: mockRunResult,
       tests: mockTestResults,
+      numberOfResets: 1,
     });
     const testMethods = mockDefaultCollector(logger, mockConnection);
 
@@ -197,6 +198,7 @@ describe('TestAll', () => {
       reruns: [],
       runIds: [testRunId],
       coverageResult: undefined,
+      numberOfResets: 1,
     });
   });
 
@@ -208,6 +210,7 @@ describe('TestAll', () => {
     const runner = new MockTestRunner({
       run: mockRunResult,
       tests: [],
+      numberOfResets: 0,
     });
     const testMethods = mockDefaultCollector(logger, mockConnection);
 
@@ -247,6 +250,7 @@ describe('TestAll', () => {
     const runner = new MockTestRunner({
       run: mockRunResult,
       tests: mockTestResults,
+      numberOfResets: 0,
     });
     const testMethods = mockDefaultCollector(logger, mockConnection);
 
@@ -294,6 +298,7 @@ describe('TestAll', () => {
     const runner = new MockTestRunner({
       run: mockRunResult,
       tests: mockTestResults,
+      numberOfResets: 0,
     });
     const testMethods = mockDefaultCollector(logger, mockConnection);
     const mockTestResult = {
@@ -343,6 +348,7 @@ describe('TestAll', () => {
     const runner = new MockTestRunner({
       run: mockRunResult,
       tests: mockTestResults,
+      numberOfResets: 0,
     });
     const testMethods = mockDefaultCollector(logger, mockConnection);
     const mockTestResult = {
@@ -398,6 +404,7 @@ describe('TestAll', () => {
     const runner = new MockTestRunner({
       run: mockRunResult,
       tests: mockTestResults,
+      numberOfResets: 0,
     });
     const testMethods = mockDefaultCollector(logger, mockConnection);
     testingServiceSyncStub.rejects(new Error('Request Error'));
@@ -465,6 +472,7 @@ describe('TestAll', () => {
     const runner = new MockTestRunner({
       run: mockRunResult,
       tests: mockTestResults,
+      numberOfResets: 0,
     });
     const testMethods = mockDefaultCollector(logger, mockConnection);
     const mockTestResult = {
@@ -524,6 +532,7 @@ describe('TestAll', () => {
     const runner = new MockTestRunner({
       run: mockRunResult,
       tests: mockTestResults,
+      numberOfResets: 0,
     });
     const testMethods = mockDefaultCollector(logger, mockConnection);
 
@@ -590,6 +599,7 @@ describe('TestAll', () => {
     const runner = new MockTestRunner({
       run: mockRunResult,
       tests: mockTestResults,
+      numberOfResets: 0,
     });
     const testMethods = mockDefaultCollector(logger, mockConnection);
 
@@ -663,9 +673,11 @@ describe('TestAll', () => {
     const runner = new MockTestRunner({
       run: mockRunResult,
       tests: [mockTestResults[0]],
+      numberOfResets: 0,
     }).addNextResult({
       run: mockRunResult,
       tests: [mockTestResults[1]],
+      numberOfResets: 0,
     });
     const testMethods = new MockTestMethodCollector(
       logger,

test/report/ClassTimeGenerator.spec.ts

@@ -100,6 +100,7 @@ describe('ClassTimeGenerator', () => {
       },
       runIds: ['job Id'],
       reruns: [],
+      numberOfResets: 0,
     });
 
     expect(logger.files.length).to.equal(2);

test/report/CoverageReporter.spec.ts

@@ -71,6 +71,7 @@ describe('CoverageReporter', () => {
           },
         ],
       },
+      numberOfResets: 0,
     });
 
     expect(executeStub.calledOnce).to.be.true;
@@ -99,6 +100,7 @@ describe('CoverageReporter', () => {
       runIds: ['job Id'],
       reruns: [],
       coverageResult: undefined,
+      numberOfResets: 0,
     });
 
     expect(executeStub.called).to.be.false;

test/report/ExecutionMapGenerator.spec.ts

@@ -100,6 +100,7 @@ describe('ExecutionMapGenerator', () => {
       },
       runIds: ['job Id'],
       reruns: [],
+      numberOfResets: 0,
     });
 
     expect(logger.files.length).to.equal(1);