diff --git a/CHANGELOG.md b/CHANGELOG.md
index 83057f2..6d23400 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,10 @@
# test-runner - Changelog
+## 3.3.0 - 2026-02-25
+
+* Add `numberOfResets` to `TestRunnerResult`, tracking how many times a test run was cancelled and restarted due to lack of progress.
+ * Included in JSON output and JUnit XML report properties.
+
## 3.2.0 - 2024-12-10
* Add `QueryHelper.setGlobalRetryOptions()` static method.
diff --git a/package.json b/package.json
index 9a6b1af..baa4da1 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "@apexdevtools/test-runner",
- "version": "3.2.0",
+ "version": "3.3.0",
"description": "Apex parallel test runner with reliability goodness",
"author": {
"name": "Apex Dev Tools Team",
diff --git a/src/command/Testall.ts b/src/command/Testall.ts
index c4d07dc..7ead67c 100644
--- a/src/command/Testall.ts
+++ b/src/command/Testall.ts
@@ -214,6 +214,8 @@ export class Testall {
Promise.resolve(missingTests),
store
);
+
+ store.numberOfResets = result.numberOfResets;
}
}
diff --git a/src/results/OutputGenerator.ts b/src/results/OutputGenerator.ts
index a241e4d..c2339d3 100644
--- a/src/results/OutputGenerator.ts
+++ b/src/results/OutputGenerator.ts
@@ -19,6 +19,7 @@ export interface TestRunSummary {
runIds: string[];
reruns: TestRerun[];
coverageResult?: CoverageReport;
+ numberOfResets: number;
}
export interface OutputGenerator {
diff --git a/src/results/ReportGenerator.ts b/src/results/ReportGenerator.ts
index 7892b19..702bd2f 100644
--- a/src/results/ReportGenerator.ts
+++ b/src/results/ReportGenerator.ts
@@ -41,9 +41,16 @@ export class ReportGenerator implements OutputGenerator {
fileName: string,
runSummary: TestRunSummary
): void {
- const { startTime, testResults, runResult, reruns } = runSummary;
+ const { startTime, testResults, runResult, reruns, numberOfResets } =
+ runSummary;
const results = testResults as ExtendedApexTestResult[];
- const summary = this.summary(startTime, results, runResult, reruns);
+ const summary = this.summary(
+ startTime,
+ results,
+ runResult,
+ reruns,
+ numberOfResets
+ );
logger.logOutputFile(
path.join(outputDirBase, fileName + '.xml'),
this.generateJunit(summary, results)
@@ -58,7 +65,8 @@ export class ReportGenerator implements OutputGenerator {
startTime: Date,
testResults: ExtendedApexTestResult[],
runResults: ApexTestRunResult,
- reruns: TestRerun[]
+ reruns: TestRerun[],
+ numberOfResets: number
): SummaryData {
// combine test and method names for fullname
testResults.forEach(test => {
@@ -130,6 +138,7 @@ export class ReportGenerator implements OutputGenerator {
username: this.username,
testRunId: runResults.AsyncApexJobId,
userId: runResults.UserId,
+ numberOfResets,
};
}
@@ -180,6 +189,7 @@ export class ReportGenerator implements OutputGenerator {
junit += ` \n`;
junit += ` \n`;
junit += ` \n`;
+ junit += ` \n`;
junit += ' \n';
testResults.forEach(test => {
const success = test.Outcome === 'Pass';
@@ -229,7 +239,8 @@ export class ReportGenerator implements OutputGenerator {
json += ` "orgId": "${summary.orgId}",\n`;
json += ` "username": "${summary.username}",\n`;
json += ` "testRunId": "${summary.testRunId}",\n`;
- json += ` "userId": "${summary.userId}"\n`;
+ json += ` "userId": "${summary.userId}",\n`;
+ json += ` "numberOfResets": ${summary.numberOfResets}\n`;
json += ' },\n';
json += ' "tests": [\n';
@@ -298,4 +309,5 @@ interface SummaryData {
username: string;
testRunId: string;
userId: string;
+ numberOfResets: number;
}
diff --git a/src/results/TestResultStore.ts b/src/results/TestResultStore.ts
index 22671c4..52b923b 100644
--- a/src/results/TestResultStore.ts
+++ b/src/results/TestResultStore.ts
@@ -21,6 +21,7 @@ export class TestResultStore {
asyncError?: TestError;
reruns: TestRerun[];
coverage?: CoverageReport;
+ numberOfResets: number;
get resultsArray() {
return Array.from(this.tests.values());
@@ -31,6 +32,7 @@ export class TestResultStore {
this.runIds = [];
this.tests = new Map();
this.reruns = [];
+ this.numberOfResets = 0;
}
public saveAsyncResult(res: TestRunnerResult): void {
@@ -43,6 +45,7 @@ export class TestResultStore {
});
this.asyncError = res.error;
+ this.numberOfResets = res.numberOfResets;
}
public saveSyncResult(reruns: TestRerun[]): void {
@@ -88,6 +91,7 @@ export class TestResultStore {
runIds: this.runIds,
reruns: this.reruns,
coverageResult: this.coverage,
+ numberOfResets: this.numberOfResets,
};
}
diff --git a/src/runner/TestRunner.ts b/src/runner/TestRunner.ts
index 317c4ae..50b88c7 100644
--- a/src/runner/TestRunner.ts
+++ b/src/runner/TestRunner.ts
@@ -44,6 +44,7 @@ export interface TestRunnerResult {
run: ApexTestRunResult;
tests: ApexTestResult[];
error?: TestError;
+ numberOfResets: number; // Track the number of times the test run has been reset due to hanging or cancellation
}
export interface TestRunner {
@@ -66,7 +67,7 @@ export class AsyncTestRunner implements TestRunner {
private readonly _testItems: TestItem[];
private readonly _options: TestRunnerOptions;
private readonly _testService: TestService;
- private _stats;
+ private _stats: TestStats;
static forClasses(
logger: Logger,
@@ -141,6 +142,9 @@ export class AsyncTestRunner implements TestRunner {
token
);
+ // Add numberOfResets to the result object
+ result.numberOfResets = this._stats.getNumberOfTimesReset();
+
// Ensure result for partial reporting
try {
if (token?.isCancellationRequested) {
@@ -220,6 +224,7 @@ export class AsyncTestRunner implements TestRunner {
return (lastResult = {
run,
tests,
+ numberOfResets: this._stats.getNumberOfTimesReset(),
});
},
diff --git a/test/command/TestDebugLogs.spec.ts b/test/command/TestDebugLogs.spec.ts
index f79d088..048e4cc 100644
--- a/test/command/TestDebugLogs.spec.ts
+++ b/test/command/TestDebugLogs.spec.ts
@@ -74,6 +74,7 @@ describe('TestDebugLogs', () => {
const runner = new MockTestRunner({
run: runnerResult,
tests: mockTestResults,
+ numberOfResets: 0,
});
const methodCollector = mockDefaultCollector(logger, mockConnection);
qhStub.query.onCall(0).resolves([]);
@@ -110,6 +111,7 @@ describe('TestDebugLogs', () => {
const runner = new MockTestRunner({
run: runnerResult,
tests: mockTestResults,
+ numberOfResets: 0,
});
const methodCollector = mockDefaultCollector(logger, mockConnection);
qhStub.query.onCall(0).resolves([{ Id: 'AnId' }]); // User Id
@@ -154,6 +156,7 @@ describe('TestDebugLogs', () => {
const runner = new MockTestRunner({
run: runnerResult,
tests: mockTestResults,
+ numberOfResets: 0,
});
const methodCollector = mockDefaultCollector(logger, mockConnection);
qhStub.query.onCall(0).resolves([{ Id: 'AnId' }]); // User Id
@@ -198,6 +201,7 @@ describe('TestDebugLogs', () => {
const runner = new MockTestRunner({
run: runnerResult,
tests: mockTestResults,
+ numberOfResets: 0,
});
const { classId, className, methodName } = defaultTestInfo;
const methodCollector = new MockTestMethodCollector(
diff --git a/test/command/Testall.spec.ts b/test/command/Testall.spec.ts
index 5b86179..a20023d 100644
--- a/test/command/Testall.spec.ts
+++ b/test/command/Testall.spec.ts
@@ -176,6 +176,7 @@ describe('TestAll', () => {
const runner = new MockTestRunner({
run: mockRunResult,
tests: mockTestResults,
+ numberOfResets: 1,
});
const testMethods = mockDefaultCollector(logger, mockConnection);
@@ -197,6 +198,7 @@ describe('TestAll', () => {
reruns: [],
runIds: [testRunId],
coverageResult: undefined,
+ numberOfResets: 1,
});
});
@@ -208,6 +210,7 @@ describe('TestAll', () => {
const runner = new MockTestRunner({
run: mockRunResult,
tests: [],
+ numberOfResets: 0,
});
const testMethods = mockDefaultCollector(logger, mockConnection);
@@ -247,6 +250,7 @@ describe('TestAll', () => {
const runner = new MockTestRunner({
run: mockRunResult,
tests: mockTestResults,
+ numberOfResets: 0,
});
const testMethods = mockDefaultCollector(logger, mockConnection);
@@ -294,6 +298,7 @@ describe('TestAll', () => {
const runner = new MockTestRunner({
run: mockRunResult,
tests: mockTestResults,
+ numberOfResets: 0,
});
const testMethods = mockDefaultCollector(logger, mockConnection);
const mockTestResult = {
@@ -343,6 +348,7 @@ describe('TestAll', () => {
const runner = new MockTestRunner({
run: mockRunResult,
tests: mockTestResults,
+ numberOfResets: 0,
});
const testMethods = mockDefaultCollector(logger, mockConnection);
const mockTestResult = {
@@ -398,6 +404,7 @@ describe('TestAll', () => {
const runner = new MockTestRunner({
run: mockRunResult,
tests: mockTestResults,
+ numberOfResets: 0,
});
const testMethods = mockDefaultCollector(logger, mockConnection);
testingServiceSyncStub.rejects(new Error('Request Error'));
@@ -465,6 +472,7 @@ describe('TestAll', () => {
const runner = new MockTestRunner({
run: mockRunResult,
tests: mockTestResults,
+ numberOfResets: 0,
});
const testMethods = mockDefaultCollector(logger, mockConnection);
const mockTestResult = {
@@ -524,6 +532,7 @@ describe('TestAll', () => {
const runner = new MockTestRunner({
run: mockRunResult,
tests: mockTestResults,
+ numberOfResets: 0,
});
const testMethods = mockDefaultCollector(logger, mockConnection);
@@ -590,6 +599,7 @@ describe('TestAll', () => {
const runner = new MockTestRunner({
run: mockRunResult,
tests: mockTestResults,
+ numberOfResets: 0,
});
const testMethods = mockDefaultCollector(logger, mockConnection);
@@ -663,9 +673,11 @@ describe('TestAll', () => {
const runner = new MockTestRunner({
run: mockRunResult,
tests: [mockTestResults[0]],
+ numberOfResets: 0,
}).addNextResult({
run: mockRunResult,
tests: [mockTestResults[1]],
+ numberOfResets: 0,
});
const testMethods = new MockTestMethodCollector(
logger,
diff --git a/test/report/ClassTimeGenerator.spec.ts b/test/report/ClassTimeGenerator.spec.ts
index 95f7109..f71bb50 100644
--- a/test/report/ClassTimeGenerator.spec.ts
+++ b/test/report/ClassTimeGenerator.spec.ts
@@ -100,6 +100,7 @@ describe('ClassTimeGenerator', () => {
},
runIds: ['job Id'],
reruns: [],
+ numberOfResets: 0,
});
expect(logger.files.length).to.equal(2);
diff --git a/test/report/CoverageReporter.spec.ts b/test/report/CoverageReporter.spec.ts
index 9d4c1c6..08bab14 100644
--- a/test/report/CoverageReporter.spec.ts
+++ b/test/report/CoverageReporter.spec.ts
@@ -71,6 +71,7 @@ describe('CoverageReporter', () => {
},
],
},
+ numberOfResets: 0,
});
expect(executeStub.calledOnce).to.be.true;
@@ -99,6 +100,7 @@ describe('CoverageReporter', () => {
runIds: ['job Id'],
reruns: [],
coverageResult: undefined,
+ numberOfResets: 0,
});
expect(executeStub.called).to.be.false;
diff --git a/test/report/ExecutionMapGenerator.spec.ts b/test/report/ExecutionMapGenerator.spec.ts
index 37f0b73..b6c3a89 100644
--- a/test/report/ExecutionMapGenerator.spec.ts
+++ b/test/report/ExecutionMapGenerator.spec.ts
@@ -100,6 +100,7 @@ describe('ExecutionMapGenerator', () => {
},
runIds: ['job Id'],
reruns: [],
+ numberOfResets: 0,
});
expect(logger.files.length).to.equal(1);
diff --git a/test/report/ReportGenerator.spec.ts b/test/report/ReportGenerator.spec.ts
index 9e0f008..cbefe3f 100644
--- a/test/report/ReportGenerator.spec.ts
+++ b/test/report/ReportGenerator.spec.ts
@@ -100,6 +100,7 @@ describe('ReportGenerator', () => {
},
runIds: ['job Id'],
reruns: [],
+ numberOfResets: 0,
});
expect(logger.files.length).to.be.equal(2);
@@ -153,6 +154,7 @@ describe('ReportGenerator', () => {
},
runIds: ['job Id'],
reruns: [],
+ numberOfResets: 0,
});
expect(logger.files.length).to.be.equal(2);
@@ -206,6 +208,7 @@ describe('ReportGenerator', () => {
},
runIds: ['job Id'],
reruns: [],
+ numberOfResets: 0,
});
expect(logger.files.length).to.be.equal(2);
@@ -220,4 +223,87 @@ describe('ReportGenerator', () => {
);
expect(content).contains('');
});
+
+ it('should generate JSON and XML outputs with key fields validated', () => {
+ const generator = new ReportGenerator(
+ 'instanceUrl',
+ 'orgId',
+ 'username',
+ 'suitename'
+ );
+
+ const logger = new CapturingLogger();
+ generator.generate(logger, '', '/test-output', {
+ startTime: new Date('2020-07-10T15:00:00.000Z'),
+ testResults: [
+ {
+ Id: 'TestId1',
+ QueueItemId: 'QueueItemId1',
+ AsyncApexJobId: 'JobId1',
+ Outcome: 'Pass',
+ ApexClass: {
+ Id: 'ClassId1',
+ Name: 'ClassName1',
+ NamespacePrefix: null,
+ },
+ MethodName: 'MethodName1',
+ Message: null,
+ StackTrace: null,
+ RunTime: 10,
+ TestTimestamp: '2020-07-10T15:00:00.000Z',
+ },
+ ],
+ runResult: {
+ AsyncApexJobId: 'JobId1',
+ StartTime: '2020-07-10T15:00:00.000Z',
+ EndTime: '2020-07-10T15:01:00.000Z',
+ Status: 'Completed',
+ TestTime: 60,
+ UserId: 'UserId1',
+ ClassesCompleted: 1,
+ ClassesEnqueued: 1,
+ MethodsCompleted: 1,
+ MethodsEnqueued: 1,
+ MethodsFailed: 0,
+ },
+ runIds: ['JobId1'],
+ reruns: [],
+ numberOfResets: 7,
+ });
+
+ // Verify JSON output
+ const jsonOutput = logger.files.find(
+ (file): boolean => file[0] === '/test-output.json'
+ );
+ expect(jsonOutput).to.not.be.undefined;
+ if (jsonOutput) {
+ const jsonContent: {
+ summary: {
+ failing: number;
+ numberOfResets: number;
+ outcome: string;
+ testRunId: string;
+ testTotalTime: number;
+ testsRan: number;
+ };
+ } = JSON.parse(jsonOutput[1]);
+ expect(jsonContent.summary.failing).to.equal(0);
+ expect(jsonContent.summary.numberOfResets).to.equal(7);
+ expect(jsonContent.summary.outcome).to.equal('Passed');
+ expect(jsonContent.summary.testRunId).to.equal('JobId1');
+ expect(jsonContent.summary.testTotalTime).to.equal(60);
+ expect(jsonContent.summary.testsRan).to.equal(1);
+ }
+
+ // Verify XML output
+
+ const content1 = logger.files[0][1];
+ parseString(
+ content1,
+ (err, result: { testsuites: { testsuite: any }[] }) => {
+ expect(err).to.be.null;
+ expect(result).to.haveOwnProperty('testsuites');
+ }
+ );
+ });
});
diff --git a/test/report/RerunReportGenerator.spec.ts b/test/report/RerunReportGenerator.spec.ts
index 4727acc..d2b9748 100644
--- a/test/report/RerunReportGenerator.spec.ts
+++ b/test/report/RerunReportGenerator.spec.ts
@@ -62,6 +62,7 @@ describe('RerunReportGenerator', () => {
},
},
],
+ numberOfResets: 0,
});
expect(logger.files.length).to.be.equal(1);
diff --git a/test/runner/TestRunner.spec.ts b/test/runner/TestRunner.spec.ts
index d1c518a..0816e06 100644
--- a/test/runner/TestRunner.spec.ts
+++ b/test/runner/TestRunner.spec.ts
@@ -113,6 +113,7 @@ describe('TestRunner', () => {
});
expect(testRunResult.run.AsyncApexJobId).to.equal(testRunId);
expect(testRunResult.run.Status).to.equal('Completed');
+ expect(testRunResult.numberOfResets).to.equal(0);
expect(logger.entries.length).to.equal(4);
expect(logger.entries[0]).to.match(
logRegex(`Test run started with AsyncApexJob Id: ${testRunId}`)
@@ -146,6 +147,7 @@ describe('TestRunner', () => {
});
expect(testRunResult.run.AsyncApexJobId).to.equal(testRunId);
expect(testRunResult.run.Status).to.equal('Completed');
+ expect(testRunResult.numberOfResets).to.equal(0);
expect(logger.entries.length).to.equal(4);
expect(logger.entries[0]).to.match(
logRegex(`Test run started with AsyncApexJob Id: ${testRunId}`)
@@ -185,6 +187,7 @@ describe('TestRunner', () => {
});
expect(testRunResult.run.AsyncApexJobId).to.equal(testRunId);
expect(testRunResult.run.Status).to.equal('Failed');
+ expect(testRunResult.numberOfResets).to.equal(0);
expect(logger.entries.length).to.equal(4);
expect(logger.entries[0]).to.match(
logRegex(`Test run started with AsyncApexJob Id: ${testRunId}`)
@@ -222,6 +225,7 @@ describe('TestRunner', () => {
});
expect(testRunResult.run.AsyncApexJobId).to.equal(testRunId);
expect(testRunResult.run.Status).to.equal('Aborted');
+ expect(testRunResult.numberOfResets).to.equal(0);
expect(logger.entries.length).to.equal(2);
expect(logger.entries[0]).to.match(
logRegex(`Test run started with AsyncApexJob Id: ${testRunId}`)
@@ -326,6 +330,7 @@ describe('TestRunner', () => {
expect(testServiceAsyncStub.calledOnce).to.be.true;
expect(testRunResult.run.AsyncApexJobId).to.equal(testRunId);
expect(testRunResult.run.Status).to.equal('Completed');
+ expect(testRunResult.numberOfResets).to.equal(0);
expect(logger.entries.length).to.equal(6);
expect(logger.entries[0]).to.match(
logRegex(`Test run started with AsyncApexJob Id: ${testRunId}`)
@@ -408,7 +413,7 @@ describe('TestRunner', () => {
['TestSample'],
{
maxTestRunRetries: 2,
- pollLimitToAssumeHangingTests: 1, // Will asumme hanging on each poll
+ pollLimitToAssumeHangingTests: 1, // Will assume hanging on each poll
aborter: mockAborter, // Skip over aborting
}
);
@@ -419,6 +424,7 @@ describe('TestRunner', () => {
expect(testServiceAsyncStub.calledTwice).to.be.true;
expect(testRunResult.run.AsyncApexJobId).to.equal(testRunId);
expect(testRunResult.run.Status).to.equal('Completed');
+ expect(testRunResult.numberOfResets).to.equal(1);
expect(logger.entries.length).to.equal(10);
expect(logger.entries[0]).to.match(
logRegex(`Test run started with AsyncApexJob Id: ${testRunId}`)
@@ -517,6 +523,7 @@ describe('TestRunner', () => {
expect(testServiceAsyncStub.calledOnce).to.be.true;
expect(testRunResult.run.AsyncApexJobId).to.equal(testRunId);
expect(testRunResult.run.Status).to.equal('Completed');
+ expect(testRunResult.numberOfResets).to.equal(0);
expect(logger.entries.length).to.equal(7);
expect(logger.entries[0]).to.match(
logRegex(`Test run started with AsyncApexJob Id: ${testRunId}`)
@@ -623,6 +630,7 @@ describe('TestRunner', () => {
expect(testServiceAsyncStub.calledOnce).to.be.true;
expect(testRunResult.run.AsyncApexJobId).to.equal(testRunId);
expect(testRunResult.run.Status).to.equal('Completed');
+ expect(testRunResult.numberOfResets).to.equal(0);
expect(logger.files.length).to.equal(1);
expect(logger.files[0][0]).to.match(
new RegExp(`^${process.cwd()}/testqueue-${isoDateFormat}.json`)
@@ -669,6 +677,7 @@ describe('TestRunner', () => {
});
expect(testRunResult.run.AsyncApexJobId).to.equal(testRunId);
expect(testRunResult.run.Status).to.equal('Completed');
+ expect(testRunResult.numberOfResets).to.equal(0);
expect(logger.entries.length).to.equal(6);
expect(logger.entries[0]).to.match(
logRegex(`Test run started with AsyncApexJob Id: ${testRunId}`)