diff --git a/CHANGELOG.md b/CHANGELOG.md index 83057f2..6d23400 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # test-runner - Changelog +## 3.3.0 - 2026-02-25 + +* Add `numberOfResets` to `TestRunnerResult`, tracking how many times a test run was cancelled and restarted due to lack of progress. + * Included in JSON output and JUnit XML report properties. + ## 3.2.0 - 2024-12-10 * Add `QueryHelper.setGlobalRetryOptions()` static method. diff --git a/package.json b/package.json index 9a6b1af..baa4da1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@apexdevtools/test-runner", - "version": "3.2.0", + "version": "3.3.0", "description": "Apex parallel test runner with reliability goodness", "author": { "name": "Apex Dev Tools Team", diff --git a/src/command/Testall.ts b/src/command/Testall.ts index c4d07dc..7ead67c 100644 --- a/src/command/Testall.ts +++ b/src/command/Testall.ts @@ -214,6 +214,8 @@ export class Testall { Promise.resolve(missingTests), store ); + + store.numberOfResets = result.numberOfResets; } } diff --git a/src/results/OutputGenerator.ts b/src/results/OutputGenerator.ts index a241e4d..c2339d3 100644 --- a/src/results/OutputGenerator.ts +++ b/src/results/OutputGenerator.ts @@ -19,6 +19,7 @@ export interface TestRunSummary { runIds: string[]; reruns: TestRerun[]; coverageResult?: CoverageReport; + numberOfResets: number; } export interface OutputGenerator { diff --git a/src/results/ReportGenerator.ts b/src/results/ReportGenerator.ts index 7892b19..702bd2f 100644 --- a/src/results/ReportGenerator.ts +++ b/src/results/ReportGenerator.ts @@ -41,9 +41,16 @@ export class ReportGenerator implements OutputGenerator { fileName: string, runSummary: TestRunSummary ): void { - const { startTime, testResults, runResult, reruns } = runSummary; + const { startTime, testResults, runResult, reruns, numberOfResets } = + runSummary; const results = testResults as ExtendedApexTestResult[]; - const summary = this.summary(startTime, results, runResult, reruns); + const summary = this.summary( + startTime, + results, + runResult, + reruns, + numberOfResets + ); logger.logOutputFile( path.join(outputDirBase, fileName + '.xml'), this.generateJunit(summary, results) @@ -58,7 +65,8 @@ export class ReportGenerator implements OutputGenerator { startTime: Date, testResults: ExtendedApexTestResult[], runResults: ApexTestRunResult, - reruns: TestRerun[] + reruns: TestRerun[], + numberOfResets: number ): SummaryData { // combine test and method names for fullname testResults.forEach(test => { @@ -130,6 +138,7 @@ export class ReportGenerator implements OutputGenerator { username: this.username, testRunId: runResults.AsyncApexJobId, userId: runResults.UserId, + numberOfResets, }; } @@ -180,6 +189,7 @@ export class ReportGenerator implements OutputGenerator { junit += ` \n`; junit += ` \n`; junit += ` \n`; + junit += ` \n`; junit += ' \n'; testResults.forEach(test => { const success = test.Outcome === 'Pass'; @@ -229,7 +239,8 @@ export class ReportGenerator implements OutputGenerator { json += ` "orgId": "${summary.orgId}",\n`; json += ` "username": "${summary.username}",\n`; json += ` "testRunId": "${summary.testRunId}",\n`; - json += ` "userId": "${summary.userId}"\n`; + json += ` "userId": "${summary.userId}",\n`; + json += ` "numberOfResets": ${summary.numberOfResets}\n`; json += ' },\n'; json += ' "tests": [\n'; @@ -298,4 +309,5 @@ interface SummaryData { username: string; testRunId: string; userId: string; + numberOfResets: number; } diff --git a/src/results/TestResultStore.ts b/src/results/TestResultStore.ts index 22671c4..52b923b 100644 --- a/src/results/TestResultStore.ts +++ b/src/results/TestResultStore.ts @@ -21,6 +21,7 @@ export class TestResultStore { asyncError?: TestError; reruns: TestRerun[]; coverage?: CoverageReport; + numberOfResets: number; get resultsArray() { return Array.from(this.tests.values()); @@ -31,6 +32,7 @@ export class TestResultStore { this.runIds = []; this.tests = new Map(); this.reruns = []; + this.numberOfResets = 0; } public saveAsyncResult(res: TestRunnerResult): void { @@ -43,6 +45,7 @@ export class TestResultStore { }); this.asyncError = res.error; + this.numberOfResets = res.numberOfResets; } public saveSyncResult(reruns: TestRerun[]): void { @@ -88,6 +91,7 @@ export class TestResultStore { runIds: this.runIds, reruns: this.reruns, coverageResult: this.coverage, + numberOfResets: this.numberOfResets, }; } diff --git a/src/runner/TestRunner.ts b/src/runner/TestRunner.ts index 317c4ae..50b88c7 100644 --- a/src/runner/TestRunner.ts +++ b/src/runner/TestRunner.ts @@ -44,6 +44,7 @@ export interface TestRunnerResult { run: ApexTestRunResult; tests: ApexTestResult[]; error?: TestError; + numberOfResets: number; // Track the number of times the test run has been reset due to hanging or cancellation } export interface TestRunner { @@ -66,7 +67,7 @@ export class AsyncTestRunner implements TestRunner { private readonly _testItems: TestItem[]; private readonly _options: TestRunnerOptions; private readonly _testService: TestService; - private _stats; + private _stats: TestStats; static forClasses( logger: Logger, @@ -141,6 +142,9 @@ export class AsyncTestRunner implements TestRunner { token ); + // Add numberOfResets to the result object + result.numberOfResets = this._stats.getNumberOfTimesReset(); + // Ensure result for partial reporting try { if (token?.isCancellationRequested) { @@ -220,6 +224,7 @@ export class AsyncTestRunner implements TestRunner { return (lastResult = { run, tests, + numberOfResets: this._stats.getNumberOfTimesReset(), }); }, diff --git a/test/command/TestDebugLogs.spec.ts b/test/command/TestDebugLogs.spec.ts index f79d088..048e4cc 100644 --- a/test/command/TestDebugLogs.spec.ts +++ b/test/command/TestDebugLogs.spec.ts @@ -74,6 +74,7 @@ describe('TestDebugLogs', () => { const runner = new MockTestRunner({ run: runnerResult, tests: mockTestResults, + numberOfResets: 0, }); const methodCollector = mockDefaultCollector(logger, mockConnection); qhStub.query.onCall(0).resolves([]); @@ -110,6 +111,7 @@ describe('TestDebugLogs', () => { const runner = new MockTestRunner({ run: runnerResult, tests: mockTestResults, + numberOfResets: 0, }); const methodCollector = mockDefaultCollector(logger, mockConnection); qhStub.query.onCall(0).resolves([{ Id: 'AnId' }]); // User Id @@ -154,6 +156,7 @@ describe('TestDebugLogs', () => { const runner = new MockTestRunner({ run: runnerResult, tests: mockTestResults, + numberOfResets: 0, }); const methodCollector = mockDefaultCollector(logger, mockConnection); qhStub.query.onCall(0).resolves([{ Id: 'AnId' }]); // User Id @@ -198,6 +201,7 @@ describe('TestDebugLogs', () => { const runner = new MockTestRunner({ run: runnerResult, tests: mockTestResults, + numberOfResets: 0, }); const { classId, className, methodName } = defaultTestInfo; const methodCollector = new MockTestMethodCollector( diff --git a/test/command/Testall.spec.ts b/test/command/Testall.spec.ts index 5b86179..a20023d 100644 --- a/test/command/Testall.spec.ts +++ b/test/command/Testall.spec.ts @@ -176,6 +176,7 @@ describe('TestAll', () => { const runner = new MockTestRunner({ run: mockRunResult, tests: mockTestResults, + numberOfResets: 1, }); const testMethods = mockDefaultCollector(logger, mockConnection); @@ -197,6 +198,7 @@ describe('TestAll', () => { reruns: [], runIds: [testRunId], coverageResult: undefined, + numberOfResets: 1, }); }); @@ -208,6 +210,7 @@ describe('TestAll', () => { const runner = new MockTestRunner({ run: mockRunResult, tests: [], + numberOfResets: 0, }); const testMethods = mockDefaultCollector(logger, mockConnection); @@ -247,6 +250,7 @@ describe('TestAll', () => { const runner = new MockTestRunner({ run: mockRunResult, tests: mockTestResults, + numberOfResets: 0, }); const testMethods = mockDefaultCollector(logger, mockConnection); @@ -294,6 +298,7 @@ describe('TestAll', () => { const runner = new MockTestRunner({ run: mockRunResult, tests: mockTestResults, + numberOfResets: 0, }); const testMethods = mockDefaultCollector(logger, mockConnection); const mockTestResult = { @@ -343,6 +348,7 @@ describe('TestAll', () => { const runner = new MockTestRunner({ run: mockRunResult, tests: mockTestResults, + numberOfResets: 0, }); const testMethods = mockDefaultCollector(logger, mockConnection); const mockTestResult = { @@ -398,6 +404,7 @@ describe('TestAll', () => { const runner = new MockTestRunner({ run: mockRunResult, tests: mockTestResults, + numberOfResets: 0, }); const testMethods = mockDefaultCollector(logger, mockConnection); testingServiceSyncStub.rejects(new Error('Request Error')); @@ -465,6 +472,7 @@ describe('TestAll', () => { const runner = new MockTestRunner({ run: mockRunResult, tests: mockTestResults, + numberOfResets: 0, }); const testMethods = mockDefaultCollector(logger, mockConnection); const mockTestResult = { @@ -524,6 +532,7 @@ describe('TestAll', () => { const runner = new MockTestRunner({ run: mockRunResult, tests: mockTestResults, + numberOfResets: 0, }); const testMethods = mockDefaultCollector(logger, mockConnection); @@ -590,6 +599,7 @@ describe('TestAll', () => { const runner = new MockTestRunner({ run: mockRunResult, tests: mockTestResults, + numberOfResets: 0, }); const testMethods = mockDefaultCollector(logger, mockConnection); @@ -663,9 +673,11 @@ describe('TestAll', () => { const runner = new MockTestRunner({ run: mockRunResult, tests: [mockTestResults[0]], + numberOfResets: 0, }).addNextResult({ run: mockRunResult, tests: [mockTestResults[1]], + numberOfResets: 0, }); const testMethods = new MockTestMethodCollector( logger, diff --git a/test/report/ClassTimeGenerator.spec.ts b/test/report/ClassTimeGenerator.spec.ts index 95f7109..f71bb50 100644 --- a/test/report/ClassTimeGenerator.spec.ts +++ b/test/report/ClassTimeGenerator.spec.ts @@ -100,6 +100,7 @@ describe('ClassTimeGenerator', () => { }, runIds: ['job Id'], reruns: [], + numberOfResets: 0, }); expect(logger.files.length).to.equal(2); diff --git a/test/report/CoverageReporter.spec.ts b/test/report/CoverageReporter.spec.ts index 9d4c1c6..08bab14 100644 --- a/test/report/CoverageReporter.spec.ts +++ b/test/report/CoverageReporter.spec.ts @@ -71,6 +71,7 @@ describe('CoverageReporter', () => { }, ], }, + numberOfResets: 0, }); expect(executeStub.calledOnce).to.be.true; @@ -99,6 +100,7 @@ describe('CoverageReporter', () => { runIds: ['job Id'], reruns: [], coverageResult: undefined, + numberOfResets: 0, }); expect(executeStub.called).to.be.false; diff --git a/test/report/ExecutionMapGenerator.spec.ts b/test/report/ExecutionMapGenerator.spec.ts index 37f0b73..b6c3a89 100644 --- a/test/report/ExecutionMapGenerator.spec.ts +++ b/test/report/ExecutionMapGenerator.spec.ts @@ -100,6 +100,7 @@ describe('ExecutionMapGenerator', () => { }, runIds: ['job Id'], reruns: [], + numberOfResets: 0, }); expect(logger.files.length).to.equal(1); diff --git a/test/report/ReportGenerator.spec.ts b/test/report/ReportGenerator.spec.ts index 9e0f008..cbefe3f 100644 --- a/test/report/ReportGenerator.spec.ts +++ b/test/report/ReportGenerator.spec.ts @@ -100,6 +100,7 @@ describe('ReportGenerator', () => { }, runIds: ['job Id'], reruns: [], + numberOfResets: 0, }); expect(logger.files.length).to.be.equal(2); @@ -153,6 +154,7 @@ describe('ReportGenerator', () => { }, runIds: ['job Id'], reruns: [], + numberOfResets: 0, }); expect(logger.files.length).to.be.equal(2); @@ -206,6 +208,7 @@ describe('ReportGenerator', () => { }, runIds: ['job Id'], reruns: [], + numberOfResets: 0, }); expect(logger.files.length).to.be.equal(2); @@ -220,4 +223,87 @@ describe('ReportGenerator', () => { ); expect(content).contains(''); }); + + it('should generate JSON and XML outputs with key fields validated', () => { + const generator = new ReportGenerator( + 'instanceUrl', + 'orgId', + 'username', + 'suitename' + ); + + const logger = new CapturingLogger(); + generator.generate(logger, '', '/test-output', { + startTime: new Date('2020-07-10T15:00:00.000Z'), + testResults: [ + { + Id: 'TestId1', + QueueItemId: 'QueueItemId1', + AsyncApexJobId: 'JobId1', + Outcome: 'Pass', + ApexClass: { + Id: 'ClassId1', + Name: 'ClassName1', + NamespacePrefix: null, + }, + MethodName: 'MethodName1', + Message: null, + StackTrace: null, + RunTime: 10, + TestTimestamp: '2020-07-10T15:00:00.000Z', + }, + ], + runResult: { + AsyncApexJobId: 'JobId1', + StartTime: '2020-07-10T15:00:00.000Z', + EndTime: '2020-07-10T15:01:00.000Z', + Status: 'Completed', + TestTime: 60, + UserId: 'UserId1', + ClassesCompleted: 1, + ClassesEnqueued: 1, + MethodsCompleted: 1, + MethodsEnqueued: 1, + MethodsFailed: 0, + }, + runIds: ['JobId1'], + reruns: [], + numberOfResets: 7, + }); + + // Verify JSON output + const jsonOutput = logger.files.find( + (file): boolean => file[0] === '/test-output.json' + ); + expect(jsonOutput).to.not.be.undefined; + if (jsonOutput) { + const jsonContent: { + summary: { + failing: number; + numberOfResets: number; + outcome: string; + testRunId: string; + testTotalTime: number; + testsRan: number; + }; + } = JSON.parse(jsonOutput[1]); + expect(jsonContent.summary.failing).to.equal(0); + expect(jsonContent.summary.numberOfResets).to.equal(7); + expect(jsonContent.summary.outcome).to.equal('Passed'); + expect(jsonContent.summary.testRunId).to.equal('JobId1'); + expect(jsonContent.summary.testTotalTime).to.equal(60); + expect(jsonContent.summary.testsRan).to.equal(1); + } + + // Verify XML output + + const content1 = logger.files[0][1]; + parseString( + content1, + (err, result: { testsuites: { testsuite: any }[] }) => { + expect(err).to.be.null; + expect(result).to.haveOwnProperty('testsuites'); + } + ); + }); }); diff --git a/test/report/RerunReportGenerator.spec.ts b/test/report/RerunReportGenerator.spec.ts index 4727acc..d2b9748 100644 --- a/test/report/RerunReportGenerator.spec.ts +++ b/test/report/RerunReportGenerator.spec.ts @@ -62,6 +62,7 @@ describe('RerunReportGenerator', () => { }, }, ], + numberOfResets: 0, }); expect(logger.files.length).to.be.equal(1); diff --git a/test/runner/TestRunner.spec.ts b/test/runner/TestRunner.spec.ts index d1c518a..0816e06 100644 --- a/test/runner/TestRunner.spec.ts +++ b/test/runner/TestRunner.spec.ts @@ -113,6 +113,7 @@ describe('TestRunner', () => { }); expect(testRunResult.run.AsyncApexJobId).to.equal(testRunId); expect(testRunResult.run.Status).to.equal('Completed'); + expect(testRunResult.numberOfResets).to.equal(0); expect(logger.entries.length).to.equal(4); expect(logger.entries[0]).to.match( logRegex(`Test run started with AsyncApexJob Id: ${testRunId}`) @@ -146,6 +147,7 @@ describe('TestRunner', () => { }); expect(testRunResult.run.AsyncApexJobId).to.equal(testRunId); expect(testRunResult.run.Status).to.equal('Completed'); + expect(testRunResult.numberOfResets).to.equal(0); expect(logger.entries.length).to.equal(4); expect(logger.entries[0]).to.match( logRegex(`Test run started with AsyncApexJob Id: ${testRunId}`) @@ -185,6 +187,7 @@ describe('TestRunner', () => { }); expect(testRunResult.run.AsyncApexJobId).to.equal(testRunId); expect(testRunResult.run.Status).to.equal('Failed'); + expect(testRunResult.numberOfResets).to.equal(0); expect(logger.entries.length).to.equal(4); expect(logger.entries[0]).to.match( logRegex(`Test run started with AsyncApexJob Id: ${testRunId}`) @@ -222,6 +225,7 @@ describe('TestRunner', () => { }); expect(testRunResult.run.AsyncApexJobId).to.equal(testRunId); expect(testRunResult.run.Status).to.equal('Aborted'); + expect(testRunResult.numberOfResets).to.equal(0); expect(logger.entries.length).to.equal(2); expect(logger.entries[0]).to.match( logRegex(`Test run started with AsyncApexJob Id: ${testRunId}`) @@ -326,6 +330,7 @@ describe('TestRunner', () => { expect(testServiceAsyncStub.calledOnce).to.be.true; expect(testRunResult.run.AsyncApexJobId).to.equal(testRunId); expect(testRunResult.run.Status).to.equal('Completed'); + expect(testRunResult.numberOfResets).to.equal(0); expect(logger.entries.length).to.equal(6); expect(logger.entries[0]).to.match( logRegex(`Test run started with AsyncApexJob Id: ${testRunId}`) @@ -408,7 +413,7 @@ describe('TestRunner', () => { ['TestSample'], { maxTestRunRetries: 2, - pollLimitToAssumeHangingTests: 1, // Will asumme hanging on each poll + pollLimitToAssumeHangingTests: 1, // Will assume hanging on each poll aborter: mockAborter, // Skip over aborting } ); @@ -419,6 +424,7 @@ describe('TestRunner', () => { expect(testServiceAsyncStub.calledTwice).to.be.true; expect(testRunResult.run.AsyncApexJobId).to.equal(testRunId); expect(testRunResult.run.Status).to.equal('Completed'); + expect(testRunResult.numberOfResets).to.equal(1); expect(logger.entries.length).to.equal(10); expect(logger.entries[0]).to.match( logRegex(`Test run started with AsyncApexJob Id: ${testRunId}`) @@ -517,6 +523,7 @@ describe('TestRunner', () => { expect(testServiceAsyncStub.calledOnce).to.be.true; expect(testRunResult.run.AsyncApexJobId).to.equal(testRunId); expect(testRunResult.run.Status).to.equal('Completed'); + expect(testRunResult.numberOfResets).to.equal(0); expect(logger.entries.length).to.equal(7); expect(logger.entries[0]).to.match( logRegex(`Test run started with AsyncApexJob Id: ${testRunId}`) @@ -623,6 +630,7 @@ describe('TestRunner', () => { expect(testServiceAsyncStub.calledOnce).to.be.true; expect(testRunResult.run.AsyncApexJobId).to.equal(testRunId); expect(testRunResult.run.Status).to.equal('Completed'); + expect(testRunResult.numberOfResets).to.equal(0); expect(logger.files.length).to.equal(1); expect(logger.files[0][0]).to.match( new RegExp(`^${process.cwd()}/testqueue-${isoDateFormat}.json`) @@ -669,6 +677,7 @@ describe('TestRunner', () => { }); expect(testRunResult.run.AsyncApexJobId).to.equal(testRunId); expect(testRunResult.run.Status).to.equal('Completed'); + expect(testRunResult.numberOfResets).to.equal(0); expect(logger.entries.length).to.equal(6); expect(logger.entries[0]).to.match( logRegex(`Test run started with AsyncApexJob Id: ${testRunId}`)