Create final status tag on test events

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java

@@ -250,29 +250,36 @@ public void onTestFinish(
       return;
     }
 
+    TestStatus testStatus = test.getStatus() != null ? test.getStatus() : TestStatus.skip;
+
     if (testExecutionHistory != null) {
-      TestStatus testStatus = test.getStatus();
       TestExecutionHistory.ExecutionOutcome outcome =
-          testExecutionHistory.registerExecution(
-              testStatus != null ? testStatus : TestStatus.skip, test.getDuration(endTime));
+          testExecutionHistory.registerExecution(testStatus, test.getDuration(endTime));
 
       RetryReason retryReason = outcome.retryReason();
       if (retryReason != null) {
         test.setTag(Tags.TEST_IS_RETRY, true);
         test.setTag(Tags.TEST_RETRY_REASON, retryReason);
       }
 
-      if (outcome.failedAllRetries()) {
-        test.setTag(Tags.TEST_HAS_FAILED_ALL_RETRIES, true);
+      if (outcome.failureSuppressed()) {
+        test.setTag(Tags.TEST_FAILURE_SUPPRESSED, true);
       }
 
-      if (outcome.lastExecution() && testModule.isAttemptToFix(test.getIdentifier())) {
-        test.setTag(Tags.TEST_TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, outcome.succeededAllRetries());
-      }
+      if (outcome.lastExecution()) {
+        test.setTag(Tags.TEST_FINAL_STATUS, outcome.finalStatus());
 
-      if (outcome.failureSuppressed()) {
-        test.setTag(Tags.TEST_FAILURE_SUPPRESSED, true);
+        if (outcome.failedAllRetries()) {
+          test.setTag(Tags.TEST_HAS_FAILED_ALL_RETRIES, true);
+        }
+
+        if (testModule.isAttemptToFix(test.getIdentifier())) {
+          test.setTag(
+              Tags.TEST_TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, outcome.succeededAllRetries());
+        }
       }
+    } else {
+      test.setTag(Tags.TEST_FINAL_STATUS, testStatus);
     }
 
     test.end(endTime);

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/ExecutionOutcomeImpl.java

@@ -1,6 +1,7 @@
 package datadog.trace.civisibility.execution;
 
 import datadog.trace.api.civisibility.execution.TestExecutionHistory;
+import datadog.trace.api.civisibility.execution.TestStatus;
 import datadog.trace.api.civisibility.telemetry.tag.RetryReason;
 import javax.annotation.Nullable;
 
@@ -11,18 +12,21 @@ class ExecutionOutcomeImpl implements TestExecutionHistory.ExecutionOutcome {
   private final boolean failedAllRetries;
   private final boolean succeededAllRetries;
   private final RetryReason retryReason;
+  private final TestStatus finalStatus;
 
   ExecutionOutcomeImpl(
       boolean failureSuppressed,
       boolean lastExecution,
       boolean failedAllRetries,
       boolean succeededAllRetries,
-      RetryReason retryReason) {
+      RetryReason retryReason,
+      TestStatus finalStatus) {
     this.failureSuppressed = failureSuppressed;
     this.lastExecution = lastExecution;
     this.failedAllRetries = failedAllRetries;
     this.succeededAllRetries = succeededAllRetries;
     this.retryReason = retryReason;
+    this.finalStatus = finalStatus;
   }
 
   @Override
@@ -50,4 +54,10 @@ public boolean succeededAllRetries() {
   public RetryReason retryReason() {
     return retryReason;
   }
+
+  @Nullable
+  @Override
+  public TestStatus finalStatus() {
+    return finalStatus;
+  }
 }

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/Regular.java

@@ -1,10 +1,7 @@
 package datadog.trace.civisibility.execution;
 
-import datadog.trace.api.civisibility.execution.TestExecutionHistory;
 import datadog.trace.api.civisibility.execution.TestExecutionPolicy;
 import datadog.trace.api.civisibility.execution.TestStatus;
-import datadog.trace.api.civisibility.telemetry.tag.RetryReason;
-import javax.annotation.Nullable;
 
 /** Regular test case execution with no alterations. */
 public class Regular implements TestExecutionPolicy {
@@ -15,7 +12,7 @@ private Regular() {}
 
   @Override
   public ExecutionOutcome registerExecution(TestStatus status, long durationMillis) {
-    return RegularExecutionOutcome.INSTANCE;
+    return new ExecutionOutcomeImpl(false, true, false, false, null, status);
   }
 
   @Override
@@ -28,40 +25,6 @@ public boolean suppressFailures() {
     return false;
   }
 
-  private static final class RegularExecutionOutcome
-      implements TestExecutionHistory.ExecutionOutcome {
-
-    static final TestExecutionHistory.ExecutionOutcome INSTANCE = new RegularExecutionOutcome();
-
-    private RegularExecutionOutcome() {}
-
-    @Override
-    public boolean failureSuppressed() {
-      return false;
-    }
-
-    @Override
-    public boolean lastExecution() {
-      return false;
-    }
-
-    @Override
-    public boolean failedAllRetries() {
-      return false;
-    }
-
-    @Override
-    public boolean succeededAllRetries() {
-      return false;
-    }
-
-    @Nullable
-    @Override
-    public RetryReason retryReason() {
-      return null;
-    }
-  }
-
   @Override
   public boolean failedTestReplayApplicable() {
     return false;

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/RetryUntilSuccessful.java

@@ -39,12 +39,20 @@ public ExecutionOutcome registerExecution(TestStatus status, long durationMillis
 
     boolean lastExecution = !retriesLeft();
     boolean retry = executions > 1; // first execution is not a retry
+    boolean failureSuppressed = status == TestStatus.fail && (!lastExecution || suppressFailures);
+    TestStatus finalStatus = null;
+    if (lastExecution) {
+      // final status is always the last status reported (or pass if a failure is suppressed)
+      finalStatus = failureSuppressed ? TestStatus.pass : status;
+    }
+
     return new ExecutionOutcomeImpl(
-        status == TestStatus.fail && (!lastExecution || suppressFailures),
+        failureSuppressed,
         lastExecution,
         lastExecution && !successfulExecutionSeen,
         false,
-        retry ? RetryReason.atr : null);
+        retry ? RetryReason.atr : null,
+        finalStatus);
   }
 
   private boolean retriesLeft() {

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/RunNTimes.java

@@ -41,12 +41,27 @@ public ExecutionOutcome registerExecution(TestStatus status, long durationMillis
 
     boolean lastExecution = !retriesLeft();
     boolean retry = executions > 1; // first execution is not a retry
+    boolean failureSuppressed = status == TestStatus.fail && suppressFailures();
+    boolean succeededAllRetries = lastExecution && successfulExecutionsSeen == executions;
+
+    TestStatus finalStatus = null;
+    if (lastExecution) {
+      // final status will only be "pass" if all retries pass (or the failures were suppressed)
+      // also, the `suppressFailures()` call works because its value cannot change between retries
+      if (succeededAllRetries || suppressFailures()) {
+        finalStatus = TestStatus.pass;
+      } else {
+        finalStatus = TestStatus.fail;
+      }
+    }
+
     return new ExecutionOutcomeImpl(
-        status == TestStatus.fail && suppressFailures(),
+        failureSuppressed,
         lastExecution,
         lastExecution && successfulExecutionsSeen == 0,
-        lastExecution && successfulExecutionsSeen == executions,
-        retry ? retryReason : null);
+        succeededAllRetries,
+        retry ? retryReason : null,
+        finalStatus);
   }
 
   private boolean retriesLeft() {

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/RunOnceIgnoreOutcome.java

@@ -14,7 +14,13 @@ public class RunOnceIgnoreOutcome implements TestExecutionPolicy {
   @Override
   public ExecutionOutcome registerExecution(TestStatus status, long durationMillis) {
     testExecuted = true;
-    return new ExecutionOutcomeImpl(status == TestStatus.fail, testExecuted, false, false, null);
+    return new ExecutionOutcomeImpl(
+        status == TestStatus.fail,
+        testExecuted,
+        false,
+        false,
+        null,
+        status == TestStatus.fail ? TestStatus.pass : status);
   }
 
   @Override

dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/execution/RegularExecutionTest.groovy

@@ -10,13 +10,17 @@ class RegularExecutionTest extends Specification {
     def executionPolicy = Regular.INSTANCE
 
     when:
-    def outcome = executionPolicy.registerExecution(TestStatus.pass, 0)
+    def outcome = executionPolicy.registerExecution(status, 0)
 
     then:
     outcome.retryReason() == null
-    !outcome.lastExecution()
+    outcome.lastExecution()
     !outcome.failureSuppressed()
     !outcome.failedAllRetries()
     !outcome.succeededAllRetries()
+    outcome.finalStatus() == status
+
+    where:
+    status << [TestStatus.pass, TestStatus.fail, TestStatus.skip]
   }
 }

dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/execution/RetryUntilSuccessfulTest.groovy

@@ -21,6 +21,7 @@ class RetryUntilSuccessfulTest extends Specification {
     outcome.failureSuppressed()
     !outcome.failedAllRetries()
     !outcome.succeededAllRetries()
+    outcome.finalStatus() == null
 
     when:
     def outcome2 = executionPolicy.registerExecution(TestStatus.pass, 0)
@@ -31,6 +32,7 @@ class RetryUntilSuccessfulTest extends Specification {
     !outcome2.failureSuppressed()
     !outcome2.failedAllRetries()
     !outcome2.succeededAllRetries()
+    outcome2.finalStatus() == TestStatus.pass
   }
 
   def "test fail all retries"() {
@@ -46,6 +48,7 @@ class RetryUntilSuccessfulTest extends Specification {
     outcome.failureSuppressed()
     !outcome.failedAllRetries()
     !outcome.succeededAllRetries()
+    outcome.finalStatus() == null
 
     when:
     def outcome2 = executionPolicy.registerExecution(TestStatus.fail, 0)
@@ -56,6 +59,7 @@ class RetryUntilSuccessfulTest extends Specification {
     outcome2.failureSuppressed()
     !outcome2.failedAllRetries()
     !outcome2.succeededAllRetries()
+    outcome2.finalStatus() == null
 
     when:
     def outcome3 = executionPolicy.registerExecution(TestStatus.fail, 0)
@@ -66,6 +70,7 @@ class RetryUntilSuccessfulTest extends Specification {
     !outcome3.failureSuppressed()
     outcome3.failedAllRetries()
     !outcome2.succeededAllRetries()
+    outcome3.finalStatus() == TestStatus.fail
   }
 
   def "test succeed on last try"() {
@@ -81,6 +86,7 @@ class RetryUntilSuccessfulTest extends Specification {
     outcome.failureSuppressed()
     !outcome.failedAllRetries()
     !outcome.succeededAllRetries()
+    outcome.finalStatus() == null
 
     when:
     def outcome2 = executionPolicy.registerExecution(TestStatus.fail, 0)
@@ -91,6 +97,7 @@ class RetryUntilSuccessfulTest extends Specification {
     outcome2.failureSuppressed()
     !outcome2.failedAllRetries()
     !outcome2.succeededAllRetries()
+    outcome2.finalStatus() == null
 
     when:
     def outcome3 = executionPolicy.registerExecution(TestStatus.pass, 0)
@@ -101,6 +108,7 @@ class RetryUntilSuccessfulTest extends Specification {
     !outcome3.failureSuppressed()
     !outcome3.failedAllRetries()
     !outcome2.succeededAllRetries()
+    outcome3.finalStatus() == TestStatus.pass
   }
 
   def "test succeed on first try"() {
@@ -116,6 +124,7 @@ class RetryUntilSuccessfulTest extends Specification {
     !outcome.failureSuppressed()
     !outcome.failedAllRetries()
     !outcome.succeededAllRetries()
+    outcome.finalStatus() == TestStatus.pass
   }
 
   def "test suppress failures"() {
@@ -131,6 +140,7 @@ class RetryUntilSuccessfulTest extends Specification {
     outcome.failureSuppressed()
     !outcome.failedAllRetries()
     !outcome.succeededAllRetries()
+    outcome.finalStatus() == null
 
     when:
     def outcome2 = executionPolicy.registerExecution(TestStatus.fail, 0)
@@ -141,6 +151,7 @@ class RetryUntilSuccessfulTest extends Specification {
     outcome2.failureSuppressed()
     !outcome2.failedAllRetries()
     !outcome2.succeededAllRetries()
+    outcome2.finalStatus() == null
 
     when:
     def outcome3 = executionPolicy.registerExecution(TestStatus.fail, 0)
@@ -151,5 +162,6 @@ class RetryUntilSuccessfulTest extends Specification {
     outcome3.failureSuppressed()
     outcome3.failedAllRetries()
     !outcome2.succeededAllRetries()
+    outcome3.finalStatus() == TestStatus.pass
   }
 }