Include severity and isFailure for Issues in the published JSON event stream (#1279)

### Motivation:

- Proposed and accepted in ST-0013 [^1], so we can move ahead and
include this in the event stream.

- This will _not_ be available in event stream version 0 as this is
considered a breaking change [^2]

### Modifications:

- Promote existing _severity/_isFailure fields -> severity/isFailure
without the underscores

This effectively makes them part of the official JSON ABI. The
underscored names will no longer be accessible, but they were always
experimental so this is not considered a blocker.

- Only emit severity/isFailure when using event stream version >= 6.3

This removes access to the underscored names from earlier versions, but
again, these were experimental.

- Remove hidden `__CommandLineArguments_v0` argument
`isWarningIssueRecordedEventEnabled`

This was only for internal testing use, but is obsolete because we test
issue severity by specifying event stream version >= 6.3.

### Checklist:

- [x] Code and documentation should follow the style of the [Style
Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md).
- [x] If public symbols are renamed or modified, DocC references should
be updated.

[^1]:
https://github.com/swiftlang/swift-evolution/blob/main/proposals/testing/0013-issue-severity-warning.md
[^2]:
https://github.com/swiftlang/swift-evolution/blob/main/proposals/testing/0013-issue-severity-warning.md#integration-with-supporting-tools


Fixes rdar://158621033

Author: jerryjrchen

Sources/Testing/ABI/Encoded/ABI.EncodedIssue.swift

@@ -26,13 +26,21 @@ extension ABI {
 
     /// The severity of this issue.
     ///
-    /// - Warning: Severity is not yet part of the JSON schema.
-    var _severity: Severity
-    
+    /// Prior to 6.3, this is nil.
+    ///
+    /// @Metadata {
+    ///   @Available(Swift, introduced: 6.3)
+    /// }
+    var severity: Severity?
+
     /// If the issue is a failing issue.
     ///
-    /// - Warning: Non-failing issues are not yet part of the JSON schema.
-    var _isFailure: Bool
+    /// Prior to 6.3, this is nil.
+    ///
+    /// @Metadata {
+    ///   @Available(Swift, introduced: 6.3)
+    /// }
+    var isFailure: Bool?
 
     /// Whether or not this issue is known to occur.
     var isKnown: Bool
@@ -51,13 +59,20 @@ extension ABI {
     var _error: EncodedError<V>?
 
     init(encoding issue: borrowing Issue, in eventContext: borrowing Event.Context) {
-      _severity = switch issue.severity {
-      case .warning: .warning
-      case .error: .error
-      }
-      _isFailure = issue.isFailure
+      // >= v0
       isKnown = issue.isKnown
       sourceLocation = issue.sourceLocation
+
+      // >= v6.3
+      if V.versionNumber >= ABI.v6_3.versionNumber {
+        severity = switch issue.severity {
+        case .warning: .warning
+        case .error: .error
+        }
+        isFailure = issue.isFailure
+      }
+
+      // Experimental
       if let backtrace = issue.sourceContext.backtrace {
         _backtrace = EncodedBacktrace(encoding: backtrace, in: eventContext)
       }

Sources/Testing/ABI/EntryPoints/EntryPoint.swift

@@ -59,11 +59,11 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha
         // Use experimental AdvancedConsoleOutputRecorder
         var advancedOptions = Event.AdvancedConsoleOutputRecorder<ABI.HighestVersion>.Options()
         advancedOptions.base = .for(.stderr)
-        
+
         let eventRecorder = Event.AdvancedConsoleOutputRecorder<ABI.HighestVersion>(options: advancedOptions) { string in
           try? FileHandle.stderr.write(string)
         }
-        
+
         configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in
           eventRecorder.record(event, in: context)
           oldEventHandler(event, context)
@@ -328,13 +328,6 @@ public struct __CommandLineArguments_v0: Sendable {
 
   /// The value of the `--attachments-path` argument.
   public var attachmentsPath: String?
-
-  /// Whether or not the experimental warning issue severity feature should be
-  /// enabled.
-  ///
-  /// This property is intended for use in testing the testing library itself.
-  /// It is not parsed as a command-line argument.
-  var isWarningIssueRecordedEventEnabled: Bool?
 }
 
 extension __CommandLineArguments_v0: Codable {
@@ -635,19 +628,15 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr
 #endif
 
   // Warning issues (experimental).
-  if args.isWarningIssueRecordedEventEnabled == true {
-    configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = true
-  } else {
-    switch args.eventStreamVersionNumber {
-    case .some(..<ABI.v6_3.versionNumber):
-      // If the event stream version was explicitly specified to a value < 6.3,
-      // disable the warning issue event to maintain legacy behavior.
-      configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = false
-    default:
-      // Otherwise the requested event stream version is ≥ 6.3, so don't change
-      // the warning issue event setting.
-      break
-    }
+  switch args.eventStreamVersionNumber {
+  case .some(..<ABI.v6_3.versionNumber):
+    // If the event stream version was explicitly specified to a value < 6.3,
+    // disable the warning issue event to maintain legacy behavior.
+    configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = false
+  default:
+    // Otherwise the requested event stream version is ≥ 6.3, so don't change
+    // the warning issue event setting.
+    break
   }
 
   return configuration

Sources/Testing/ExitTests/ExitTest.swift

@@ -1046,10 +1046,11 @@ extension ExitTest {
         // TODO: improve fidelity of issue kind reporting (especially those without associated values)
         .unconditional
       }
-      let severity: Issue.Severity = switch issue._severity {
+      let severity: Issue.Severity = switch issue.severity {
       case .warning:
         .warning
-      case .error:
+      case .error, nil:
+        // Prior to 6.3, all Issues are errors
         .error
       }
       let sourceContext = SourceContext(

Tests/TestingTests/EntryPointTests.swift

@@ -30,11 +30,12 @@ struct EntryPointTests {
     }
   }
 
-  @Test("Entry point with WarningIssues feature enabled exits with success if all issues have severity < .error")
-  func warningIssues() async throws {
+  @Test("Entry point using event stream version 0 exits with success if all issues have severity < .error")
+  func warningIssuesDisabled() async throws {
     var arguments = __CommandLineArguments_v0()
     arguments.filter = ["_recordWarningIssue"]
     arguments.includeHiddenTests = true
+    // WarningIssues is available >= 6.3
     arguments.eventStreamSchemaVersion = "0"
     arguments.verbosity = .min
 
@@ -50,13 +51,13 @@ struct EntryPointTests {
     #expect(exitCode == EXIT_SUCCESS)
   }
 
-  @Test("Entry point with WarningIssues feature enabled propagates warning issues and exits with success if all issues have severity < .error")
+
+  @Test("Entry point using event stream version 6.3 propagates warning issues and exits with success if all issues have severity < .error")
   func warningIssuesEnabled() async throws {
     var arguments = __CommandLineArguments_v0()
     arguments.filter = ["_recordWarningIssue"]
     arguments.includeHiddenTests = true
-    arguments.eventStreamSchemaVersion = "0"
-    arguments.isWarningIssueRecordedEventEnabled = true
+    arguments.eventStreamSchemaVersion = "6.3"
     arguments.verbosity = .min
 
     let exitCode = await confirmation("Warning issue recorded", expectedCount: 1) { issueRecorded in

Tests/TestingTests/SwiftPMTests.swift

@@ -20,6 +20,18 @@ private func configurationForEntryPoint(withArguments args: [String]) throws ->
   return try configurationForEntryPoint(from: args)
 }
 
+/// Reads event stream output from the provided file matching event stream
+/// version `V`.
+private func decodedEventStreamRecords<V: ABI.Version>(fromPath filePath: String) throws -> [ABI.Record<V>] {
+  try FileHandle(forReadingAtPath: filePath).readToEnd()
+    .split(whereSeparator: \.isASCIINewline)
+    .map { line in
+      try line.withUnsafeBytes { line in
+        return try JSON.decode(ABI.Record<V>.self, from: line)
+      }
+    }
+}
+
 @Suite("Swift Package Manager Integration Tests")
 struct SwiftPMTests {
   @Test("Command line arguments are available")
@@ -286,6 +298,40 @@ struct SwiftPMTests {
     #expect(versionTypeInfo == nil)
   }
 
+  @Test("Severity and isFailure fields included in version 6.3")
+  func validateEventStreamContents() async throws {
+    let tempDirPath = try temporaryDirectory()
+    let temporaryFilePath = appendPathComponent("\(UInt64.random(in: 0 ..< .max))", to: tempDirPath)
+    defer {
+      _ = remove(temporaryFilePath)
+    }
+
+    do {
+      let test = Test {
+        Issue.record("Test warning", severity: .warning)
+      }
+
+      let configuration = try configurationForEntryPoint(withArguments:
+          ["PATH", "--event-stream-output-path", temporaryFilePath, "--experimental-event-stream-version", "6.3"]
+      )
+
+      await test.run(configuration: configuration)
+    }
+
+    let issueEventRecords = try decodedEventStreamRecords(fromPath: temporaryFilePath)
+      .compactMap { (record: ABI.Record<ABI.v6_3>) in
+        if case let .event(event) = record.kind, event.kind == .issueRecorded {
+          return event
+        }
+        return nil
+      }
+
+    let issue = try #require(issueEventRecords.first?.issue)
+    #expect(issueEventRecords.count == 1)
+    #expect(issue.isFailure == false)
+    #expect(issue.severity == .warning)
+  }
+
   @Test("--event-stream-output-path argument (writes to a stream and can be read back)",
         arguments: [
           ("--event-stream-output-path", "--event-stream-version", ABI.v0.versionNumber),
@@ -320,13 +366,7 @@ struct SwiftPMTests {
       configuration.handleEvent(Event(.runEnded, testID: nil, testCaseID: nil), in: eventContext)
     }
 
-    let decodedRecords = try FileHandle(forReadingAtPath: temporaryFilePath).readToEnd()
-      .split(whereSeparator: \.isASCIINewline)
-      .map { line in
-        try line.withUnsafeBytes { line in
-          try JSON.decode(ABI.Record<V>.self, from: line)
-        }
-      }
+    let decodedRecords: [ABI.Record<V>] = try decodedEventStreamRecords(fromPath: temporaryFilePath)
 
     let testRecords = decodedRecords.compactMap { record in
       if case let .test(test) = record.kind {