Prevent completed scope diagnostic commands from failing smoke tests

dd-java-agent/instrumentation-testing/build.gradle

@@ -62,6 +62,7 @@ dependencies {
   implementation "org.junit.platform:junit-platform-runner:${libs.versions.junit.platform.get()}"
 
   testImplementation project(':dd-java-agent:instrumentation:datadog:tracing:trace-annotation')
+  testImplementation sourceSets.smokeAgent.output
 
   testImplementation group: 'cglib', name: 'cglib', version: '3.2.5'
   // test instrumenting java 1.1 bytecode

dd-java-agent/instrumentation-testing/src/smokeAgent/java/datadog/trace/agent/test/smoke/ScopeDiagnosticsAgent.java

@@ -19,13 +19,20 @@
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Properties;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.jar.JarEntry;
 import java.util.jar.JarFile;
 
 /** File-controlled companion agent used only by smoke-test child processes. */
 public final class ScopeDiagnosticsAgent {
   private static final AtomicLong REPORT_IDS = new AtomicLong();
+  private static final long COMMAND_READ_TIMEOUT_NANOS = TimeUnit.SECONDS.toNanos(5);
+
+  @FunctionalInterface
+  interface CommandReader {
+    Properties read() throws IOException;
+  }
 
   private ScopeDiagnosticsAgent() {}
 
@@ -104,11 +111,16 @@ private static void control(File directory, Method start, Method finish, boolean
     while (!Thread.currentThread().isInterrupted()) {
       try {
         if (commandFile.exists()) {
-          Properties command = read(commandFile);
+          Properties command = readCommand(commandFile);
           long seq = Long.parseLong(command.getProperty("seq"));
           if (seq != previous) {
             previous = seq;
             try {
+              // The client waits for this response before publishing another command.
+              // Consume the file now so idle polls cannot reread a completed command.
+              if (!commandFile.delete()) {
+                throw new IOException("Cannot consume scope diagnostic command: " + commandFile);
+              }
               Properties result;
               String action = command.getProperty("action");
               if ("start".equals(action)) {
@@ -176,14 +188,33 @@ private static Map<String, byte[]> payload() throws Exception {
     return classes;
   }
 
-  private static Properties read(File file) throws Exception {
+  private static Properties read(File file) throws IOException {
     Properties properties = new Properties();
     try (InputStream input = new FileInputStream(file)) {
       properties.load(input);
     }
     return properties;
   }
 
+  private static Properties readCommand(File file) throws IOException, InterruptedException {
+    return readCommand(() -> read(file), COMMAND_READ_TIMEOUT_NANOS);
+  }
+
+  static Properties readCommand(CommandReader reader, long timeoutNanos)
+      throws IOException, InterruptedException {
+    long deadline = System.nanoTime() + timeoutNanos;
+    for (; ; ) {
+      try {
+        return reader.read();
+      } catch (IOException failure) {
+        if (System.nanoTime() >= deadline) {
+          throw failure;
+        }
+        Thread.sleep(20);
+      }
+    }
+  }
+
   private static Properties response(long seq, Properties data) {
     Properties result = new Properties();
     result.putAll(data);

dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/smoke/ScopeDiagnosticsCommandReadTest.java

@@ -0,0 +1,49 @@
+package datadog.trace.agent.test.smoke;
+
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import java.io.IOException;
+import java.util.Properties;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.junit.jupiter.api.Test;
+
+class ScopeDiagnosticsCommandReadTest {
+  @Test
+  void retriesTransientReadFailure() throws Exception {
+    AtomicInteger reads = new AtomicInteger();
+    Properties command = new Properties();
+
+    Properties result =
+        ScopeDiagnosticsAgent.readCommand(
+            () -> {
+              if (reads.incrementAndGet() == 1) {
+                throw new IOException("temporary read failure");
+              }
+              return command;
+            },
+            SECONDS.toNanos(5));
+
+    assertSame(command, result);
+    assertEquals(2, reads.get());
+  }
+
+  @Test
+  void retainsReadFailureWhenDeadlineExpires() {
+    IOException failure = new IOException("persistent read failure");
+
+    IOException result =
+        assertThrows(
+            IOException.class,
+            () ->
+                ScopeDiagnosticsAgent.readCommand(
+                    () -> {
+                      throw failure;
+                    },
+                    0));
+
+    assertSame(failure, result);
+  }
+}

dd-smoke-tests/src/test/java/datadog/smoketest/ScopeDiagnosticsAgentTest.java

@@ -79,6 +79,7 @@ void serverWindowsObserveWorkWithoutRetainingPreviousLeak() throws Exception {
       diagnostics.start(child);
       exercise(input, output, "resolved");
       diagnostics.finish(child);
+      assertFalse(Files.exists(diagnostics.directory().resolve("command")));
       Properties second = result(diagnostics.directory().resolve("response-4"));
       assertTrue(Integer.parseInt(second.getProperty("eventCount")) > 0);
       assertTrue(second.getProperty("timeline").contains("diagnostic-resolved"));