HDDS-13803. Client aware tracing

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingConfig.java

@@ -45,7 +45,9 @@ public class TracingConfig extends ReconfigurableConfig {
       type = ConfigType.BOOLEAN,
       reconfigurable = true,
       tags = { ConfigTag.OZONE, ConfigTag.HDDS },
-      description = "If true, tracing is initialized and spans may be exported (subject to sampling)."
+      description = "If true, Ozone initializes its own tracer and exports spans (subject to sampling). "
+          + "If false, the Ozone client does not use that tracer; optional child spans under an "
+          + "application trace are controlled by ozone.tracing.client.application-aware."
   )
   private boolean tracingEnabled;
 
@@ -79,6 +81,24 @@ public class TracingConfig extends ReconfigurableConfig {
   )
   private String spanSampling;
 
+  @Config(
+      key = "ozone.tracing.client.application-aware",
+      defaultValue = "true",
+      type = ConfigType.BOOLEAN,
+      reconfigurable = true,
+      tags = { ConfigTag.OZONE, ConfigTag.HDDS },
+      description = "This is used when ozone.tracing.enabled is false. If true and "
+      + "an active span exists in the current OpenTelemetry Context (application "
+      + "GlobalOpenTelemetry), the client may create child spans under that trace. If no "
+      + "active span exists, no spans are created. If false, no spans are created regardless "
+      + "of application context."
+  )
+  private boolean clientApplicationAware;
+
+  public boolean isClientApplicationAware() {
+    return clientApplicationAware;
+  }
+
   public boolean isTracingEnabled() {
     return tracingEnabled;
   }

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java

@@ -17,6 +17,7 @@
 
 package org.apache.hadoop.hdds.tracing;
 
+import io.opentelemetry.api.GlobalOpenTelemetry;
 import io.opentelemetry.api.OpenTelemetry;
 import io.opentelemetry.api.common.AttributeKey;
 import io.opentelemetry.api.common.Attributes;
@@ -48,10 +49,12 @@
 public final class TracingUtil {
   private static final Logger LOG = LoggerFactory.getLogger(TracingUtil.class);
   private static final String NULL_SPAN_AS_STRING = "";
+  private static final String OZONE_CLIENT_TRACER_SCOPE = "org.apache.hadoop.ozone.client";
 
   private static volatile boolean isInit = false;
   private static Tracer tracer = OpenTelemetry.noop().getTracer("noop");
   private static SdkTracerProvider sdkTracerProvider;
+  private static volatile TracingConfig clientTracingConfig;
 
   private TracingUtil() {
   }
@@ -61,6 +64,7 @@ private TracingUtil() {
    */
   public static synchronized void initTracing(
       String serviceName, TracingConfig tracingConfig) {
+    clientTracingConfig = tracingConfig;
     if (!tracingConfig.isTracingEnabled() || isInit) {
       return;
     }
@@ -74,6 +78,38 @@ public static synchronized void initTracing(
     }
   }
 
+  /**
+   * When ozone tracing is off , client tracing is off and no context exist thn
+   * span creation should be skipped.
+   */
+  private static boolean shouldByPassSpanCreation() {
+    if (clientTracingConfig == null) {
+      return false;
+    }
+    //if ozone tracing is true then resolve tracer as usual and create span
+    if (clientTracingConfig.isTracingEnabled()) {
+      return false;
+    }
+    boolean hasValidContext = Span.current().getSpanContext().isValid();
+    boolean clientApplicationTracing = clientTracingConfig.isClientApplicationAware();
+
+    //if ozone tracing is false but context exists and client tracing is true then create span.
+    return !(hasValidContext && clientApplicationTracing);
+  }
+
+  /**
+   * When Ozone OTLP tracing is off but the app has an active span, use the app SDK via GlobalOpenTelemetry.
+   */
+  private static Tracer resolveTracerForNewSpan(Span parentSpan) {
+    if (clientTracingConfig != null
+        && !clientTracingConfig.isTracingEnabled()
+        && clientTracingConfig.isClientApplicationAware()
+        && parentSpan.getSpanContext().isValid()) {
+      return GlobalOpenTelemetry.get().getTracer(OZONE_CLIENT_TRACER_SCOPE);
+    }
+    return tracer;
+  }
+
   /**
    * Receives serviceName and configurationSource.
    * Delegates tracing initiation to {@link #initTracing(String, TracingConfig)}.
@@ -254,15 +290,19 @@ static Map<String, LoopSampler> parseSpanSamplingConfig(String configStr) {
    */
   public static <E extends Exception> void executeInNewSpan(String spanName,
       CheckedRunnable<E> runnable) throws E {
+    if (shouldByPassSpanCreation()) {
+      runnable.run();
+      return;
+    }
     Span span = buildSpan(spanName);
     executeInSpan(span, runnable);
   }
 
-  /**
-   * Execute {@code supplier} inside an activated new span.
-   */
   public static <R, E extends Exception> R executeInNewSpan(String spanName,
       CheckedSupplier<R, E> supplier) throws E {
+    if (shouldByPassSpanCreation()) {
+      return supplier.get();
+    }
     Span span = buildSpan(spanName);
     return executeInSpan(span, supplier);
   }
@@ -317,6 +357,9 @@ public static <E extends Exception> void executeAsChildSpan(String spanName,
    * in case of Exceptions.
    */
   public static TraceCloseable createActivatedSpan(String spanName) {
+    if (shouldByPassSpanCreation()) {
+      return () -> { };
+    }
     Span span = buildSpan(spanName);
     Scope scope = span.makeCurrent();
     return () -> {
@@ -380,11 +423,11 @@ private void parse(String carrier) {
   private static Span buildSpan(String spanName) {
     Context currentContext = Context.current();
     Span parentSpan = Span.fromContext(currentContext);
+    Tracer spanTracer = resolveTracerForNewSpan(parentSpan);
 
     if (parentSpan.getSpanContext().isValid()) {
-      return tracer.spanBuilder(spanName).setParent(currentContext).startSpan();
-    } else {
-      return tracer.spanBuilder(spanName).setNoParent().startSpan();
+      return spanTracer.spanBuilder(spanName).setParent(currentContext).startSpan();
     }
+    return spanTracer.spanBuilder(spanName).setNoParent().startSpan();
   }
 }

hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingUtilClientApplicationAware.java

@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.tracing;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+import io.opentelemetry.api.GlobalOpenTelemetry;
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.Tracer;
+import io.opentelemetry.context.Scope;
+import io.opentelemetry.sdk.OpenTelemetrySdk;
+import io.opentelemetry.sdk.common.CompletableResultCode;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.data.SpanData;
+import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
+import io.opentelemetry.sdk.trace.export.SpanExporter;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import org.apache.hadoop.hdds.conf.InMemoryConfigurationForTesting;
+import org.apache.hadoop.hdds.conf.MutableConfigurationSource;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests client application-aware tracing: when Ozone OTLP tracing is off but an
+ * active span exists, spans use {@link GlobalOpenTelemetry}; otherwise the
+ * static Ozone {@link TracingUtil} tracer is used, or span creation is skipped.
+ */
+public class TestTracingUtilClientApplicationAware {
+
+  private static final String OZONE_CLIENT_TRACER_SCOPE = "org.apache.hadoop.ozone.client";
+
+  private final List<SpanData> exportedSpans = new CopyOnWriteArrayList<>();
+  private SdkTracerProvider testSdkTracerProvider;
+
+  @BeforeEach
+  public void setUp() {
+    GlobalOpenTelemetry.resetForTest();
+    TracingUtil.reconfigureTracing("client", clientTracingConfig(false, true));
+  }
+
+  @AfterEach
+  public void tearDown() {
+    if (testSdkTracerProvider != null) {
+      testSdkTracerProvider.shutdown();
+      testSdkTracerProvider = null;
+    }
+    GlobalOpenTelemetry.resetForTest();
+    exportedSpans.clear();
+    TracingUtil.reconfigureTracing("client", clientTracingConfig(false, true));
+  }
+
+  private static TracingConfig clientTracingConfig(boolean tracingEnabled,
+                                                   boolean clientApplicationAware) {
+    MutableConfigurationSource conf = new InMemoryConfigurationForTesting();
+    conf.setBoolean("ozone.tracing.enabled", tracingEnabled);
+    conf.setBoolean("ozone.tracing.client.application-aware", clientApplicationAware);
+    return conf.getObject(TracingConfig.class);
+  }
+
+  /**
+   * Registers a global SDK that records ended spans into {@link #exportedSpans}.
+   */
+  private void registerGlobalTestSdk() {
+    SpanExporter exporter = new SpanExporter() {
+      @Override
+      public CompletableResultCode export(Collection<SpanData> spans) {
+        exportedSpans.addAll(spans);
+        return CompletableResultCode.ofSuccess();
+      }
+
+      @Override
+      public CompletableResultCode flush() {
+        return CompletableResultCode.ofSuccess();
+      }
+
+      @Override
+      public CompletableResultCode shutdown() {
+        return CompletableResultCode.ofSuccess();
+      }
+    };
+    testSdkTracerProvider = SdkTracerProvider.builder()
+        .addSpanProcessor(SimpleSpanProcessor.create(exporter))
+        .build();
+    GlobalOpenTelemetry.set(
+        OpenTelemetrySdk.builder()
+            .setTracerProvider(testSdkTracerProvider)
+            .build());
+  }
+
+  private boolean hasExportedSpanNamed(String name) {
+    return exportedSpans.stream().anyMatch(s -> name.equals(s.getName()));
+  }
+
+  @Test
+  public void testOzoneTracingOnUsesStaticTracerNotGlobalOpenTelemetry() throws Exception {
+    registerGlobalTestSdk();
+    TracingUtil.reconfigureTracing("client", clientTracingConfig(true, true));
+
+    TracingUtil.executeInNewSpan("ozone-client-span", () -> {
+      assertThat(Span.current().getSpanContext().isValid()).isTrue();
+    });
+
+    assertThat(hasExportedSpanNamed("ozone-client-span"))
+        .withFailMessage("When Ozone client tracing is on, spans must use the Ozone SDK tracer, "
+            + "not GlobalOpenTelemetry, so the test global exporter must not see them.")
+        .isFalse();
+  }
+
+  @Test
+  public void testOzoneTracingOffWithAppAwareAndParentUsesGlobalTracer() throws Exception {
+    registerGlobalTestSdk();
+    TracingUtil.reconfigureTracing("client", clientTracingConfig(false, true));
+
+    Tracer appTracer = GlobalOpenTelemetry.get().getTracer("test-app");
+    Span parent = appTracer.spanBuilder("parent").startSpan();
+    final String parentSpanId = parent.getSpanContext().getSpanId();
+    try (Scope ignored = parent.makeCurrent()) {
+      TracingUtil.executeInNewSpan("child-from-global", () ->
+          assertThat(Span.current().getSpanContext().isValid()).isTrue());
+    } finally {
+      parent.end();
+    }
+    testSdkTracerProvider.forceFlush();
+
+    assertThat(hasExportedSpanNamed("child-from-global"))
+        .withFailMessage("With Ozone tracing off, client application-aware on, and a valid parent span, "
+            + "the child must be created with GlobalOpenTelemetry.")
+        .isTrue();
+
+    // Child span must use the Ozone client scope on GlobalOpenTelemetry (not the test-app parent scope).
+    assertThat(exportedSpans)
+        .filteredOn(s -> "child-from-global".equals(s.getName()))
+        .singleElement()
+        .extracting(s -> s.getInstrumentationScopeInfo().getName())
+        .isEqualTo(OZONE_CLIENT_TRACER_SCOPE);
+
+    // Child must link to the application parent span in the same trace.
+    assertThat(exportedSpans)
+        .filteredOn(s -> "child-from-global".equals(s.getName()))
+        .singleElement()
+        .extracting(SpanData::getParentSpanId)
+        .isEqualTo(parentSpanId);
+  }
+
+  @Test
+  public void testOzoneTracingOffApplicationAwareFalseSkipsSpan() throws Exception {
+    registerGlobalTestSdk();
+    TracingUtil.reconfigureTracing("client", clientTracingConfig(false, false));
+
+    Tracer appTracer = GlobalOpenTelemetry.get().getTracer("test-app");
+    Span parent = appTracer.spanBuilder("parent").startSpan();
+    try (Scope ignored = parent.makeCurrent()) {
+      TracingUtil.executeInNewSpan("should-not-exist", () -> {
+        assertEquals(
+            parent.getSpanContext(),
+            Span.current().getSpanContext(),
+            "When application-aware is false, TracingUtil should not create a child span; "
+                + "the active span should remain the application parent.");
+      });
+    } finally {
+      parent.end();
+    }
+    testSdkTracerProvider.forceFlush();
+
+    assertThat(hasExportedSpanNamed("should-not-exist")).isFalse();
+  }
+
+  @Test
+  public void testOzoneTracingOffNoParentContextSkipsSpan() throws Exception {
+    registerGlobalTestSdk();
+    TracingUtil.reconfigureTracing("client", clientTracingConfig(false, true));
+
+    TracingUtil.executeInNewSpan("no-parent-bypass", () ->
+        assertThat(Span.current().getSpanContext().isValid())
+            .withFailMessage("Without a valid parent, span creation should be bypassed.")
+            .isFalse());
+    testSdkTracerProvider.forceFlush();
+
+    assertThat(hasExportedSpanNamed("no-parent-bypass")).isFalse();
+  }
+
+  @Test
+  public void testCreateActivatedSpanBypassWhenConditionsNotMet() throws Exception {
+    registerGlobalTestSdk();
+    TracingUtil.reconfigureTracing("client", clientTracingConfig(false, false));
+
+    try (TracingUtil.TraceCloseable closeable =
+             TracingUtil.createActivatedSpan("activated-bypass")) {
+      assertNotNull(closeable);
+    }
+    if (testSdkTracerProvider != null) {
+      testSdkTracerProvider.forceFlush();
+    }
+    assertThat(hasExportedSpanNamed("activated-bypass")).isFalse();
+  }
+}