diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingConfig.java index ddbc6754379..39c932e9d9d 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingConfig.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingConfig.java @@ -45,7 +45,9 @@ public class TracingConfig extends ReconfigurableConfig { type = ConfigType.BOOLEAN, reconfigurable = true, tags = { ConfigTag.OZONE, ConfigTag.HDDS }, - description = "If true, tracing is initialized and spans may be exported (subject to sampling)." + description = "If true, Ozone initializes its own tracer and exports spans (subject to sampling). " + + "If false, the Ozone client does not use that tracer; optional child spans under an " + + "application trace are controlled by ozone.tracing.client.application-aware." ) private boolean tracingEnabled; @@ -79,6 +81,24 @@ public class TracingConfig extends ReconfigurableConfig { ) private String spanSampling; + @Config( + key = "ozone.tracing.client.application-aware", + defaultValue = "true", + type = ConfigType.BOOLEAN, + reconfigurable = true, + tags = { ConfigTag.OZONE, ConfigTag.HDDS }, + description = "This is used when ozone.tracing.enabled is false. If true and " + + "an active span exists in the current OpenTelemetry Context (application " + + "GlobalOpenTelemetry), the client may create child spans under that trace. If no " + + "active span exists, no spans are created. If false, no spans are created regardless " + + "of application context." + ) + private boolean clientApplicationAware; + + public boolean isClientApplicationAware() { + return clientApplicationAware; + } + public boolean isTracingEnabled() { return tracingEnabled; } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java index 9b7f6347fef..28c4ed4920e 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java @@ -17,6 +17,7 @@ package org.apache.hadoop.hdds.tracing; +import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.Attributes; @@ -48,10 +49,12 @@ public final class TracingUtil { private static final Logger LOG = LoggerFactory.getLogger(TracingUtil.class); private static final String NULL_SPAN_AS_STRING = ""; + private static final String OZONE_CLIENT_TRACER_SCOPE = "org.apache.hadoop.ozone.client"; private static volatile boolean isInit = false; private static Tracer tracer = OpenTelemetry.noop().getTracer("noop"); private static SdkTracerProvider sdkTracerProvider; + private static volatile TracingConfig clientTracingConfig; private TracingUtil() { } @@ -61,6 +64,7 @@ private TracingUtil() { */ public static synchronized void initTracing( String serviceName, TracingConfig tracingConfig) { + clientTracingConfig = tracingConfig; if (!tracingConfig.isTracingEnabled() || isInit) { return; } @@ -74,6 +78,38 @@ public static synchronized void initTracing( } } + /** + * When ozone tracing is off , client tracing is off and no context exist thn + * span creation should be skipped. + */ + private static boolean shouldByPassSpanCreation() { + if (clientTracingConfig == null) { + return false; + } + //if ozone tracing is true then resolve tracer as usual and create span + if (clientTracingConfig.isTracingEnabled()) { + return false; + } + boolean hasValidContext = Span.current().getSpanContext().isValid(); + boolean clientApplicationTracing = clientTracingConfig.isClientApplicationAware(); + + //if ozone tracing is false but context exists and client tracing is true then create span. + return !(hasValidContext && clientApplicationTracing); + } + + /** + * When Ozone OTLP tracing is off but the app has an active span, use the app SDK via GlobalOpenTelemetry. + */ + private static Tracer resolveTracerForNewSpan(Span parentSpan) { + if (clientTracingConfig != null + && !clientTracingConfig.isTracingEnabled() + && clientTracingConfig.isClientApplicationAware() + && parentSpan.getSpanContext().isValid()) { + return GlobalOpenTelemetry.get().getTracer(OZONE_CLIENT_TRACER_SCOPE); + } + return tracer; + } + /** * Receives serviceName and configurationSource. * Delegates tracing initiation to {@link #initTracing(String, TracingConfig)}. @@ -254,15 +290,19 @@ static Map parseSpanSamplingConfig(String configStr) { */ public static void executeInNewSpan(String spanName, CheckedRunnable runnable) throws E { + if (shouldByPassSpanCreation()) { + runnable.run(); + return; + } Span span = buildSpan(spanName); executeInSpan(span, runnable); } - /** - * Execute {@code supplier} inside an activated new span. - */ public static R executeInNewSpan(String spanName, CheckedSupplier supplier) throws E { + if (shouldByPassSpanCreation()) { + return supplier.get(); + } Span span = buildSpan(spanName); return executeInSpan(span, supplier); } @@ -317,6 +357,9 @@ public static void executeAsChildSpan(String spanName, * in case of Exceptions. */ public static TraceCloseable createActivatedSpan(String spanName) { + if (shouldByPassSpanCreation()) { + return () -> { }; + } Span span = buildSpan(spanName); Scope scope = span.makeCurrent(); return () -> { @@ -380,11 +423,11 @@ private void parse(String carrier) { private static Span buildSpan(String spanName) { Context currentContext = Context.current(); Span parentSpan = Span.fromContext(currentContext); + Tracer spanTracer = resolveTracerForNewSpan(parentSpan); if (parentSpan.getSpanContext().isValid()) { - return tracer.spanBuilder(spanName).setParent(currentContext).startSpan(); - } else { - return tracer.spanBuilder(spanName).setNoParent().startSpan(); + return spanTracer.spanBuilder(spanName).setParent(currentContext).startSpan(); } + return spanTracer.spanBuilder(spanName).setNoParent().startSpan(); } } diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingUtilClientApplicationAware.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingUtilClientApplicationAware.java new file mode 100644 index 00000000000..012c0ab2566 --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingUtilClientApplicationAware.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.tracing; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Scope; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import io.opentelemetry.sdk.trace.export.SpanExporter; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import org.apache.hadoop.hdds.conf.InMemoryConfigurationForTesting; +import org.apache.hadoop.hdds.conf.MutableConfigurationSource; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Tests client application-aware tracing: when Ozone OTLP tracing is off but an + * active span exists, spans use {@link GlobalOpenTelemetry}; otherwise the + * static Ozone {@link TracingUtil} tracer is used, or span creation is skipped. + */ +public class TestTracingUtilClientApplicationAware { + + private static final String OZONE_CLIENT_TRACER_SCOPE = "org.apache.hadoop.ozone.client"; + + private final List exportedSpans = new CopyOnWriteArrayList<>(); + private SdkTracerProvider testSdkTracerProvider; + + @BeforeEach + public void setUp() { + GlobalOpenTelemetry.resetForTest(); + TracingUtil.reconfigureTracing("client", clientTracingConfig(false, true)); + } + + @AfterEach + public void tearDown() { + if (testSdkTracerProvider != null) { + testSdkTracerProvider.shutdown(); + testSdkTracerProvider = null; + } + GlobalOpenTelemetry.resetForTest(); + exportedSpans.clear(); + TracingUtil.reconfigureTracing("client", clientTracingConfig(false, true)); + } + + private static TracingConfig clientTracingConfig(boolean tracingEnabled, + boolean clientApplicationAware) { + MutableConfigurationSource conf = new InMemoryConfigurationForTesting(); + conf.setBoolean("ozone.tracing.enabled", tracingEnabled); + conf.setBoolean("ozone.tracing.client.application-aware", clientApplicationAware); + return conf.getObject(TracingConfig.class); + } + + /** + * Registers a global SDK that records ended spans into {@link #exportedSpans}. + */ + private void registerGlobalTestSdk() { + SpanExporter exporter = new SpanExporter() { + @Override + public CompletableResultCode export(Collection spans) { + exportedSpans.addAll(spans); + return CompletableResultCode.ofSuccess(); + } + + @Override + public CompletableResultCode flush() { + return CompletableResultCode.ofSuccess(); + } + + @Override + public CompletableResultCode shutdown() { + return CompletableResultCode.ofSuccess(); + } + }; + testSdkTracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .build(); + GlobalOpenTelemetry.set( + OpenTelemetrySdk.builder() + .setTracerProvider(testSdkTracerProvider) + .build()); + } + + private boolean hasExportedSpanNamed(String name) { + return exportedSpans.stream().anyMatch(s -> name.equals(s.getName())); + } + + @Test + public void testOzoneTracingOnUsesStaticTracerNotGlobalOpenTelemetry() throws Exception { + registerGlobalTestSdk(); + TracingUtil.reconfigureTracing("client", clientTracingConfig(true, true)); + + TracingUtil.executeInNewSpan("ozone-client-span", () -> { + assertThat(Span.current().getSpanContext().isValid()).isTrue(); + }); + + assertThat(hasExportedSpanNamed("ozone-client-span")) + .withFailMessage("When Ozone client tracing is on, spans must use the Ozone SDK tracer, " + + "not GlobalOpenTelemetry, so the test global exporter must not see them.") + .isFalse(); + } + + @Test + public void testOzoneTracingOffWithAppAwareAndParentUsesGlobalTracer() throws Exception { + registerGlobalTestSdk(); + TracingUtil.reconfigureTracing("client", clientTracingConfig(false, true)); + + Tracer appTracer = GlobalOpenTelemetry.get().getTracer("test-app"); + Span parent = appTracer.spanBuilder("parent").startSpan(); + final String parentSpanId = parent.getSpanContext().getSpanId(); + try (Scope ignored = parent.makeCurrent()) { + TracingUtil.executeInNewSpan("child-from-global", () -> + assertThat(Span.current().getSpanContext().isValid()).isTrue()); + } finally { + parent.end(); + } + testSdkTracerProvider.forceFlush(); + + assertThat(hasExportedSpanNamed("child-from-global")) + .withFailMessage("With Ozone tracing off, client application-aware on, and a valid parent span, " + + "the child must be created with GlobalOpenTelemetry.") + .isTrue(); + + // Child span must use the Ozone client scope on GlobalOpenTelemetry (not the test-app parent scope). + assertThat(exportedSpans) + .filteredOn(s -> "child-from-global".equals(s.getName())) + .singleElement() + .extracting(s -> s.getInstrumentationScopeInfo().getName()) + .isEqualTo(OZONE_CLIENT_TRACER_SCOPE); + + // Child must link to the application parent span in the same trace. + assertThat(exportedSpans) + .filteredOn(s -> "child-from-global".equals(s.getName())) + .singleElement() + .extracting(SpanData::getParentSpanId) + .isEqualTo(parentSpanId); + } + + @Test + public void testOzoneTracingOffApplicationAwareFalseSkipsSpan() throws Exception { + registerGlobalTestSdk(); + TracingUtil.reconfigureTracing("client", clientTracingConfig(false, false)); + + Tracer appTracer = GlobalOpenTelemetry.get().getTracer("test-app"); + Span parent = appTracer.spanBuilder("parent").startSpan(); + try (Scope ignored = parent.makeCurrent()) { + TracingUtil.executeInNewSpan("should-not-exist", () -> { + assertEquals( + parent.getSpanContext(), + Span.current().getSpanContext(), + "When application-aware is false, TracingUtil should not create a child span; " + + "the active span should remain the application parent."); + }); + } finally { + parent.end(); + } + testSdkTracerProvider.forceFlush(); + + assertThat(hasExportedSpanNamed("should-not-exist")).isFalse(); + } + + @Test + public void testOzoneTracingOffNoParentContextSkipsSpan() throws Exception { + registerGlobalTestSdk(); + TracingUtil.reconfigureTracing("client", clientTracingConfig(false, true)); + + TracingUtil.executeInNewSpan("no-parent-bypass", () -> + assertThat(Span.current().getSpanContext().isValid()) + .withFailMessage("Without a valid parent, span creation should be bypassed.") + .isFalse()); + testSdkTracerProvider.forceFlush(); + + assertThat(hasExportedSpanNamed("no-parent-bypass")).isFalse(); + } + + @Test + public void testCreateActivatedSpanBypassWhenConditionsNotMet() throws Exception { + registerGlobalTestSdk(); + TracingUtil.reconfigureTracing("client", clientTracingConfig(false, false)); + + try (TracingUtil.TraceCloseable closeable = + TracingUtil.createActivatedSpan("activated-bypass")) { + assertNotNull(closeable); + } + if (testSdkTracerProvider != null) { + testSdkTracerProvider.forceFlush(); + } + assertThat(hasExportedSpanNamed("activated-bypass")).isFalse(); + } +}