From 99adc0fd54f1f52cd6ecea4cb3a4da73f3388f99 Mon Sep 17 00:00:00 2001 From: sravani-revuri Date: Tue, 14 Apr 2026 10:50:14 +0530 Subject: [PATCH 1/3] HDDS-13803. Client aware tracing --- .../hadoop/hdds/tracing/TracingConfig.java | 16 ++ .../hadoop/hdds/tracing/TracingUtil.java | 59 ++++- ...TestTracingUtilClientApplicationAware.java | 207 ++++++++++++++++++ 3 files changed, 274 insertions(+), 8 deletions(-) create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingUtilClientApplicationAware.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingConfig.java index ddbc6754379..e2e394352f0 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingConfig.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingConfig.java @@ -79,6 +79,22 @@ public class TracingConfig extends ReconfigurableConfig { ) private String spanSampling; + @Config( + key = "ozone.tracing.client.application-aware", + defaultValue = "true", + type = ConfigType.BOOLEAN, + reconfigurable = true, + tags = { ConfigTag.OZONE, ConfigTag.HDDS }, + description = "When true and Ozone's own tracing is disabled, the client may still " + + "emit child spans using the application's OpenTelemetry (GlobalOpenTelemetry) " + + "when an active span exists in Context." + ) + private boolean clientApplicationAware; + + public boolean isClientApplicationAware() { + return clientApplicationAware; + } + public boolean isTracingEnabled() { return tracingEnabled; } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java index 9b7f6347fef..ad864ee3750 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java @@ -17,6 +17,7 @@ package org.apache.hadoop.hdds.tracing; +import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.Attributes; @@ -48,10 +49,12 @@ public final class TracingUtil { private static final Logger LOG = LoggerFactory.getLogger(TracingUtil.class); private static final String NULL_SPAN_AS_STRING = ""; + private static final String OZONE_CLIENT_TRACER_SCOPE = "org.apache.hadoop.ozone.client"; private static volatile boolean isInit = false; private static Tracer tracer = OpenTelemetry.noop().getTracer("noop"); private static SdkTracerProvider sdkTracerProvider; + private static volatile TracingConfig clientTracingConfig; private TracingUtil() { } @@ -61,6 +64,7 @@ private TracingUtil() { */ public static synchronized void initTracing( String serviceName, TracingConfig tracingConfig) { + clientTracingConfig = tracingConfig; if (!tracingConfig.isTracingEnabled() || isInit) { return; } @@ -74,6 +78,38 @@ public static synchronized void initTracing( } } + /** + * When ozone tracing is off , client tracing is off and no context exist thn + * span creation should be skipped. + */ + private static boolean shouldByPassSpanCreation() { + if (clientTracingConfig == null) { + return false; + } + //if ozone tracing is true then resolve tracer as usual and create span + if (clientTracingConfig.isTracingEnabled()) { + return false; + } + boolean hasValidContext = Span.current().getSpanContext().isValid(); + boolean clientApplicationTracing = clientTracingConfig.isClientApplicationAware(); + + //if ozone tracing is false but context exists and client tracing is true then create span. + return !(hasValidContext && clientApplicationTracing); + } + + /** + * When Ozone OTLP tracing is off but the app has an active span, use the app SDK via GlobalOpenTelemetry. + */ + private static Tracer resolveTracerForNewSpan(Span parentSpan) { + if (clientTracingConfig != null + && !clientTracingConfig.isTracingEnabled() + && clientTracingConfig.isClientApplicationAware() + && parentSpan.getSpanContext().isValid()) { + return GlobalOpenTelemetry.get().getTracer(OZONE_CLIENT_TRACER_SCOPE); + } + return tracer; + } + /** * Receives serviceName and configurationSource. * Delegates tracing initiation to {@link #initTracing(String, TracingConfig)}. @@ -253,16 +289,20 @@ static Map parseSpanSamplingConfig(String configStr) { * If a parent span exists in the current context, this becomes a child span. */ public static void executeInNewSpan(String spanName, - CheckedRunnable runnable) throws E { + CheckedRunnable runnable) throws E { + if (shouldByPassSpanCreation()) { + runnable.run(); + return; + } Span span = buildSpan(spanName); executeInSpan(span, runnable); } - /** - * Execute {@code supplier} inside an activated new span. - */ public static R executeInNewSpan(String spanName, - CheckedSupplier supplier) throws E { + CheckedSupplier supplier) throws E { + if (shouldByPassSpanCreation()) { + return supplier.get(); + } Span span = buildSpan(spanName); return executeInSpan(span, supplier); } @@ -317,6 +357,9 @@ public static void executeAsChildSpan(String spanName, * in case of Exceptions. */ public static TraceCloseable createActivatedSpan(String spanName) { + if (shouldByPassSpanCreation()) { + return () -> { }; + } Span span = buildSpan(spanName); Scope scope = span.makeCurrent(); return () -> { @@ -380,11 +423,11 @@ private void parse(String carrier) { private static Span buildSpan(String spanName) { Context currentContext = Context.current(); Span parentSpan = Span.fromContext(currentContext); + Tracer spanTracer = resolveTracerForNewSpan(parentSpan); if (parentSpan.getSpanContext().isValid()) { - return tracer.spanBuilder(spanName).setParent(currentContext).startSpan(); - } else { - return tracer.spanBuilder(spanName).setNoParent().startSpan(); + return spanTracer.spanBuilder(spanName).setParent(currentContext).startSpan(); } + return spanTracer.spanBuilder(spanName).setNoParent().startSpan(); } } diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingUtilClientApplicationAware.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingUtilClientApplicationAware.java new file mode 100644 index 00000000000..a853b5f295a --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingUtilClientApplicationAware.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.tracing; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Scope; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import io.opentelemetry.sdk.trace.export.SpanExporter; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import org.apache.hadoop.hdds.conf.InMemoryConfigurationForTesting; +import org.apache.hadoop.hdds.conf.MutableConfigurationSource; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Tests client application-aware tracing: when Ozone OTLP tracing is off but an + * active span exists, spans use {@link GlobalOpenTelemetry}; otherwise the + * static Ozone {@link TracingUtil} tracer is used, or span creation is skipped. + */ +public class TestTracingUtilClientApplicationAware { + + private static final String OZONE_CLIENT_TRACER_SCOPE = "org.apache.hadoop.ozone.client"; + + private final List exportedSpans = new CopyOnWriteArrayList<>(); + private SdkTracerProvider testSdkTracerProvider; + + @BeforeEach + public void setUp() { + GlobalOpenTelemetry.resetForTest(); + TracingUtil.reconfigureTracing("client", clientTracingConfig(false, true)); + } + + @AfterEach + public void tearDown() { + if (testSdkTracerProvider != null) { + testSdkTracerProvider.shutdown(); + testSdkTracerProvider = null; + } + GlobalOpenTelemetry.resetForTest(); + exportedSpans.clear(); + TracingUtil.reconfigureTracing("client", clientTracingConfig(false, true)); + } + + private static TracingConfig clientTracingConfig(boolean tracingEnabled, + boolean clientApplicationAware) { + MutableConfigurationSource conf = new InMemoryConfigurationForTesting(); + conf.setBoolean("ozone.tracing.enabled", tracingEnabled); + conf.setBoolean("ozone.tracing.client.application-aware", clientApplicationAware); + return conf.getObject(TracingConfig.class); + } + + /** + * Registers a global SDK that records ended spans into {@link #exportedSpans}. + */ + private void registerGlobalTestSdk() { + SpanExporter exporter = new SpanExporter() { + @Override + public CompletableResultCode export(Collection spans) { + exportedSpans.addAll(spans); + return CompletableResultCode.ofSuccess(); + } + + @Override + public CompletableResultCode flush() { + return CompletableResultCode.ofSuccess(); + } + + @Override + public CompletableResultCode shutdown() { + return CompletableResultCode.ofSuccess(); + } + }; + testSdkTracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .build(); + GlobalOpenTelemetry.set( + OpenTelemetrySdk.builder() + .setTracerProvider(testSdkTracerProvider) + .build()); + } + + private boolean hasExportedSpanNamed(String name) { + return exportedSpans.stream().anyMatch(s -> name.equals(s.getName())); + } + + @Test + public void testOzoneTracingOnUsesStaticTracerNotGlobalOpenTelemetry() throws Exception { + registerGlobalTestSdk(); + TracingUtil.reconfigureTracing("client", clientTracingConfig(true, true)); + + TracingUtil.executeInNewSpan("ozone-client-span", () -> { + assertThat(Span.current().getSpanContext().isValid()).isTrue(); + }); + + assertThat(hasExportedSpanNamed("ozone-client-span")) + .withFailMessage("When Ozone client tracing is on, spans must use the Ozone SDK tracer, " + + "not GlobalOpenTelemetry, so the test global exporter must not see them.") + .isFalse(); + } + + @Test + public void testOzoneTracingOffWithAppAwareAndParentUsesGlobalTracer() throws Exception { + registerGlobalTestSdk(); + TracingUtil.reconfigureTracing("client", clientTracingConfig(false, true)); + + Tracer appTracer = GlobalOpenTelemetry.get().getTracer("test-app"); + Span parent = appTracer.spanBuilder("parent").startSpan(); + try (Scope ignored = parent.makeCurrent()) { + TracingUtil.executeInNewSpan("child-from-global", () -> + assertThat(Span.current().getSpanContext().isValid()).isTrue()); + } finally { + parent.end(); + } + testSdkTracerProvider.forceFlush(); + + assertThat(hasExportedSpanNamed("child-from-global")) + .withFailMessage("With Ozone tracing off, client application-aware on, and a valid parent span, " + + "the child must be created with GlobalOpenTelemetry.") + .isTrue(); + + assertThat(exportedSpans) + .filteredOn(s -> "child-from-global".equals(s.getName())) + .singleElement() + .extracting(s -> s.getInstrumentationScopeInfo().getName()) + .isEqualTo(OZONE_CLIENT_TRACER_SCOPE); + } + + @Test + public void testOzoneTracingOffApplicationAwareFalseSkipsSpan() throws Exception { + registerGlobalTestSdk(); + TracingUtil.reconfigureTracing("client", clientTracingConfig(false, false)); + + Tracer appTracer = GlobalOpenTelemetry.get().getTracer("test-app"); + Span parent = appTracer.spanBuilder("parent").startSpan(); + try (Scope ignored = parent.makeCurrent()) { + TracingUtil.executeInNewSpan("should-not-exist", () -> { + assertEquals( + parent.getSpanContext(), + Span.current().getSpanContext(), + "When application-aware is false, TracingUtil should not create a child span; " + + "the active span should remain the application parent."); + }); + } finally { + parent.end(); + } + testSdkTracerProvider.forceFlush(); + + assertThat(hasExportedSpanNamed("should-not-exist")).isFalse(); + } + + @Test + public void testOzoneTracingOffNoParentContextSkipsSpan() throws Exception { + registerGlobalTestSdk(); + TracingUtil.reconfigureTracing("client", clientTracingConfig(false, true)); + + TracingUtil.executeInNewSpan("no-parent-bypass", () -> + assertThat(Span.current().getSpanContext().isValid()) + .withFailMessage("Without a valid parent, span creation should be bypassed.") + .isFalse()); + testSdkTracerProvider.forceFlush(); + + assertThat(hasExportedSpanNamed("no-parent-bypass")).isFalse(); + } + + @Test + public void testCreateActivatedSpanBypassWhenConditionsNotMet() throws Exception { + registerGlobalTestSdk(); + TracingUtil.reconfigureTracing("client", clientTracingConfig(false, false)); + + try (TracingUtil.TraceCloseable closeable = + TracingUtil.createActivatedSpan("activated-bypass")) { + assertNotNull(closeable); + } + if (testSdkTracerProvider != null) { + testSdkTracerProvider.forceFlush(); + } + assertThat(hasExportedSpanNamed("activated-bypass")).isFalse(); + } +} From a638fb390b5b035627989131ea60490e10161bbf Mon Sep 17 00:00:00 2001 From: sravani-revuri Date: Wed, 15 Apr 2026 14:17:31 +0530 Subject: [PATCH 2/3] fixed spacing and added parent check to test --- .../java/org/apache/hadoop/hdds/tracing/TracingUtil.java | 4 ++-- .../tracing/TestTracingUtilClientApplicationAware.java | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java index ad864ee3750..28c4ed4920e 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java @@ -289,7 +289,7 @@ static Map parseSpanSamplingConfig(String configStr) { * If a parent span exists in the current context, this becomes a child span. */ public static void executeInNewSpan(String spanName, - CheckedRunnable runnable) throws E { + CheckedRunnable runnable) throws E { if (shouldByPassSpanCreation()) { runnable.run(); return; @@ -299,7 +299,7 @@ public static void executeInNewSpan(String spanName, } public static R executeInNewSpan(String spanName, - CheckedSupplier supplier) throws E { + CheckedSupplier supplier) throws E { if (shouldByPassSpanCreation()) { return supplier.get(); } diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingUtilClientApplicationAware.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingUtilClientApplicationAware.java index a853b5f295a..012c0ab2566 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingUtilClientApplicationAware.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingUtilClientApplicationAware.java @@ -133,6 +133,7 @@ public void testOzoneTracingOffWithAppAwareAndParentUsesGlobalTracer() throws Ex Tracer appTracer = GlobalOpenTelemetry.get().getTracer("test-app"); Span parent = appTracer.spanBuilder("parent").startSpan(); + final String parentSpanId = parent.getSpanContext().getSpanId(); try (Scope ignored = parent.makeCurrent()) { TracingUtil.executeInNewSpan("child-from-global", () -> assertThat(Span.current().getSpanContext().isValid()).isTrue()); @@ -146,11 +147,19 @@ public void testOzoneTracingOffWithAppAwareAndParentUsesGlobalTracer() throws Ex + "the child must be created with GlobalOpenTelemetry.") .isTrue(); + // Child span must use the Ozone client scope on GlobalOpenTelemetry (not the test-app parent scope). assertThat(exportedSpans) .filteredOn(s -> "child-from-global".equals(s.getName())) .singleElement() .extracting(s -> s.getInstrumentationScopeInfo().getName()) .isEqualTo(OZONE_CLIENT_TRACER_SCOPE); + + // Child must link to the application parent span in the same trace. + assertThat(exportedSpans) + .filteredOn(s -> "child-from-global".equals(s.getName())) + .singleElement() + .extracting(SpanData::getParentSpanId) + .isEqualTo(parentSpanId); } @Test From 38ef61e969fbb5594dd362e01b11c62c5797d769 Mon Sep 17 00:00:00 2001 From: sravani-revuri Date: Tue, 21 Apr 2026 14:42:22 +0530 Subject: [PATCH 3/3] added more detailed description --- .../apache/hadoop/hdds/tracing/TracingConfig.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingConfig.java index e2e394352f0..39c932e9d9d 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingConfig.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingConfig.java @@ -45,7 +45,9 @@ public class TracingConfig extends ReconfigurableConfig { type = ConfigType.BOOLEAN, reconfigurable = true, tags = { ConfigTag.OZONE, ConfigTag.HDDS }, - description = "If true, tracing is initialized and spans may be exported (subject to sampling)." + description = "If true, Ozone initializes its own tracer and exports spans (subject to sampling). " + + "If false, the Ozone client does not use that tracer; optional child spans under an " + + "application trace are controlled by ozone.tracing.client.application-aware." ) private boolean tracingEnabled; @@ -85,9 +87,11 @@ public class TracingConfig extends ReconfigurableConfig { type = ConfigType.BOOLEAN, reconfigurable = true, tags = { ConfigTag.OZONE, ConfigTag.HDDS }, - description = "When true and Ozone's own tracing is disabled, the client may still " - + "emit child spans using the application's OpenTelemetry (GlobalOpenTelemetry) " - + "when an active span exists in Context." + description = "This is used when ozone.tracing.enabled is false. If true and " + + "an active span exists in the current OpenTelemetry Context (application " + + "GlobalOpenTelemetry), the client may create child spans under that trace. If no " + + "active span exists, no spans are created. If false, no spans are created regardless " + + "of application context." ) private boolean clientApplicationAware;