From 54c578635be3450e119a326278ee85cf8d1584bf Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Thu, 26 Mar 2026 19:17:11 +0100 Subject: [PATCH 01/15] feat(profiler): add parentSpanId and getCurrentTicks to profiling interface --- .../datadog/trace/core/DDSpanContext.java | 5 ++++ .../instrumentation/api/ProfilerContext.java | 5 ++++ .../api/ProfilingContextIntegration.java | 24 +++++++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java index a17d7684a71..47a177dff07 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java @@ -372,6 +372,11 @@ public long getRootSpanId() { return getRootSpanContextOrThis().spanId; } + @Override + public long getParentSpanId() { + return parentId; + } + @Override public int getEncodedOperationName() { return encodedOperationName; diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilerContext.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilerContext.java index 2fc52a0a073..ddf27ded238 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilerContext.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilerContext.java @@ -9,6 +9,11 @@ public interface ProfilerContext { */ long getRootSpanId(); + /** + * @return the span id of the parent span, or 0 if this is the root + */ + long getParentSpanId(); + int getEncodedOperationName(); CharSequence getOperationName(); diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java index 4accced983a..e5d671096c9 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java @@ -34,6 +34,30 @@ default int encodeResourceName(CharSequence constant) { return 0; } + /** Returns the current TSC tick count for the calling thread. */ + default long getCurrentTicks() { + return 0L; + } + + /** + * Emits a TaskBlock event covering a blocking interval on the current thread. + * + * @param startTicks TSC tick at block entry (obtained from {@link #getCurrentTicks()}) + * @param spanId the span ID active when blocking began + * @param rootSpanId the local root span ID active when blocking began + * @param blocker identity hash code of the blocking object, or 0 if none + * @param unblockingSpanId the span ID of the thread that unblocked this thread, or 0 if unknown + */ + default void recordTaskBlock( + long startTicks, long spanId, long rootSpanId, long blocker, long unblockingSpanId) {} + + /** + * Emits a SpanNode event when a span finishes, recording its identity, timing, and encoding. + * + * @param span the finished span + */ + default void onSpanFinished(AgentSpan span) {} + String name(); final class NoOp implements ProfilingContextIntegration { From 8ed9d5791e1ea3b39d2123832f8d97f8cd4b7c9f Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Thu, 26 Mar 2026 19:21:19 +0100 Subject: [PATCH 02/15] feat(profiler): wire parentSpanId, startTicks, SpanNode and TaskBlock to DatadogProfiler --- .../profiling-ddprof/build.gradle | 1 + .../profiling/ddprof/DatadogProfiler.java | 38 ++++- .../ddprof/DatadogProfilingIntegration.java | 49 +++++- .../ddprof/DatadogProfilerSpanNodeTest.java | 157 ++++++++++++++++++ 4 files changed, 237 insertions(+), 8 deletions(-) create mode 100644 dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerSpanNodeTest.java diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/build.gradle b/dd-java-agent/agent-profiling/profiling-ddprof/build.gradle index 2fa61e6d34b..52ec99874ef 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/build.gradle +++ b/dd-java-agent/agent-profiling/profiling-ddprof/build.gradle @@ -38,6 +38,7 @@ dependencies { testImplementation libs.bundles.jmc testImplementation libs.bundles.junit5 + testImplementation libs.bundles.mockito } diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java index 3d9325168e7..616f7694402 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java @@ -332,8 +332,10 @@ String cmdStartProfiling(Path file) throws IllegalStateException { return cmdString; } - public void recordTraceRoot(long rootSpanId, String endpoint, String operation) { - if (!profiler.recordTraceRoot(rootSpanId, endpoint, operation, MAX_NUM_ENDPOINTS)) { + public void recordTraceRoot( + long rootSpanId, long parentSpanId, long startTicks, String endpoint, String operation) { + if (!profiler.recordTraceRoot( + rootSpanId, parentSpanId, startTicks, endpoint, operation, MAX_NUM_ENDPOINTS)) { log.debug( "Endpoint event not written because more than {} distinct endpoints have been encountered." + " This avoids excessive memory overhead.", @@ -341,6 +343,10 @@ public void recordTraceRoot(long rootSpanId, String endpoint, String operation) } } + public long getCurrentTicks() { + return profiler.getCurrentTicks(); + } + public int operationNameOffset() { return offsetOf(OPERATION); } @@ -455,6 +461,34 @@ boolean shouldRecordQueueTimeEvent(long startMillis) { return System.currentTimeMillis() - startMillis >= queueTimeThresholdMillis; } + void recordTaskBlockEvent( + long startTicks, long spanId, long rootSpanId, long blocker, long unblockingSpanId) { + if (profiler != null) { + long endTicks = profiler.getCurrentTicks(); + profiler.recordTaskBlock(startTicks, endTicks, spanId, rootSpanId, blocker, unblockingSpanId); + } + } + + public void recordSpanNodeEvent( + long spanId, + long parentSpanId, + long rootSpanId, + long startNanos, + long durationNanos, + int encodedOperation, + int encodedResource) { + if (profiler != null) { + profiler.recordSpanNode( + spanId, + parentSpanId, + rootSpanId, + startNanos, + durationNanos, + encodedOperation, + encodedResource); + } + } + void recordQueueTimeEvent( long startTicks, Object task, diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java index f6b09476df4..f4f8d1987ac 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java @@ -93,6 +93,31 @@ public String name() { return "ddprof"; } + @Override + public long getCurrentTicks() { + return DDPROF.getCurrentTicks(); + } + + @Override + public void recordTaskBlock( + long startTicks, long spanId, long rootSpanId, long blocker, long unblockingSpanId) { + DDPROF.recordTaskBlockEvent(startTicks, spanId, rootSpanId, blocker, unblockingSpanId); + } + + @Override + public void onSpanFinished(AgentSpan span) { + if (span == null || !(span.context() instanceof ProfilerContext)) return; + ProfilerContext ctx = (ProfilerContext) span.context(); + DDPROF.recordSpanNodeEvent( + ctx.getSpanId(), + ctx.getParentSpanId(), + ctx.getRootSpanId(), + span.getStartTime(), + span.getDurationNano(), + ctx.getEncodedOperationName(), + ctx.getEncodedResourceName()); + } + public void clearContext() { DDPROF.clearSpanContext(); DDPROF.clearContextValue(SPAN_NAME_INDEX); @@ -115,15 +140,25 @@ public void onRootSpanFinished(AgentSpan rootSpan, EndpointTracker tracker) { CharSequence resourceName = rootSpan.getResourceName(); CharSequence operationName = rootSpan.getOperationName(); if (resourceName != null && operationName != null) { + long startTicks = + (tracker instanceof RootSpanTracker) ? ((RootSpanTracker) tracker).startTicks : 0L; + long parentSpanId = 0L; + if (rootSpan.context() instanceof ProfilerContext) { + parentSpanId = ((ProfilerContext) rootSpan.context()).getParentSpanId(); + } DDPROF.recordTraceRoot( - rootSpan.getSpanId(), resourceName.toString(), operationName.toString()); + rootSpan.getSpanId(), + parentSpanId, + startTicks, + resourceName.toString(), + operationName.toString()); } } } @Override public EndpointTracker onRootSpanStarted(AgentSpan rootSpan) { - return NoOpEndpointTracker.INSTANCE; + return new RootSpanTracker(DDPROF.getCurrentTicks()); } @Override @@ -135,12 +170,14 @@ public Timing start(TimerType type) { } /** - * This implementation is actually stateless, so we don't actually need a tracker object, but - * we'll create a singleton to avoid returning null and risking NPEs elsewhere. + * Captures the TSC tick at root span start so we can emit real duration in the Endpoint event. */ - private static final class NoOpEndpointTracker implements EndpointTracker { + private static final class RootSpanTracker implements EndpointTracker { + final long startTicks; - public static final NoOpEndpointTracker INSTANCE = new NoOpEndpointTracker(); + RootSpanTracker(long startTicks) { + this.startTicks = startTicks; + } @Override public void endpointWritten(AgentSpan span) {} diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerSpanNodeTest.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerSpanNodeTest.java new file mode 100644 index 00000000000..a5cad42552c --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerSpanNodeTest.java @@ -0,0 +1,157 @@ +package com.datadog.profiling.ddprof; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; +import datadog.trace.bootstrap.instrumentation.api.ProfilerContext; +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link DatadogProfilingIntegration#onSpanFinished(AgentSpan)}. + * + *

Because {@link DatadogProfiler} wraps a native library, we verify the filtering logic and + * dispatch path without asserting on the native event itself. Native calls simply must not throw + * (the {@code if (profiler != null)} guard inside {@link DatadogProfiler} protects them on systems + * where the native library is unavailable). + */ +class DatadogProfilerSpanNodeTest { + + /** + * When the span's context does NOT implement {@link ProfilerContext}, {@code onSpanFinished} + * should be a no-op and must not throw. + */ + @Test + void onSpanFinished_nonProfilerContext_isNoOp() { + DatadogProfilingIntegration integration = new DatadogProfilingIntegration(); + AgentSpan span = mock(AgentSpan.class); + AgentSpanContext ctx = mock(AgentSpanContext.class); // plain context, NOT a ProfilerContext + when(span.context()).thenReturn(ctx); + + assertDoesNotThrow(() -> integration.onSpanFinished(span)); + } + + /** + * When the span's context DOES implement {@link ProfilerContext}, {@code onSpanFinished} extracts + * fields and attempts to emit a SpanNode event. Must not throw regardless of whether the native + * profiler is loaded. + */ + @Test + void onSpanFinished_profilerContext_doesNotThrow() { + DatadogProfilingIntegration integration = new DatadogProfilingIntegration(); + + // Mockito can create a mock that implements multiple interfaces + AgentSpanContext ctx = mock(AgentSpanContext.class, org.mockito.Answers.RETURNS_DEFAULTS); + ProfilerContext profilerCtx = mock(ProfilerContext.class); + + // We need a single object that satisfies both instanceof checks. + // Use a hand-rolled stub instead. + TestContext combinedCtx = new TestContext(42L, 7L, 1L, 3, 5); + + AgentSpan span = mock(AgentSpan.class); + when(span.context()).thenReturn(combinedCtx); + when(span.getStartTime()).thenReturn(1_700_000_000_000_000_000L); + when(span.getDurationNano()).thenReturn(1_000_000L); + + assertDoesNotThrow(() -> integration.onSpanFinished(span)); + } + + /** Null span must not throw (guard at top of onSpanFinished). */ + @Test + void onSpanFinished_nullSpan_doesNotThrow() { + DatadogProfilingIntegration integration = new DatadogProfilingIntegration(); + assertDoesNotThrow(() -> integration.onSpanFinished(null)); + } + + // --------------------------------------------------------------------------- + // Stub: a single object that satisfies both AgentSpanContext and ProfilerContext + // --------------------------------------------------------------------------- + + private static final class TestContext implements AgentSpanContext, ProfilerContext { + + private final long spanId; + private final long parentSpanId; + private final long rootSpanId; + private final int encodedOp; + private final int encodedResource; + + TestContext( + long spanId, long parentSpanId, long rootSpanId, int encodedOp, int encodedResource) { + this.spanId = spanId; + this.parentSpanId = parentSpanId; + this.rootSpanId = rootSpanId; + this.encodedOp = encodedOp; + this.encodedResource = encodedResource; + } + + // ProfilerContext + @Override + public long getSpanId() { + return spanId; + } + + @Override + public long getParentSpanId() { + return parentSpanId; + } + + @Override + public long getRootSpanId() { + return rootSpanId; + } + + @Override + public int getEncodedOperationName() { + return encodedOp; + } + + @Override + public CharSequence getOperationName() { + return "test-op"; + } + + @Override + public int getEncodedResourceName() { + return encodedResource; + } + + @Override + public CharSequence getResourceName() { + return "test-resource"; + } + + // AgentSpanContext + @Override + public datadog.trace.api.DDTraceId getTraceId() { + return datadog.trace.api.DDTraceId.ZERO; + } + + @Override + public datadog.trace.bootstrap.instrumentation.api.AgentTraceCollector getTraceCollector() { + return datadog.trace.bootstrap.instrumentation.api.AgentTracer.NoopAgentTraceCollector + .INSTANCE; + } + + @Override + public int getSamplingPriority() { + return datadog.trace.api.sampling.PrioritySampling.UNSET; + } + + @Override + public Iterable> baggageItems() { + return java.util.Collections.emptyList(); + } + + @Override + public datadog.trace.api.datastreams.PathwayContext getPathwayContext() { + return null; + } + + @Override + public boolean isRemote() { + return false; + } + } +} From 93337846d8b8e0f8f62c3efef138d8632c4a763d Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Thu, 26 Mar 2026 19:24:30 +0100 Subject: [PATCH 03/15] feat(profiler): emit SpanNode events for all spans on trace completion --- dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java b/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java index 0b835c77da3..d292352434d 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java @@ -1255,6 +1255,9 @@ void write(final SpanList trace) { if (null != rootSpan) { onRootSpanFinished(rootSpan, rootSpan.getEndpointTracker()); } + for (DDSpan span : writtenTrace) { + profilingContextIntegration.onSpanFinished(span); + } } private List interceptCompleteTrace(SpanList originalTrace) { From b5306f176c0c81a0873c602085ffb59761ee0e9b Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Fri, 27 Mar 2026 11:24:32 +0100 Subject: [PATCH 04/15] feat(profiler): add LockSupport.park/unpark instrumentation for causal DAG edges --- .../lock-support-profiling/build.gradle | 11 ++ .../LockSupportProfilingInstrumentation.java | 116 +++++++++++++ ...ckSupportProfilingInstrumentationTest.java | 153 ++++++++++++++++++ settings.gradle.kts | 1 + 4 files changed, 281 insertions(+) create mode 100644 dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/build.gradle create mode 100644 dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java create mode 100644 dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/test/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentationTest.java diff --git a/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/build.gradle b/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/build.gradle new file mode 100644 index 00000000000..9935e818b49 --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/build.gradle @@ -0,0 +1,11 @@ +apply from: "$rootDir/gradle/java.gradle" + +muzzle { + pass { + coreJdk() + } +} + +dependencies { + testImplementation libs.bundles.junit5 +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java b/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java new file mode 100644 index 00000000000..3c4cb2f5c2a --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java @@ -0,0 +1,116 @@ +package datadog.trace.instrumentation.locksupport; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.isDeclaredBy; +import static net.bytebuddy.matcher.ElementMatchers.isMethod; +import static net.bytebuddy.matcher.ElementMatchers.isStatic; +import static net.bytebuddy.matcher.ElementMatchers.nameStartsWith; + +import com.google.auto.service.AutoService; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.ProfilerContext; +import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration; +import java.util.concurrent.ConcurrentHashMap; +import net.bytebuddy.asm.Advice; +import net.bytebuddy.matcher.ElementMatchers; + +/** + * Instruments {@link java.util.concurrent.locks.LockSupport#park} variants to emit {@code + * datadog.TaskBlock} JFR events. These events record the span, root-span, and duration of every + * blocking interval, enabling critical-path analysis across async handoffs. + * + *

Also instruments {@link java.util.concurrent.locks.LockSupport#unpark} to capture the span ID + * of the unblocking thread, which is then recorded in the TaskBlock event. + * + *

Only fires when a Datadog span is active on the calling thread, so there is no overhead on + * threads that are not part of a traced request. + */ +@AutoService(InstrumenterModule.class) +public class LockSupportProfilingInstrumentation extends InstrumenterModule.Profiling + implements Instrumenter.ForKnownTypes, Instrumenter.HasMethodAdvice { + + public LockSupportProfilingInstrumentation() { + super("lock-support-profiling"); + } + + @Override + public String[] knownMatchingTypes() { + return new String[] {"java.util.concurrent.locks.LockSupport"}; + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + isMethod() + .and(isStatic()) + .and(nameStartsWith("park")) + .and(isDeclaredBy(named("java.util.concurrent.locks.LockSupport"))), + getClass().getName() + "$ParkAdvice"); + transformer.applyAdvice( + isMethod() + .and(isStatic()) + .and(ElementMatchers.named("unpark")) + .and(isDeclaredBy(named("java.util.concurrent.locks.LockSupport"))), + getClass().getName() + "$UnparkAdvice"); + } + + /** Holds shared state accessible from both {@link ParkAdvice} and {@link UnparkAdvice}. */ + public static final class State { + /** Maps target thread to the span ID of the thread that called {@code unpark()} on it. */ + public static final ConcurrentHashMap UNPARKING_SPAN = new ConcurrentHashMap<>(); + } + + public static final class ParkAdvice { + + @Advice.OnMethodEnter(suppress = Throwable.class) + public static long[] before(@Advice.Argument(value = 0, optional = true) Object blocker) { + AgentSpan span = AgentTracer.activeSpan(); + if (!(span instanceof ProfilerContext)) { + return null; + } + ProfilerContext ctx = (ProfilerContext) span; + ProfilingContextIntegration profiling = AgentTracer.get().getProfilingContext(); + long startTicks = profiling.getCurrentTicks(); + if (startTicks == 0L) { + // profiler not active + return null; + } + long blockerHash = blocker != null ? System.identityHashCode(blocker) : 0L; + return new long[] {startTicks, ctx.getSpanId(), ctx.getRootSpanId(), blockerHash}; + } + + @Advice.OnMethodExit(suppress = Throwable.class, onThrowable = Throwable.class) + public static void after(@Advice.Enter long[] state) { + if (state == null) { + return; + } + Long unblockingSpanId = State.UNPARKING_SPAN.remove(Thread.currentThread()); + AgentTracer.get() + .getProfilingContext() + .recordTaskBlock( + state[0], + state[1], + state[2], + state[3], + unblockingSpanId != null ? unblockingSpanId : 0L); + } + } + + public static final class UnparkAdvice { + + @Advice.OnMethodEnter(suppress = Throwable.class) + public static void before(@Advice.Argument(0) Thread thread) { + if (thread == null) { + return; + } + AgentSpan span = AgentTracer.activeSpan(); + if (!(span instanceof ProfilerContext)) { + return; + } + State.UNPARKING_SPAN.put(thread, ((ProfilerContext) span).getSpanId()); + } + } +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/test/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentationTest.java b/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/test/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentationTest.java new file mode 100644 index 00000000000..06967b29667 --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/test/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentationTest.java @@ -0,0 +1,153 @@ +package datadog.trace.instrumentation.locksupport; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.instrumentation.locksupport.LockSupportProfilingInstrumentation.State; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link LockSupportProfilingInstrumentation}. + * + *

These tests exercise the {@link State} map directly, verifying the mechanism used to + * communicate the unblocking span ID from {@code UnparkAdvice} to {@code ParkAdvice}. + */ +class LockSupportProfilingInstrumentationTest { + + @BeforeEach + void clearState() { + State.UNPARKING_SPAN.clear(); + } + + @AfterEach + void cleanupState() { + State.UNPARKING_SPAN.clear(); + } + + // ------------------------------------------------------------------------- + // State map — basic contract + // ------------------------------------------------------------------------- + + @Test + void state_put_and_remove() { + Thread t = Thread.currentThread(); + long spanId = 12345L; + + State.UNPARKING_SPAN.put(t, spanId); + Long retrieved = State.UNPARKING_SPAN.remove(t); + + assertNotNull(retrieved); + assertEquals(spanId, (long) retrieved); + // After removal the entry should be gone + assertNull(State.UNPARKING_SPAN.get(t)); + } + + @Test + void state_remove_returns_null_when_absent() { + Thread t = new Thread(() -> {}); + assertNull(State.UNPARKING_SPAN.remove(t)); + } + + @Test + void state_is_initially_empty() { + assertTrue(State.UNPARKING_SPAN.isEmpty()); + } + + // ------------------------------------------------------------------------- + // Multithreaded: unpark thread populates map, parked thread reads it + // ------------------------------------------------------------------------- + + /** + * Simulates the UnparkAdvice → ParkAdvice handoff: + * + *

    + *
  1. Thread A (the "parked" thread) blocks on a latch. + *
  2. Thread B (the "unparking" thread) places its span ID in {@code State.UNPARKING_SPAN} for + * Thread A and then releases the latch. + *
  3. Thread A wakes up, reads and removes the span ID from the map. + *
+ */ + @Test + void unparking_spanId_is_visible_to_parked_thread() throws InterruptedException { + long unparkingSpanId = 99887766L; + + CountDownLatch ready = new CountDownLatch(1); + CountDownLatch go = new CountDownLatch(1); + AtomicLong capturedSpanId = new AtomicLong(-1L); + AtomicReference parkedThreadRef = new AtomicReference<>(); + + Thread parkedThread = + new Thread( + () -> { + parkedThreadRef.set(Thread.currentThread()); + ready.countDown(); + try { + go.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + // Simulate what ParkAdvice.after does: read and remove unblocking span id + Long unblockingId = State.UNPARKING_SPAN.remove(Thread.currentThread()); + capturedSpanId.set(unblockingId != null ? unblockingId : 0L); + }); + + parkedThread.start(); + ready.await(); // wait for parked thread to register itself + + // Simulate what UnparkAdvice.before does: record unparking span id + State.UNPARKING_SPAN.put(parkedThread, unparkingSpanId); + go.countDown(); // unblock parked thread + + parkedThread.join(2_000); + assertFalse(parkedThread.isAlive(), "Test thread did not finish in time"); + assertEquals( + unparkingSpanId, + capturedSpanId.get(), + "Parked thread should have read the unblocking span id placed by unparking thread"); + } + + /** + * Verifies that if no entry exists for the parked thread (i.e. the thread was unblocked by a + * non-traced thread), the {@code remove} returns {@code null} and the code falls back to 0. + */ + @Test + void no_unparking_entry_yields_zero() throws InterruptedException { + AtomicLong capturedSpanId = new AtomicLong(-1L); + + Thread parkedThread = + new Thread( + () -> { + Long unblockingId = State.UNPARKING_SPAN.remove(Thread.currentThread()); + capturedSpanId.set(unblockingId != null ? unblockingId : 0L); + }); + parkedThread.start(); + parkedThread.join(2_000); + + assertEquals( + 0L, capturedSpanId.get(), "Should fall back to 0 when no unparking span id is recorded"); + } + + // ------------------------------------------------------------------------- + // ParkAdvice.after — null state is a no-op + // ------------------------------------------------------------------------- + + /** + * When {@code ParkAdvice.before} returns {@code null} (profiler not active or no active span), + * {@code ParkAdvice.after} must be a no-op and must not throw. + */ + @Test + void parkAdvice_after_null_state_isNoOp() { + // Should not throw and should not touch State.UNPARKING_SPAN + LockSupportProfilingInstrumentation.ParkAdvice.after(null); + assertTrue(State.UNPARKING_SPAN.isEmpty()); + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index dbe66b33670..45918440adb 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -325,6 +325,7 @@ include( ":dd-java-agent:instrumentation:datadog:dynamic-instrumentation:span-origin", ":dd-java-agent:instrumentation:datadog:profiling:enable-wallclock-profiling", ":dd-java-agent:instrumentation:datadog:profiling:exception-profiling", + ":dd-java-agent:instrumentation:datadog:profiling:lock-support-profiling", ":dd-java-agent:instrumentation:datadog:tracing:trace-annotation", ":dd-java-agent:instrumentation:datanucleus-4.0.5", ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-3.0", From 5c5b32058a6cf911e18c999cce6f3332ce385ec0 Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Wed, 8 Apr 2026 16:36:31 +0200 Subject: [PATCH 05/15] feat(profiler): wire submittingSpanId into QueueTime events --- .../profiling/ddprof/DatadogProfiler.java | 16 ++++++++++++---- .../ddprof/DatadogProfilingIntegration.java | 6 +++++- .../profiling/ddprof/QueueTimeTracker.java | 7 +++++-- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java index 616f7694402..ead4967f8a7 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java @@ -453,8 +453,8 @@ public void recordSetting(String name, String value, String unit) { profiler.recordSetting(name, value, unit); } - public QueueTimeTracker newQueueTimeTracker() { - return new QueueTimeTracker(this, profiler.getCurrentTicks()); + public QueueTimeTracker newQueueTimeTracker(long submittingSpanId) { + return new QueueTimeTracker(this, profiler.getCurrentTicks(), submittingSpanId); } boolean shouldRecordQueueTimeEvent(long startMillis) { @@ -495,7 +495,8 @@ void recordQueueTimeEvent( Class scheduler, Class queueType, int queueLength, - Thread origin) { + Thread origin, + long submittingSpanId) { if (profiler != null) { // note: because this type traversal can update secondary_super_cache (see JDK-8180450) // we avoid doing this unless we are absolutely certain we will record the event @@ -503,7 +504,14 @@ void recordQueueTimeEvent( if (taskType != null) { long endTicks = profiler.getCurrentTicks(); profiler.recordQueueTime( - startTicks, endTicks, taskType, scheduler, queueType, queueLength, origin); + startTicks, + endTicks, + taskType, + scheduler, + queueType, + queueLength, + origin, + submittingSpanId); } } } diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java index f4f8d1987ac..08620220154 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java @@ -6,6 +6,7 @@ import datadog.trace.api.profiling.ProfilingScope; import datadog.trace.api.profiling.Timing; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; import datadog.trace.bootstrap.instrumentation.api.ProfilerContext; import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration; @@ -164,7 +165,10 @@ public EndpointTracker onRootSpanStarted(AgentSpan rootSpan) { @Override public Timing start(TimerType type) { if (IS_PROFILING_QUEUEING_TIME_ENABLED && type == TimerType.QUEUEING) { - return DDPROF.newQueueTimeTracker(); + AgentSpan span = AgentTracer.activeSpan(); + long submittingSpanId = + (span instanceof ProfilerContext) ? ((ProfilerContext) span).getSpanId() : 0L; + return DDPROF.newQueueTimeTracker(submittingSpanId); } return Timing.NoOp.INSTANCE; } diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/QueueTimeTracker.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/QueueTimeTracker.java index e76a3dd5561..23106402a2b 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/QueueTimeTracker.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/QueueTimeTracker.java @@ -9,6 +9,7 @@ public class QueueTimeTracker implements QueueTiming { private final Thread origin; private final long startTicks; private final long startMillis; + private final long submittingSpanId; private WeakReference weakTask; // FIXME this can be eliminated by altering the instrumentation // since it is known when the item is polled from the queue @@ -16,11 +17,12 @@ public class QueueTimeTracker implements QueueTiming { private Class queue; private int queueLength; - public QueueTimeTracker(DatadogProfiler profiler, long startTicks) { + public QueueTimeTracker(DatadogProfiler profiler, long startTicks, long submittingSpanId) { this.profiler = profiler; this.origin = Thread.currentThread(); this.startTicks = startTicks; this.startMillis = System.currentTimeMillis(); + this.submittingSpanId = submittingSpanId; } @Override @@ -49,7 +51,8 @@ public void report() { Object task = this.weakTask.get(); if (task != null) { // indirection reduces shallow size of the tracker instance - profiler.recordQueueTimeEvent(startTicks, task, scheduler, queue, queueLength, origin); + profiler.recordQueueTimeEvent( + startTicks, task, scheduler, queue, queueLength, origin, submittingSpanId); } } From ebd23a2dba7fab1078e52afde2c6636471c183e5 Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Fri, 17 Apr 2026 13:37:54 +0200 Subject: [PATCH 06/15] fix(profiler): use span.context() for ProfilerContext check in QueueTime tracker --- .../profiling/ddprof/DatadogProfilingIntegration.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java index 08620220154..8865a63c2ec 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java @@ -166,8 +166,10 @@ public EndpointTracker onRootSpanStarted(AgentSpan rootSpan) { public Timing start(TimerType type) { if (IS_PROFILING_QUEUEING_TIME_ENABLED && type == TimerType.QUEUEING) { AgentSpan span = AgentTracer.activeSpan(); - long submittingSpanId = - (span instanceof ProfilerContext) ? ((ProfilerContext) span).getSpanId() : 0L; + long submittingSpanId = 0L; + if (span != null && span.context() instanceof ProfilerContext) { + submittingSpanId = ((ProfilerContext) span.context()).getSpanId(); + } return DDPROF.newQueueTimeTracker(submittingSpanId); } return Timing.NoOp.INSTANCE; From bc3445460d1fed6441fb9b25bd8e64489c237e8e Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Fri, 17 Apr 2026 13:42:40 +0200 Subject: [PATCH 07/15] feat(profiler): emit synthetic SpanNode events for ThreadPoolExecutor worker threads --- .../java/concurrent/TPEHelper.java | 38 ++++++++++++++++++- .../ddprof/DatadogProfilingIntegration.java | 34 +++++++++++++++++ .../api/ProfilingContextIntegration.java | 18 +++++++++ 3 files changed, 88 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/TPEHelper.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/TPEHelper.java index 12c4498fe6e..f6e4d249d4a 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/TPEHelper.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/TPEHelper.java @@ -7,6 +7,9 @@ import datadog.trace.api.InstrumenterConfig; import datadog.trace.bootstrap.ContextStore; import datadog.trace.bootstrap.instrumentation.api.AgentScope; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.ProfilerContext; import java.util.Set; import java.util.concurrent.ThreadPoolExecutor; @@ -25,6 +28,8 @@ public final class TPEHelper { private static final Set excludedClasses; // A ThreadLocal to store the Scope between beforeExecute and afterExecute if wrapping is not used private static final ThreadLocal threadLocalScope; + // Stores System.nanoTime() at task activation so onTaskDeactivation can compute duration + private static final ThreadLocal threadLocalActivationNano; private static final ClassValue WRAP = GenericClassValue.of( @@ -42,8 +47,10 @@ public final class TPEHelper { excludedClasses = config.getTraceThreadPoolExecutorsExclude(); if (useWrapping) { threadLocalScope = null; + threadLocalActivationNano = null; } else { threadLocalScope = new ThreadLocal<>(); + threadLocalActivationNano = new ThreadLocal<>(); } } @@ -82,7 +89,18 @@ public static AgentScope startScope(ContextStore contextStore, if (task == null || exclude(RUNNABLE, task)) { return null; } - return AdviceUtils.startTaskScope(contextStore, task); + AgentScope scope = AdviceUtils.startTaskScope(contextStore, task); + if (scope != null && threadLocalActivationNano != null) { + long startNano = System.nanoTime(); + threadLocalActivationNano.set(startNano); + AgentSpan span = scope.span(); + if (span != null && span.context() instanceof ProfilerContext) { + AgentTracer.get() + .getProfilingContext() + .onTaskActivation((ProfilerContext) span.context(), startNano); + } + } + return scope; } public static void setThreadLocalScope(AgentScope scope, Runnable task) { @@ -112,7 +130,23 @@ public static void endScope(AgentScope scope, Runnable task) { if (task == null || exclude(RUNNABLE, task)) { return; } - AdviceUtils.endTaskScope(scope); + try { + if (scope != null && threadLocalActivationNano != null) { + Long startNano = threadLocalActivationNano.get(); + // noinspection ThreadLocalSetWithNull + threadLocalActivationNano.set(null); + if (startNano != null) { + AgentSpan span = scope.span(); + if (span != null && span.context() instanceof ProfilerContext) { + AgentTracer.get() + .getProfilingContext() + .onTaskDeactivation((ProfilerContext) span.context(), startNano); + } + } + } + } finally { + AdviceUtils.endTaskScope(scope); + } } public static void cancelTask(ContextStore contextStore, Runnable task) { diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java index 8865a63c2ec..855d8fbe778 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java @@ -119,6 +119,40 @@ public void onSpanFinished(AgentSpan span) { ctx.getEncodedResourceName()); } + // Calibration offset so System.nanoTime() values can be expressed as epoch nanoseconds. + // Computed once at class load to keep activation/deactivation overhead minimal. + private static final long EPOCH_NANOS_OFFSET = + System.currentTimeMillis() * 1_000_000L - System.nanoTime(); + + @Override + public void onTaskActivation(ProfilerContext profilerContext, long startTicks) { + // startTicks captured by TPEHelper is the authoritative start; nothing to do here. + } + + @Override + public void onTaskDeactivation(ProfilerContext profilerContext, long startTicks) { + if (profilerContext == null) { + return; + } + long endNano = System.nanoTime(); + long startNano = startTicks; // startTicks carries nanoTime at activation (see TPEHelper) + long durationNanos = endNano - startNano; + if (durationNanos <= 0) { + return; + } + long startNanos = startNano + EPOCH_NANOS_OFFSET; + long syntheticSpanId = + profilerContext.getSpanId() ^ ((long) Thread.currentThread().getId() << 32) ^ startNano; + DDPROF.recordSpanNodeEvent( + syntheticSpanId, + profilerContext.getSpanId(), + profilerContext.getRootSpanId(), + startNanos, + durationNanos, + profilerContext.getEncodedOperationName(), + profilerContext.getEncodedResourceName()); + } + public void clearContext() { DDPROF.clearSpanContext(); DDPROF.clearContextValue(SPAN_NAME_INDEX); diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java index e5d671096c9..7da107b0517 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java @@ -58,6 +58,24 @@ default void recordTaskBlock( */ default void onSpanFinished(AgentSpan span) {} + /** + * Called when a span context continuation activates on a worker thread (task execution start). + * Implementations record the start tick so a synthetic SpanNode can be emitted at deactivation. + * + * @param profilerContext the activated span context + * @param startTicks TSC tick at activation (from {@link #getCurrentTicks()}) + */ + default void onTaskActivation(ProfilerContext profilerContext, long startTicks) {} + + /** + * Called when a span context continuation deactivates on a worker thread (task execution end). + * Implementations emit a lightweight synthetic SpanNode covering the worker thread interval. + * + * @param profilerContext the deactivated span context + * @param startTicks TSC tick recorded at {@link #onTaskActivation} + */ + default void onTaskDeactivation(ProfilerContext profilerContext, long startTicks) {} + String name(); final class NoOp implements ProfilingContextIntegration { From 46ebbf27c2d63d82d36094d5006eb5127d38b9a0 Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Fri, 17 Apr 2026 14:43:38 +0200 Subject: [PATCH 08/15] feat(profiler): emit synthetic SpanNode events for lambda tasks via Wrapper --- .../java/concurrent/Wrapper.java | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/Wrapper.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/Wrapper.java index 5945ac4742f..940dcc8c8bd 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/Wrapper.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/Wrapper.java @@ -6,6 +6,9 @@ import static datadog.trace.bootstrap.instrumentation.java.concurrent.ExcludeFilter.exclude; import datadog.trace.bootstrap.instrumentation.api.AgentScope; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.ProfilerContext; import java.util.concurrent.RunnableFuture; public class Wrapper implements Runnable, AutoCloseable { @@ -44,7 +47,23 @@ public Wrapper(T delegate, AgentScope.Continuation continuation) { @Override public void run() { try (AgentScope scope = activate()) { - delegate.run(); + long startNano = 0L; + ProfilerContext profilerCtx = null; + if (scope != null) { + AgentSpan span = scope.span(); + if (span != null && span.context() instanceof ProfilerContext) { + profilerCtx = (ProfilerContext) span.context(); + startNano = System.nanoTime(); + AgentTracer.get().getProfilingContext().onTaskActivation(profilerCtx, startNano); + } + } + try { + delegate.run(); + } finally { + if (profilerCtx != null) { + AgentTracer.get().getProfilingContext().onTaskDeactivation(profilerCtx, startNano); + } + } } } From 18c3e3b70fd168a51583032100b37690599031d4 Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Sat, 18 Apr 2026 23:57:39 +0200 Subject: [PATCH 09/15] fix(profiler): use span.context() for ProfilerContext check in LockSupport instrumentation --- .../locksupport/LockSupportProfilingInstrumentation.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java b/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java index 3c4cb2f5c2a..16fb39bdfb1 100644 --- a/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java +++ b/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java @@ -68,10 +68,10 @@ public static final class ParkAdvice { @Advice.OnMethodEnter(suppress = Throwable.class) public static long[] before(@Advice.Argument(value = 0, optional = true) Object blocker) { AgentSpan span = AgentTracer.activeSpan(); - if (!(span instanceof ProfilerContext)) { + if (span == null || !(span.context() instanceof ProfilerContext)) { return null; } - ProfilerContext ctx = (ProfilerContext) span; + ProfilerContext ctx = (ProfilerContext) span.context(); ProfilingContextIntegration profiling = AgentTracer.get().getProfilingContext(); long startTicks = profiling.getCurrentTicks(); if (startTicks == 0L) { @@ -107,10 +107,10 @@ public static void before(@Advice.Argument(0) Thread thread) { return; } AgentSpan span = AgentTracer.activeSpan(); - if (!(span instanceof ProfilerContext)) { + if (span == null || !(span.context() instanceof ProfilerContext)) { return; } - State.UNPARKING_SPAN.put(thread, ((ProfilerContext) span).getSpanId()); + State.UNPARKING_SPAN.put(thread, ((ProfilerContext) span.context()).getSpanId()); } } } From cbe182d8bc5cd4968e22ab665797a507a35518cc Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Sat, 18 Apr 2026 23:58:00 +0200 Subject: [PATCH 10/15] fix(profiler): compute epoch offset dynamically in onTaskDeactivation to avoid clock drift --- .../profiling/ddprof/DatadogProfilingIntegration.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java index 855d8fbe778..3397ce28690 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java @@ -119,11 +119,6 @@ public void onSpanFinished(AgentSpan span) { ctx.getEncodedResourceName()); } - // Calibration offset so System.nanoTime() values can be expressed as epoch nanoseconds. - // Computed once at class load to keep activation/deactivation overhead minimal. - private static final long EPOCH_NANOS_OFFSET = - System.currentTimeMillis() * 1_000_000L - System.nanoTime(); - @Override public void onTaskActivation(ProfilerContext profilerContext, long startTicks) { // startTicks captured by TPEHelper is the authoritative start; nothing to do here. @@ -140,7 +135,11 @@ public void onTaskDeactivation(ProfilerContext profilerContext, long startTicks) if (durationNanos <= 0) { return; } - long startNanos = startNano + EPOCH_NANOS_OFFSET; + // Compute epoch offset fresh each time: avoids cumulative drift between System.nanoTime() + // (monotonic, ignores NTP/wall-clock adjustments) and System.currentTimeMillis() over the + // JVM lifetime. Residual error is bounded by the 1 ms resolution of currentTimeMillis(). + long epochOffset = System.currentTimeMillis() * 1_000_000L - endNano; + long startNanos = startNano + epochOffset; long syntheticSpanId = profilerContext.getSpanId() ^ ((long) Thread.currentThread().getId() << 32) ^ startNano; DDPROF.recordSpanNodeEvent( From b765b79628d9f61eed166b3d0f6c8979eaadf9bb Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Sun, 19 Apr 2026 20:35:20 +0200 Subject: [PATCH 11/15] feat(profiling): add execution thread API to ProfilerContext --- .../bootstrap/instrumentation/api/ProfilerContext.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilerContext.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilerContext.java index ddf27ded238..1c0b75cb80b 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilerContext.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilerContext.java @@ -21,4 +21,14 @@ public interface ProfilerContext { int getEncodedResourceName(); CharSequence getResourceName(); + + /** Java thread ID of the thread that finished this span (captured at span finish time). */ + default long getExecutionThreadId() { + return 0; + } + + /** Name of the thread that finished this span (captured at span finish time). */ + default String getExecutionThreadName() { + return ""; + } } From 812c2a4c6d91dde1e5ce6728ff8ec2b587cbe0d3 Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Sun, 19 Apr 2026 20:37:57 +0200 Subject: [PATCH 12/15] feat(profiling): capture span execution thread at finish time in DDSpanContext --- .../main/java/datadog/trace/core/DDSpan.java | 5 +++++ .../datadog/trace/core/DDSpanContext.java | 21 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java b/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java index 772cbe90866..a96e82e8c9c 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java @@ -160,6 +160,11 @@ private void finishAndAddToTrace(final long durationNano) { wrapper.onSpanFinished(); } this.metrics.onSpanFinished(); + // Capture the execution thread while still on the span's own finishing thread. + // CoreTracer.write() is called later from the event loop; the info captured here is the + // authoritative source for SpanNode thread attribution (see SpanExecutionThreadEvent). + context.captureExecutionThread( + Thread.currentThread().getId(), Thread.currentThread().getName()); TraceCollector.PublishState publishState = context.getTraceCollector().onPublish(this); log.debug("Finished span ({}): {}", publishState, this); } else { diff --git a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java index 47a177dff07..31ea83905cd 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java @@ -160,6 +160,8 @@ public class DDSpanContext private final boolean injectBaggageAsTags; private volatile int encodedOperationName; private volatile int encodedResourceName; + private volatile long executionThreadId = 0; + private volatile String executionThreadName = ""; /** * Metastruct keys are associated to the current span, they will not propagate to the children @@ -387,6 +389,25 @@ public int getEncodedResourceName() { return encodedResourceName; } + public ProfilingContextIntegration getProfilingContextIntegration() { + return profilingContextIntegration; + } + + void captureExecutionThread(long threadId, String threadName) { + this.executionThreadId = threadId; + this.executionThreadName = threadName; + } + + @Override + public long getExecutionThreadId() { + return executionThreadId; + } + + @Override + public String getExecutionThreadName() { + return executionThreadName; + } + public String getServiceName() { return serviceName; } From 2c670d1487228ce7aa37a610d93dfaa121f335aa Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Sun, 19 Apr 2026 20:40:11 +0200 Subject: [PATCH 13/15] feat(profiling): emit SpanExecutionThread JFR event on span finish --- .../ddprof/DatadogProfilingIntegration.java | 12 +++++++ .../ddprof/SpanExecutionThreadEvent.java | 33 +++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/SpanExecutionThreadEvent.java diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java index 3397ce28690..c3cde56c1f7 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java @@ -117,6 +117,18 @@ public void onSpanFinished(AgentSpan span) { span.getDurationNano(), ctx.getEncodedOperationName(), ctx.getEncodedResourceName()); + // Emit the actual execution thread captured in finishAndAddToTrace() so the backend can + // correctly attribute each span to the thread that ran it, rather than the event loop thread + // that calls CoreTracer.write() and commits the SpanNode event above. + long executionThreadId = ctx.getExecutionThreadId(); + String executionThreadName = ctx.getExecutionThreadName(); + if (executionThreadId > 0 && executionThreadName != null && !executionThreadName.isEmpty()) { + SpanExecutionThreadEvent event = new SpanExecutionThreadEvent(); + event.spanId = ctx.getSpanId(); + event.executionThreadId = executionThreadId; + event.executionThreadName = executionThreadName; + event.commit(); + } } @Override diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/SpanExecutionThreadEvent.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/SpanExecutionThreadEvent.java new file mode 100644 index 00000000000..a3f066bc282 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/SpanExecutionThreadEvent.java @@ -0,0 +1,33 @@ +package com.datadog.profiling.ddprof; + +import jdk.jfr.Category; +import jdk.jfr.Event; +import jdk.jfr.Label; +import jdk.jfr.Name; +import jdk.jfr.StackTrace; + +/** + * Pure-Java JFR event that records the actual execution thread for each span. Emitted from {@code + * DatadogProfilingIntegration.onSpanFinished()} using the thread information captured in {@code + * DDSpan.finishAndAddToTrace()} — on the span's own finishing thread, not from the event loop that + * calls {@code CoreTracer.write()}. + * + *

The profiling backend ({@code CausalDagExtractor}) reads this event to override the incorrect + * {@code EVENT_THREAD} on {@code datadog.SpanNode} events (which are emitted at trace completion + * from the event loop thread, causing all DAG nodes to appear as event-loop threads). + */ +@Name("datadog.SpanExecutionThread") +@Label("Span Execution Thread") +@Category("Datadog") +@StackTrace(false) +class SpanExecutionThreadEvent extends Event { + + @Label("Span ID") + long spanId; + + @Label("Execution Thread ID") + long executionThreadId; + + @Label("Execution Thread Name") + String executionThreadName; +} From d715a9b09920fc03fca5aea7f544e6567073f972 Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Tue, 21 Apr 2026 02:19:16 +0200 Subject: [PATCH 14/15] fix(profiling): capture execution thread in phasedFinish() --- .../main/java/datadog/trace/core/DDSpan.java | 6 +++++ .../datadog/trace/core/DDSpanTest.groovy | 23 +++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java b/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java index a96e82e8c9c..495e1ade63c 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java @@ -260,6 +260,12 @@ public final boolean phasedFinish() { } // Flip the negative bit of the result to allow verifying that publish() is only called once. if (DURATION_NANO_UPDATER.compareAndSet(this, 0, Math.max(1, durationNano) | Long.MIN_VALUE)) { + // Mirror finishAndAddToTrace(): capture the execution thread while still on the span's + // own finishing thread so that SpanExecutionThreadEvent is emitted correctly. + // Without this, phasedFinish() spans (e.g. Netty HTTP-client) have executionThreadId=0 + // and fall back to the event-loop thread that calls CoreTracer.write(). + context.captureExecutionThread( + Thread.currentThread().getId(), Thread.currentThread().getName()); log.debug("Finished span (PHASED): {}", this); return true; } else { diff --git a/dd-trace-core/src/test/groovy/datadog/trace/core/DDSpanTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/core/DDSpanTest.groovy index e7882432d9c..058e4963a1a 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/core/DDSpanTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/core/DDSpanTest.groovy @@ -188,6 +188,29 @@ class DDSpanTest extends DDCoreSpecification { writer.size() == 1 } + def "phasedFinish captures execution thread for SpanExecutionThread attribution"() { + // Regression test: phasedFinish() was missing a captureExecutionThread() call, so spans + // finished via phasedFinish()+publish() had executionThreadId=0 and no SpanExecutionThread + // event was emitted. They fell back to the event-loop thread (wrong attribution). + setup: + def span = tracer.buildSpan("test").start() + + when: + def currentThreadId = Thread.currentThread().id + def currentThreadName = Thread.currentThread().name + span.phasedFinish() + + then: "execution thread is captured on the finishing thread" + span.context().executionThreadId == currentThreadId + span.context().executionThreadName == currentThreadName + // Verify the guard in DatadogProfilingIntegration.onSpanFinished() would pass: + span.context().executionThreadId > 0 + span.context().executionThreadName != null && !span.context().executionThreadName.isEmpty() + + cleanup: + span.publish() + } + def "starting with a timestamp disables nanotime"() { setup: def mod = TimeUnit.MILLISECONDS.toNanos(1) From 22a0a1fe7511da705a516f94ec1d1485afc20732 Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Tue, 21 Apr 2026 11:30:30 +0200 Subject: [PATCH 15/15] fix(profiling): skip SpanExecutionThread emission for virtual threads --- .../main/java/datadog/trace/core/DDSpan.java | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java b/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java index 495e1ade63c..54ccd6b7490 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java @@ -9,6 +9,7 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.NANOSECONDS; +import datadog.environment.ThreadSupport; import datadog.trace.api.Config; import datadog.trace.api.DDSpanId; import datadog.trace.api.DDTags; @@ -163,8 +164,15 @@ private void finishAndAddToTrace(final long durationNano) { // Capture the execution thread while still on the span's own finishing thread. // CoreTracer.write() is called later from the event loop; the info captured here is the // authoritative source for SpanNode thread attribution (see SpanExecutionThreadEvent). - context.captureExecutionThread( - Thread.currentThread().getId(), Thread.currentThread().getName()); + // Virtual threads have their own stable thread ID that does NOT match any carrier thread + // lane in the JFR timeline. Emitting a SpanExecutionThreadEvent with a virtual thread ID + // would override the SpanNode's EVENT_THREAD (carrier OS tid) with a value that cannot be + // matched, producing worse attribution. Skip capture for virtual threads so the backend + // falls back to the SpanNode's EVENT_THREAD — the best available carrier-thread attribution. + if (!ThreadSupport.isVirtual()) { + context.captureExecutionThread( + Thread.currentThread().getId(), Thread.currentThread().getName()); + } TraceCollector.PublishState publishState = context.getTraceCollector().onPublish(this); log.debug("Finished span ({}): {}", publishState, this); } else { @@ -264,8 +272,12 @@ public final boolean phasedFinish() { // own finishing thread so that SpanExecutionThreadEvent is emitted correctly. // Without this, phasedFinish() spans (e.g. Netty HTTP-client) have executionThreadId=0 // and fall back to the event-loop thread that calls CoreTracer.write(). - context.captureExecutionThread( - Thread.currentThread().getId(), Thread.currentThread().getName()); + // Skip virtual threads — their ID does not match any carrier thread lane in the JFR + // timeline (same reasoning as finishAndAddToTrace). + if (!ThreadSupport.isVirtual()) { + context.captureExecutionThread( + Thread.currentThread().getId(), Thread.currentThread().getName()); + } log.debug("Finished span (PHASED): {}", this); return true; } else {