diff --git a/ddprof-lib/src/main/cpp/counters.h b/ddprof-lib/src/main/cpp/counters.h index 5f00c2da71..4fc18a7e97 100644 --- a/ddprof-lib/src/main/cpp/counters.h +++ b/ddprof-lib/src/main/cpp/counters.h @@ -291,11 +291,8 @@ X(REFERENCE_CHAIN_CANDIDATE_COUNT, "reference_chain_candidate_count") \ X(REFERENCE_CHAIN_CANDIDATES_FOUND, "reference_chain_candidates_found") \ /* hasLeakSignal() gate evaluations refused while the OOM urgency latch is \ - * set and selectLeakCandidates() found zero candidates - a heap-wide leak \ - * no single klass explains, so no reference-chain search may start. \ - * One increment per refused evaluation, not per urgency episode: evaluation \ - * runs only after canAffordNewSearch()'s safepoint budget check and follows \ - * the ramped cadence. */ \ + * set and selectLeakCandidates() found zero candidates (a heap-wide leak no \ + * single klass explains). Counts evaluations (~1/s while latched), not episodes. */ \ X(REFERENCE_CHAIN_URGENT_NO_CANDIDATE, "reference_chain_urgent_no_candidate") \ /* Heap-floor seqlock: samples DROPPED when a writer claims a slot whose \ * seq is still the sentinel - only possible if the ring lapped an \ diff --git a/ddprof-lib/src/main/cpp/javaApi.cpp b/ddprof-lib/src/main/cpp/javaApi.cpp index 671168222d..b2605d22d9 100644 --- a/ddprof-lib/src/main/cpp/javaApi.cpp +++ b/ddprof-lib/src/main/cpp/javaApi.cpp @@ -1473,6 +1473,75 @@ Java_com_datadoghq_profiler_JavaProfiler_isUrgentForTest0(JNIEnv *env, return tracker->isUrgentForTest() ? JNI_TRUE : JNI_FALSE; } +// Places the previous pass a controlled distance in the past and sets the effective cadence +// shouldRunPass()'s minimum-spacing gate enforces - see setPassSpacingForTest(). +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_setReferenceChainPassSpacingForTest0( + JNIEnv *env, jclass unused, jlong ms_since_last_pass, jlong cadence_ms) { + ProfiledThread::initCurrentThreadSignalSafe(); + ReferenceChainTracker::instance()->setPassSpacingForTest( + (u64)ms_since_last_pass, (u64)cadence_ms); +} + +// Sets the root-enumeration scheduling state (last attempt's time and measured in-safepoint +// cost) - see setRootEnumStateForTest(). +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_setReferenceChainRootEnumStateForTest0( + JNIEnv *env, jclass unused, jlong ms_since_last_enum, + jlong last_enum_cost_ms) { + ProfiledThread::initCurrentThreadSignalSafe(); + ReferenceChainTracker::instance()->setRootEnumStateForTest( + (long)ms_since_last_enum, (u64)last_enum_cost_ms); +} + +// Evaluates shouldRunRootEnum() with a synthetic clock so a test can place the last attempt a +// controlled distance in the past. +extern "C" DLLEXPORT jboolean JNICALL +Java_com_datadoghq_profiler_JavaProfiler_shouldRunRootEnumForTest0( + JNIEnv *env, jclass unused, jboolean manual_first_pass) { + ProfiledThread::initCurrentThreadSignalSafe(); + return ReferenceChainTracker::instance()->shouldRunRootEnumForTest( + OS::nanotime(), manual_first_pass == JNI_TRUE) + ? JNI_TRUE + : JNI_FALSE; +} + +// Forces a started, RUNNING search with fresh pain budgets so shouldRunPassForTest0() evaluates +// the running-search gates deterministically - see forceRunningSearchForTest(). +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_forceRunningReferenceChainSearchForTest0( + JNIEnv *env, jclass unused) { + ProfiledThread::initCurrentThreadSignalSafe(); + ReferenceChainTracker::instance()->forceRunningSearchForTest(); +} + +// Sets the canary chase's candidate count and found-bits mask so shouldRunPassForTest0() +// evaluates the canary lane - see setCanaryChaseStateForTest(). +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_setReferenceChainCanaryStateForTest0( + JNIEnv *env, jclass unused, jint candidate_count, jlong found_bits) { + ProfiledThread::initCurrentThreadSignalSafe(); + ReferenceChainTracker::instance()->setCanaryChaseStateForTest( + (int)candidate_count, (u64)found_bits); +} + +// Current adaptive root-enumeration interval floor - see rootEnumIntervalFloorNs(). +extern "C" DLLEXPORT jlong JNICALL +Java_com_datadoghq_profiler_JavaProfiler_rootEnumIntervalFloorNsForTest0( + JNIEnv *env, jclass unused) { + ProfiledThread::initCurrentThreadSignalSafe(); + return (jlong)ReferenceChainTracker::instance() + ->rootEnumIntervalFloorNsForTest(); +} + +// Live GC-finish epoch - see gcFinishEpoch() (relaxed atomic load, no lock needed). +extern "C" DLLEXPORT jlong JNICALL +Java_com_datadoghq_profiler_JavaProfiler_referenceChainGcFinishEpochForTest0( + JNIEnv *env, jclass unused) { + ProfiledThread::initCurrentThreadSignalSafe(); + return (jlong)ReferenceChainTracker::instance()->gcFinishEpoch(); +} + #else // !DEBUG // Release builds: no-op stubs so the Java natives still link. @@ -1595,6 +1664,41 @@ Java_com_datadoghq_profiler_JavaProfiler_backdateLeakSignalActivationForTest0( return 0; } +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_setReferenceChainPassSpacingForTest0( + JNIEnv *env, jclass unused, jlong ms_since_last_pass, jlong cadence_ms) {} + +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_setReferenceChainRootEnumStateForTest0( + JNIEnv *env, jclass unused, jlong ms_since_last_enum, + jlong last_enum_cost_ms) {} + +extern "C" DLLEXPORT jboolean JNICALL +Java_com_datadoghq_profiler_JavaProfiler_shouldRunRootEnumForTest0( + JNIEnv *env, jclass unused, jboolean manual_first_pass) { + return JNI_FALSE; +} + +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_forceRunningReferenceChainSearchForTest0( + JNIEnv *env, jclass unused) {} + +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_setReferenceChainCanaryStateForTest0( + JNIEnv *env, jclass unused, jint candidate_count, jlong found_bits) {} + +extern "C" DLLEXPORT jlong JNICALL +Java_com_datadoghq_profiler_JavaProfiler_rootEnumIntervalFloorNsForTest0( + JNIEnv *env, jclass unused) { + return 0; +} + +extern "C" DLLEXPORT jlong JNICALL +Java_com_datadoghq_profiler_JavaProfiler_referenceChainGcFinishEpochForTest0( + JNIEnv *env, jclass unused) { + return 0; +} + extern "C" DLLEXPORT void JNICALL Java_com_datadoghq_profiler_JavaProfiler_setLeakSignalActivatedNsForTest0( JNIEnv *env, jclass unused, jlong activatedNs) {} diff --git a/ddprof-lib/src/main/cpp/referenceChainFrontier.h b/ddprof-lib/src/main/cpp/referenceChainFrontier.h index 666a71cb18..d5b662814e 100644 --- a/ddprof-lib/src/main/cpp/referenceChainFrontier.h +++ b/ddprof-lib/src/main/cpp/referenceChainFrontier.h @@ -47,6 +47,9 @@ constexpr u8 CANARY_STUCK = 3; // starts from a brand-new-search shape (release-then-restart, fresh TTL clock, no urgency // carry-over). constexpr u8 RECORDING_END = 4; +// The root enumerator returned a JVMTI error on MAX_CONSECUTIVE_ENUM_ERRORS consecutive attempts - +// the root picture is unusable and retrying is not making progress. +constexpr u8 ROOT_ENUM_ERROR = 5; } // namespace SearchAbandonReason // Metadata for one tagged frontier object. diff --git a/ddprof-lib/src/main/cpp/referenceChainTraversal.cpp b/ddprof-lib/src/main/cpp/referenceChainTraversal.cpp index 1e3440a7a8..288b9aba36 100644 --- a/ddprof-lib/src/main/cpp/referenceChainTraversal.cpp +++ b/ddprof-lib/src/main/cpp/referenceChainTraversal.cpp @@ -171,8 +171,8 @@ void ReferenceChainTracker::runPassManualWalk(jvmtiEnv *jvmti, JNIEnv *jni, // Shared wall-clock ceiling for this whole call's static-field sweep, expandFrontier(), and // rotation sub-calls below (see _pass_deadline_ns's own comment) - deliberately NOT applied to - // root/stack-ref enumeration itself, which is instead cadence-gated by run_root_enum/ - // ROOT_ENUM_MIN_INTERVAL_NS. + // root/stack-ref enumeration itself, which is instead interval-gated by shouldRunRootEnum() + // (ROOT_ENUM_MIN_INTERVAL_NS and the adaptive extension). _pass_deadline_ns = _effective_pause_target_ms > 0 ? OS::nanotime() + (u64)_effective_pause_target_ms * 1000000ULL : 0; @@ -213,7 +213,8 @@ void ReferenceChainTracker::runPassManualWalk(jvmtiEnv *jvmti, JNIEnv *jni, jvmtiError root_err = jvmti->IterateOverReachableObjects( heapRootCallback, stackRefCallback, /*object_ref_callback=*/nullptr, &ctx); - *safepoint_ns += OS::nanotime() - root_enum_start_ns; + u64 root_enum_safepoint_ns = OS::nanotime() - root_enum_start_ns; + *safepoint_ns += root_enum_safepoint_ns; // expand_budget is spent independently of root_enum_budget below (see // ROOT_ENUM_MIN_INTERVAL_NS's own comment) - ctx.edges_admitted is written straight into @@ -221,23 +222,39 @@ void ReferenceChainTracker::runPassManualWalk(jvmtiEnv *jvmti, JNIEnv *jni, // whatever root enumeration admitted. *edges_admitted = ctx.edges_admitted; _last_root_enum_ns = OS::nanotime(); + // Every attempt - completed, truncated, or errored - re-pays the JVM's unconditional + // all-thread EA deoptimization (see ROOT_ENUM_MIN_INTERVAL_NS's comment), so the measured + // in-safepoint cost extends the next attempt's interval floor regardless of the outcome. + _last_root_enum_safepoint_ms = root_enum_safepoint_ns / 1000000; if (root_err != JVMTI_ERROR_NONE) { + // The enumeration attempt errored out - the root picture is incomplete, so record it: + // runPass()'s termination ladder must not treat the search as settled. The consecutive-error + // count feeds the ladder's ROOT_ENUM_ERROR abandon; any successful attempt resets it. + _root_enum_incomplete = true; + _consecutive_enum_errors++; *truncated = true; *frontier_cap_hit = false; - _root_enum_truncated_last_time = false; return; } if (ctx.truncated) { + // A truncated attempt ends this pass - an incomplete enumeration leaves the frontier's root + // picture inconsistent with the expansion phases below, so the next pass expands on the + // roots already admitted. Retry timing is shouldRunRootEnum()'s (ROOT_ENUM_MIN_INTERVAL_NS). + // Record the incompleteness so the ladder neither completes nor no-progress- + // abandons while unenumerated roots may remain. The FRONTIER_CAP_HIT sub-case also lands + // here; that is deliberately harmless - runPass()'s frontier-cap abandon precedes any read + // of the flag and restartSearch() clears it. + _root_enum_incomplete = true; + _consecutive_enum_errors = 0; *truncated = true; *frontier_cap_hit = ctx.frontier_cap_hit; - // Only a budget-exhausted truncation (not a frontier-cap-hit, which abandons the search - // outright) is grounds to retry root enumeration on the very next pass - see - // _root_enum_truncated_last_time's own comment. - _root_enum_truncated_last_time = !ctx.frontier_cap_hit; return; } - _root_enum_truncated_last_time = false; + // Completed attempt (no error, no truncation): the root picture is now consistent, so the + // ladder may treat the frontier as the only pending work. + _root_enum_incomplete = false; + _consecutive_enum_errors = 0; } int expand_phase_edges_admitted = 0; @@ -1050,8 +1067,7 @@ bool ReferenceChainTracker::runPass(jvmtiEnv *jvmti, JNIEnv *jni, // (runPassManualWalk()'s own comment) - there is no "first pass walks the whole graph inline" // shortcut here, so every pass (first or resumed) takes the same expand-frontier shape. u64 now_ns = OS::nanotime(); - bool run_root_enum = manual_first_pass || _root_enum_truncated_last_time || - (now_ns - _last_root_enum_ns >= ROOT_ENUM_MIN_INTERVAL_NS); + bool run_root_enum = shouldRunRootEnum(now_ns, manual_first_pass); int frontier_size_before_pass = _frontier != nullptr ? _frontier->size() : 0; @@ -1070,20 +1086,16 @@ bool ReferenceChainTracker::runPass(jvmtiEnv *jvmti, JNIEnv *jni, err = JVMTI_ERROR_NONE; store(_passes_run, load(_passes_run) + 1); - _last_pass_gc_finish_epoch = gcFinishEpoch(); store(_last_pass_ns, OS::nanotime()); if (!run_root_enum) { - // A pass that ran root/stack-ref enumeration spends _first_pass_budget, not _effective_budget - - // its duration is not a signal about the per-pass cost updatePacing() is trying to regulate - // (expandFrontier()'s cheap, per-node expansion calls), so feeding it in here would throttle - // _effective_budget down for every one of those unrelated later passes based on a single, - // deliberately oversized outlier. + // Only non-root-enum passes feed the budget/cadence controller - root-enum passes are + // excluded (see the else branch for why). updatePacing(safepoint_ns); } else { - // Excluded from the budget/cadence controller above, but not from the borrow ceiling's - // revocation check (see maybeRevokeBorrowForRootEnumPass()'s own comment) - a root-enum pass's - // in-safepoint cost is real pause time and must still be able to revoke a borrowed-budget grant - // the pacing controller would otherwise keep believing is safe. + // A root-enum pass's duration is not a per-pass cost signal for updatePacing(): it spends + // _first_pass_budget, not _effective_budget, and feeding the outlier in would throttle every + // later expansion pass. The cost is still real pause time, so it can still revoke a + // borrowed-budget grant (maybeRevokeBorrowForRootEnumPass()). maybeRevokeBorrowForRootEnumPass(safepoint_ns); } // accumulate this pass's own in-safepoint cost toward the running total restartSearch() will @@ -1097,7 +1109,10 @@ bool ReferenceChainTracker::runPass(jvmtiEnv *jvmti, JNIEnv *jni, _cpu_pain_budget.spend(non_safepoint_ns / 1000000); // Apply terminal conditions in priority order. - bool has_pending_frontier = truncated; + // _root_enum_incomplete counts as pending frontier: while roots remain unenumerated (truncated + // or errored attempt, or a floor-wait with a previous attempt incomplete) the search must + // neither terminate nor no-progress-abandon. + bool has_pending_frontier = truncated || _root_enum_incomplete; int frontier_size_after = _frontier->size(); if (frontier_cap_hit) { // Frontier table is full -- no new entries can ever be admitted, so frontier_size_after can @@ -1110,15 +1125,14 @@ bool ReferenceChainTracker::runPass(jvmtiEnv *jvmti, JNIEnv *jni, frontier_size_after); } else if (!has_pending_frontier && _watched_leak_klass_count == 0) { storeRelease(_search_state, (u8)SearchState::COMPLETED); - } else if (_ttl_ms > 0 && + } else if (_ttl_ms > 0 && !_root_enum_incomplete && (OS::nanotime() - load(_search_start_ns)) / 1000000 >= (u64)_ttl_ms) { - // TTL bounds stop-the-world work independently of frontier progress. - // _search_start_ns is a plain OS::nanotime() timestamp (see its assignment - // above), not a TSC tick count, so the elapsed time must be converted with - // a straight ns-to-ms division rather than TSC::ticks_to_millis() (which - // divides by TSC::frequency() and would misconvert whenever that - // frequency differs from 1 GHz). + // Deferred while _root_enum_incomplete: abandoning mid-enumeration loses the unenumerated + // roots - the clock keeps running, so the abandon fires on the first pass after the + // enumeration completes. _search_start_ns is a plain OS::nanotime() timestamp, not a TSC tick + // count: elapsed time is converted with a straight ns-to-ms division, never TSC::ticks_to_ + // millis() (which divides by TSC::frequency()). store(_abandon_reason, (u8)SearchAbandonReason::TTL); storeRelease(_search_state, (u8)SearchState::ABANDONED); enqueuePendingAbandonedEvent(); @@ -1127,18 +1141,31 @@ bool ReferenceChainTracker::runPass(jvmtiEnv *jvmti, JNIEnv *jni, _ttl_ms, (unsigned long long)((OS::nanotime() - load(_search_start_ns)) / 1000000)); + } else if (_consecutive_enum_errors >= MAX_CONSECUTIVE_ENUM_ERRORS) { + // The enumerator has errored on MAX_CONSECUTIVE_ENUM_ERRORS consecutive attempts (any + // successful attempt resets the counter) - bound the retries even with ttl=0. + store(_abandon_reason, (u8)SearchAbandonReason::ROOT_ENUM_ERROR); + storeRelease(_search_state, (u8)SearchState::ABANDONED); + enqueuePendingAbandonedEvent(); + TEST_LOG_SUMMARY("ReferenceChainTracker::runPass root-enum errors -- abandoning search " + "(consecutive_errors=%u)", + _consecutive_enum_errors); } else if (_passes_since_last_progress >= NO_PROGRESS_PASS_LIMIT && - !urgencyLatched()) { + !urgencyLatched() && !_root_enum_incomplete) { // The frontier hasn't grown for NO_PROGRESS_PASS_LIMIT consecutive passes — the search is // genuinely stuck (not just slow), so abandon. Reads the published latch; calling isUrgent() // here would advance the machine from a second, freshly-read projection. + // Belt-and-braces: the progress block below already holds the counter during floor-wait + // passes; this guard also covers a pass that both runs an enum chunk and stays frontier-flat. store(_abandon_reason, (u8)SearchAbandonReason::TTL); storeRelease(_search_state, (u8)SearchState::ABANDONED); enqueuePendingAbandonedEvent(); - } else if (_candidate_count > 0 && + } else if (_candidate_count > 0 && !_root_enum_incomplete && __builtin_popcountll(_candidate_found_bits) == (u64)_candidate_count) { // Canary early termination: all leaked candidates have been found -- the search is complete. + // Gated on !_root_enum_incomplete: an unenumerated root region may still admit + // further candidates, so "all found" is only honest once an enumeration completed. storeRelease(_search_state, (u8)SearchState::COMPLETED); Counters::increment(REFERENCE_CHAIN_CANDIDATES_FOUND, __builtin_popcountll(_candidate_found_bits)); @@ -1156,9 +1183,13 @@ bool ReferenceChainTracker::runPass(jvmtiEnv *jvmti, JNIEnv *jni, } } - // Track progress: if the frontier grew this pass, reset the no-progress counter. + // Track progress: frontier growth resets the no-progress counter; a pass that only waited out + // the enumeration interval while _root_enum_incomplete holds it - no stuck signal (it also + // feeds the canary-stuck abandon). The canary-specific progress counter is never held. if (frontier_size_after > frontier_size_before_pass) { _passes_since_last_progress = 0; + } else if (_root_enum_incomplete && !run_root_enum) { + // Held: enumeration still owes chunks; this pass only waited out the interval floor. } else { _passes_since_last_progress++; } @@ -1223,17 +1254,18 @@ bool ReferenceChainTracker::runPass(jvmtiEnv *jvmti, JNIEnv *jni, TEST_LOG_SUMMARY("ReferenceChainTracker::runPass done: err=%d edges_admitted=%d truncated=%d " "frontier_cap_hit=%d searchState=%d abandonReason=%d frontierSize=%d " "effectiveBudget=%d effectiveCadenceNs=%llu pendingExpand=%zu priorityExpand=%zu " - "candidateFound=%d/%d discoveredCounts=[%d,%d,%d,%d,%d]", + "candidateFound=%d/%d discoveredCounts=[%d,%d,%d,%d,%d] rootEnumIncomplete=%d", (int)err, edges_admitted, truncated, frontier_cap_hit, (int)load(_search_state), (int)_abandon_reason, _frontier->size(), _effective_budget, - (unsigned long long)_effective_cadence_ns, + (unsigned long long)load(_effective_cadence_ns), _pending_expand.size(), _priority_expand.size(), (int)__builtin_popcountll(_candidate_found_bits), _candidate_count, _candidate_count > 0 ? _candidate_discovered_count[0] : 0, _candidate_count > 1 ? _candidate_discovered_count[1] : 0, _candidate_count > 2 ? _candidate_discovered_count[2] : 0, _candidate_count > 3 ? _candidate_discovered_count[3] : 0, - _candidate_count > 4 ? _candidate_discovered_count[4] : 0); + _candidate_count > 4 ? _candidate_discovered_count[4] : 0, + (int)_root_enum_incomplete); return err == JVMTI_ERROR_NONE; } @@ -1285,20 +1317,24 @@ void ReferenceChainTracker::updatePacing(u64 pass_wall_ns) { int64_t overflow = desired - clamped; _effective_budget = (int)clamped; + // One load() and one store() of _effective_cadence_ns: threadLoop()'s urgency ramp writes it + // outside _engine_lock, so the access must be atomic. The read-modify-write as a whole is not + // atomic - it races the urgency ramp by design (while urgent the ramp owns the field) - the + // load()/store() only removes the mixed plain/atomic data race. if (overflow < 0) { // Still over the pause-time ceiling even at the minimum budget - widen the fallback interval // instead of shrinking the budget further. u64 step = (u64)(-overflow) * CADENCE_NS_PER_EDGE_OVERFLOW; - _effective_cadence_ns = - std::min(_effective_cadence_ns + step, MAX_EFFECTIVE_CADENCE_NS); + u64 cur = load(_effective_cadence_ns); + store(_effective_cadence_ns, std::min(cur + step, MAX_EFFECTIVE_CADENCE_NS)); } else if (overflow > 0) { // Comfortably under the ceiling even at the maximum (config) budget - relax the fallback // interval. u64 step = (u64)overflow * CADENCE_NS_PER_EDGE_OVERFLOW; - _effective_cadence_ns = - step >= _effective_cadence_ns - ? MIN_EFFECTIVE_CADENCE_NS - : std::max(_effective_cadence_ns - step, MIN_EFFECTIVE_CADENCE_NS); + u64 cur = load(_effective_cadence_ns); + store(_effective_cadence_ns, + step >= cur ? MIN_EFFECTIVE_CADENCE_NS + : std::max(cur - step, MIN_EFFECTIVE_CADENCE_NS)); } // overflow == 0: the budget clamp alone fully absorbed this pass's correction - leave the cadence // at its current value. diff --git a/ddprof-lib/src/main/cpp/referenceChains.cpp b/ddprof-lib/src/main/cpp/referenceChains.cpp index 88d4ddc6f8..92ead2b739 100644 --- a/ddprof-lib/src/main/cpp/referenceChains.cpp +++ b/ddprof-lib/src/main/cpp/referenceChains.cpp @@ -304,7 +304,7 @@ Error ReferenceChainTracker::start(Arguments &args) { _pause_target_ms = args._reference_chains_pause_target_ms; _effective_pause_target_ms = _pause_target_ms; _effective_budget = _budget; - _effective_cadence_ns = PASS_CADENCE_NS; + _effective_cadence_ns = PASS_CADENCE_NS; // plain: the BFS thread is not running yet _candidate_count = 0; _candidate_found_bits = 0; memset(_candidate_discovered_count, 0, sizeof(_candidate_discovered_count)); @@ -494,7 +494,7 @@ void ReferenceChainTracker::stopThread() { bool ReferenceChainTracker::applyOomRamp(double seconds_to_oom) { bool urgent = seconds_to_oom >= 0 && seconds_to_oom < OOM_RAMP_START_S; long target_ms = _pause_target_ms; - u64 cadence_ns = _effective_cadence_ns; + u64 cadence_ns = load(_effective_cadence_ns); if (urgent) { double x = 1.0 - seconds_to_oom / OOM_RAMP_START_S; // 0 at 30min out, 1 at OOM target_ms = std::lround(_pause_target_ms * @@ -518,7 +518,7 @@ bool ReferenceChainTracker::applyOomRamp(double seconds_to_oom) { } // Cadence ownership on every branch: urgent iterations set it here, the episode's end (or // stop()) restores PASS_CADENCE_NS, and otherwise updatePacing() owns it. - _effective_cadence_ns = cadence_ns; + store(_effective_cadence_ns, cadence_ns); } if (target_ms != _effective_pause_target_ms) { _effective_pause_target_ms = target_ms; @@ -537,6 +537,7 @@ bool ReferenceChainTracker::applyOomRamp(double seconds_to_oom) { closeUrgencyEpisode(); // The episode owned the cadence while it ran; hand it back at the fallback in the same // iteration so the idle loop stops waking at the ramped rate. + TEST_LOG_SUMMARY("ReferenceChainTracker::threadLoop urgency budget restore " "budget=%d", _budget); } @@ -568,7 +569,7 @@ void ReferenceChainTracker::threadLoop() { return; } DEBUG_ONLY(rcDebugLevelRefresh(true)); // apply the override file before the first log line - TEST_LOG_SUMMARY("ReferenceChainTracker::threadLoop started, cadence=%lluns rc_debug_level=%d", (unsigned long long)_effective_cadence_ns, rcDebugLevel()); + TEST_LOG_SUMMARY("ReferenceChainTracker::threadLoop started, cadence=%lluns rc_debug_level=%d", (unsigned long long)load(_effective_cadence_ns), rcDebugLevel()); int iteration = 0; while (_running.load(std::memory_order_acquire)) { @@ -580,7 +581,7 @@ void ReferenceChainTracker::threadLoop() { bool urgent = applyOomRamp(seconds_to_oom); // applyOomRamp() keeps _effective_cadence_ns current, so the sleep below and // shouldRunPass()'s cadence gate always agree on the same value. - u64 cadence_ns = _effective_cadence_ns; + u64 cadence_ns = load(_effective_cadence_ns); // Third trigger for LivenessTracker::cleanup_table() (see // LivenessTracker::maybeForceCleanup()'s own comment): track()'s table-overflow branch and // flush_table()'s JFR cadence can both starve under ObjectSampler's PID-controlled sampling @@ -626,11 +627,11 @@ void ReferenceChainTracker::threadLoop() { if (should_run) { TEST_LOG_SUMMARY("ReferenceChainTracker::threadLoop iteration=%d shouldRunPass=%d searchState=%d " "passesRun=%d effectiveCadenceNs=%llu effectiveBudget=%d gcFinishEpoch=%llu " - "lastPassGcFinishEpoch=%llu nowMinusLastPassNs=%llu", + "nowMinusLastPassNs=%llu", ++iteration, should_run, (int)_search_state, _passes_run, - (unsigned long long)_effective_cadence_ns, _effective_budget, - (unsigned long long)gcFinishEpoch(), (unsigned long long)_last_pass_gc_finish_epoch, - (unsigned long long)(now_ns - _last_pass_ns)); + (unsigned long long)load(_effective_cadence_ns), _effective_budget, + (unsigned long long)gcFinishEpoch(), + (unsigned long long)(now_ns - load(_last_pass_ns))); runPassSerialized(jvmti, jni); } finishLoopIterationSerialized(jvmti, jni); @@ -725,12 +726,28 @@ bool ReferenceChainTracker::shouldRunPass(u64 now_ns, double seconds_to_oom) { // Adaptive CPU budget: 100x refill while a canary chase is open - NOT a rate control (the canary // lane's rate is bounded by _canary_backoff_ns's progress-driven exponential backoff, see its own // comment) but a double-throttle guard: the base refill rate is tuned for the ordinary ~1 pass/s - // whole-graph cadence and would otherwise starve a chase the backoff has already paced. + // whole-graph cadence and would otherwise starve a chase the backoff has already paced. Applied + // before the minimum-pass-spacing gate so the wait it induces drains at the current rate, not + // the previous episode's. double multiplier = canary_active ? CANARY_PAIN_BUDGET_REFILL_MULTIPLIER : 1.0; _cpu_pain_budget.setRefillRate( std::min(_pain_budget_refill_rate * multiplier, 1.0), now_ns); + // Minimum pass spacing on every authorization lane: bounds safepointing pass frequency via the + // FIXED floor MIN_PASS_SPACING_NS - updatePacing() paces each pass's pause length, not the pass + // rate. The OOM ramp is the only exemption (while urgent it owns _effective_cadence_ns outright). + u64 cadence_ns = load(_effective_cadence_ns); + u64 min_spacing_ns = + _oom_ramp_active ? cadence_ns : std::max(cadence_ns, MIN_PASS_SPACING_NS); + if (now_ns - load(_last_pass_ns) < min_spacing_ns) { + TEST_LOG_SUMMARY("ReferenceChainTracker::shouldRunPass held off by minimum pass spacing " + "since_last_pass_ms=%llu effective_cadence_ns=%llu min_spacing_ns=%llu", + (unsigned long long)((now_ns - load(_last_pass_ns)) / 1000000ULL), + (unsigned long long)cadence_ns, + (unsigned long long)min_spacing_ns); + return false; + } if (!_cpu_pain_budget.canStartNow(now_ns)) { TEST_LOG_SUMMARY("ReferenceChainTracker::shouldRunPass blocked by cpu_pain_budget " "balance=%.1fms refill_rate=%.4f canary_active=%d " @@ -764,28 +781,42 @@ bool ReferenceChainTracker::shouldRunPass(u64 now_ns, double seconds_to_oom) { (unsigned long long)_canary_pass_ema_ms); return true; } - u64 gc_finish_epoch = gcFinishEpoch(); - if (gc_finish_epoch != _last_pass_gc_finish_epoch) { - // Triggering section: "a GC just happened, a pass may be worth running soon". - TEST_LOG_SUMMARY("ReferenceChainTracker::shouldRunPass -> true (gcFinishEpoch=%llu != " - "lastPassGcFinishEpoch=%llu)", - (unsigned long long)gc_finish_epoch, - (unsigned long long)_last_pass_gc_finish_epoch); + // Passes run on a pure cadence: the spacing gate above (max of the controller's cadence and + // the fixed MIN_PASS_SPACING_NS floor) is the only pacing law - a completed GC neither bypasses + // it nor earns an earlier pass. updatePacing() widens or relaxes _effective_cadence_ns from the + // measured pause-time signal - see its own comment. + TEST_LOG_SUMMARY("ReferenceChainTracker::shouldRunPass -> true (now_ns=%llu last_pass_ns=%llu " + "delta=%llu effectiveCadenceNs=%llu)", + (unsigned long long)now_ns, (unsigned long long)load(_last_pass_ns), + (unsigned long long)(now_ns - load(_last_pass_ns)), + (unsigned long long)cadence_ns); + return true; +} + +// Wall-clock interval the next root/stack-ref enumeration attempt must wait out - see the +// ROOT_ENUM_* constants' comments (referenceChains.h) for why the floor extends with the last +// attempt's measured in-safepoint cost and why it is capped. +u64 ReferenceChainTracker::rootEnumIntervalFloorNs() const { + u64 interval_ns = ROOT_ENUM_MIN_INTERVAL_NS; + if (_last_root_enum_safepoint_ms > 0) { + u64 scaled_ns = std::min( + _last_root_enum_safepoint_ms * ROOT_ENUM_INTERVAL_NS_PER_COST_MS, + ROOT_ENUM_MAX_INTERVAL_NS); + interval_ns = std::max(interval_ns, scaled_ns); + } + return interval_ns; +} + +// See the declaration's comment (referenceChains.h). +bool ReferenceChainTracker::shouldRunRootEnum(u64 now_ns, + bool manual_first_pass) const { + // manual_first_pass is checked ahead of the interval: _last_root_enum_ns is 0 before the first + // attempt, and an agent starting early in the JVM's lifetime would otherwise see + // now_ns - 0 fall below the interval floor and skip the search's only mandatory enumeration. + if (manual_first_pass) { return true; } - // Pause-time pacing controller: compares against _effective_cadence_ns, not the fixed - // PASS_CADENCE_NS - see that field's own comment (referenceChains.h) for how updatePacing() - // widens or relaxes it from the measured pause-time signal. - bool cadence_elapsed = now_ns - _last_pass_ns >= _effective_cadence_ns; - // Only log when the cadence actually elapsed (a pass will run). - if (cadence_elapsed) { - TEST_LOG_SUMMARY("ReferenceChainTracker::shouldRunPass -> true (now_ns=%llu last_pass_ns=%llu " - "delta=%llu effectiveCadenceNs=%llu)", - (unsigned long long)now_ns, (unsigned long long)_last_pass_ns, - (unsigned long long)(now_ns - _last_pass_ns), - (unsigned long long)_effective_cadence_ns); - } - return cadence_elapsed; + return now_ns - _last_root_enum_ns >= rootEnumIntervalFloorNs(); } // Search restart gate (this class's own header comment). Deliberately a probe (max=1) rather than @@ -1005,6 +1036,17 @@ bool ReferenceChainTracker::canAffordNewSearch(u64 now_ns, double seconds_to_oom _pain_budget_refill_rate); return false; // still cooling down from the last search's own cost } + // Root-enum interval floor also gates a restarted search's first enumeration: restartSearch() + // deliberately keeps _last_root_enum_ns, closing the enumerate -> early-terminate -> restart -> + // enumerate loop. The != 0 guard preserves the brand-new-tracker exemption. + if (_last_root_enum_ns != 0 && + now_ns - _last_root_enum_ns < rootEnumIntervalFloorNs()) { + TEST_LOG_SUMMARY("ReferenceChainTracker::canAffordNewSearch blocked by " + "root_enum_interval_floor since_last_enum_ms=%llu floor_ns=%llu", + (unsigned long long)((now_ns - _last_root_enum_ns) / 1000000ULL), + (unsigned long long)rootEnumIntervalFloorNs()); + return false; + } return hasLeakSignal(seconds_to_oom); } @@ -1068,7 +1110,8 @@ void ReferenceChainTracker::restartSearch() { _leak_parent_fanout.clear(); _leak_tags_assigned = 0; _leak_tags_resolved = 0; - _last_pass_gc_finish_epoch = 0; + _root_enum_incomplete = false; + _consecutive_enum_errors = 0; store(_last_pass_ns, (u64)0); store(_passes_run, 0); // Reset back to their just-constructed values (0 / -1) like every other per-search field this @@ -1153,7 +1196,6 @@ void ReferenceChainTracker::resetSearchStateForTest(jvmtiEnv *jvmti, _leak_parent_fanout.clear(); _leak_tags_assigned = 0; _leak_tags_resolved = 0; - _last_pass_gc_finish_epoch = 0; store(_last_pass_ns, (u64)0); store(_passes_run, 0); _passes_since_last_progress = 0; @@ -1172,6 +1214,14 @@ void ReferenceChainTracker::resetSearchStateForTest(jvmtiEnv *jvmti, _last_resolved_class_count = 0; _last_static_field_class_count = -1; _static_field_sweep_cursor = 0; + // Reset the pacing/enumeration state the scheduling gates read: an earlier test's seeded + // cadence or root-enum cost would otherwise throttle every later test's BFS thread; _last_pass_ns + // is already zeroed above, so the spacing gate is open. Plain write: the BFS thread is stopped. + _effective_cadence_ns = PASS_CADENCE_NS; + _last_root_enum_ns = 0; + _last_root_enum_safepoint_ms = 0; + _root_enum_incomplete = false; + _consecutive_enum_errors = 0; _static_field_sweep_cycle_truncated = false; _candidate_count = 0; _candidate_found_bits = 0; diff --git a/ddprof-lib/src/main/cpp/referenceChains.h b/ddprof-lib/src/main/cpp/referenceChains.h index 668b4a3199..b5c149bc44 100644 --- a/ddprof-lib/src/main/cpp/referenceChains.h +++ b/ddprof-lib/src/main/cpp/referenceChains.h @@ -272,7 +272,11 @@ class ReferenceChainTracker { // against, replacing the fixed PASS_CADENCE_NS constant below in that role once updatePacing() // starts adjusting it - see PASS_CADENCE_NS's own comment for why that constant survives as this // field's starting value rather than being deleted outright. - u64 _effective_cadence_ns; + // Written by the BFS thread (threadLoop()'s urgency ramp, outside _engine_lock) and by + // runPass()/updatePacing() and test seams on other threads: every access that can overlap + // another thread's goes through arch.h load()/store(). Plain access only where the BFS thread + // is not running (start(), resetSearchStateForTest()). + volatile u64 _effective_cadence_ns; // Budget-borrowing: extra headroom updatePacing() has temporarily granted above _budget's own // ceiling, earned by a sustained run of comfortably- under-target passes (see @@ -601,9 +605,6 @@ class ReferenceChainTracker { // re-queues - see that method's own comment. static constexpr int LEAK_ACCUMULATION_ROTATION_BUDGET = 16; - // Snapshot of gcFinishEpoch() as of the end of the last pass. - u64 _last_pass_gc_finish_epoch; - // OS::nanotime() as of the end of the last pass. Written by runPass() on the BFS thread; also // read cross-thread by buildAbandonedEvent()'s elapsed-time calculation, so // volatile/load()-accessed like _search_state above. @@ -697,9 +698,19 @@ class ReferenceChainTracker { static constexpr int AUTO_FIRST_PASS_BUDGET_CAP = 200000; // Minimum wall-clock gap between root/stack-ref enumeration attempts (runPassManualWalk()'s - // IterateOverReachableObjects call) after the search's own first pass. + // IterateOverReachableObjects call). Every attempt deoptimizes every compiled frame holding a + // scalar-replaced object (one VM_DeoptimizeFrame each, re-created by the JIT between attempts), + // so a truncated attempt must never retry on the very next pass. static constexpr u64 ROOT_ENUM_MIN_INTERVAL_NS = 2000000000ULL; // 2s + // Interval extension per millisecond of measured in-safepoint cost of the last attempt: an + // attempt that held the VM for 100ms pushes the next one out by a further 2s; the cap bounds it. + static constexpr u64 ROOT_ENUM_INTERVAL_NS_PER_COST_MS = 20000000ULL; // 20ms per cost ms + + // Upper bound: only enumeration admits new threads and JNI globals, so an expensive attempt + // stretches the interval but never disables it. + static constexpr u64 ROOT_ENUM_MAX_INTERVAL_NS = 30000000000ULL; // 30s + // Pause-time pacing controller: bounds and conversion constants for updatePacing()'s // budget/cadence adjustment - see that method's own comment for the full mechanism. static constexpr int MIN_EFFECTIVE_BUDGET = 2000; @@ -707,6 +718,11 @@ class ReferenceChainTracker { // Bounds for _effective_cadence_ns. The lower bound is not 0: threadLoop() sleeps for exactly // this many nanoseconds each loop iteration (below), so a true 0 would busy-loop the BFS thread. static constexpr u64 MIN_EFFECTIVE_CADENCE_NS = 10000000ULL; // 10ms + + // Fixed minimum spacing between passes, independent of updatePacing()'s output: the controller + // paces each pass's pause LENGTH, not the pass RATE - without this floor a cheap expansion pass + // relaxes _effective_cadence_ns to MIN_EFFECTIVE_CADENCE_NS and admits ~100 safepointing passes/s. + static constexpr u64 MIN_PASS_SPACING_NS = 100000000ULL; // 100ms static constexpr u64 MAX_EFFECTIVE_CADENCE_NS = PASS_CADENCE_NS * 4; // 4s // Conversion factor from "edges of budget signal updatePacing()'s clamp could not absorb" to a @@ -756,12 +772,21 @@ class ReferenceChainTracker { // first pass). u64 _last_root_enum_ns = 0; - // Set true when the most recent root/stack-ref enumeration attempt ended via BUDGET_EXHAUSTED - // (not FRONTIER_CAP_HIT, which stops admitting new frontier entries but leaves the search RUNNING - // - see runPass()'s frontier_cap_hit handling) - runPass() treats this as grounds to retry root - // enumeration on the very next pass regardless of ROOT_ENUM_MIN_INTERVAL_NS, so a - // still-incomplete attempt is not left waiting out the full interval before continuing. - bool _root_enum_truncated_last_time = false; + // In-safepoint milliseconds spent inside the most recent root/stack-ref enumeration attempt, + // including the unconditional all-thread EA deoptimization the JVM performs before its heap + // walk (see ROOT_ENUM_MIN_INTERVAL_NS's comment). Extends the next attempt's interval floor + // (rootEnumIntervalFloorNs()); 0 until the first attempt completes. + u64 _last_root_enum_safepoint_ms = 0; + + // Set when a root/stack-ref enumeration attempt ends truncated or errored: unenumerated roots + // may still hold reachable chains. Cleared on a completed attempt, by restartSearch(), and by + // resetSearchStateForTest(); runPass()'s termination ladder treats it as pending frontier. + bool _root_enum_incomplete = false; + + // Consecutive root-enumeration attempts ended by a JVMTI error; reset by any successful + // attempt, restartSearch(), and resetSearchStateForTest(). At MAX_CONSECUTIVE_ENUM_ERRORS the + // ladder abandons the search (ROOT_ENUM_ERROR) - the only bound when ttl=0. + u32 _consecutive_enum_errors = 0; ReferenceChainTracker() : _enabled(false), @@ -787,7 +812,7 @@ class ReferenceChainTracker { _admission_growth_baseline(0), _admission_sample_seq(0), _admission_decision(false), _search_state(SearchState::RUNNING), _abandon_reason(SearchAbandonReason::NONE), _search_start_ns(0), - _last_pass_gc_finish_epoch(0), _last_pass_ns(0), + _last_pass_ns(0), _passes_run(0), _root_kind_rotation_cursor(1), _stale_expanded_rotation_cursor(1), @@ -811,10 +836,22 @@ class ReferenceChainTracker { // consumes the caller's per-iteration OOM projection (threadLoop()'s single secondsToOOM() // read, passed down to the restart gate and the urgency latch so every decision of one // iteration sees the same sample); the one-arg form reads secondsToOOM() once and forwards - // (JNI shouldRunPassForTest(), gtest accessors). + // (JNI shouldRunPassForTest(), gtest accessors). Both authorizations are subject to the + // minimum pass spacing in shouldRunPass() - see its own comment. bool shouldRunPass(u64 now_ns); bool shouldRunPass(u64 now_ns, double seconds_to_oom); + // Wall-clock interval the next root/stack-ref enumeration attempt must wait out since + // _last_root_enum_ns: ROOT_ENUM_MIN_INTERVAL_NS extended by the measured in-safepoint cost of + // the last attempt (ROOT_ENUM_INTERVAL_NS_PER_COST_MS), capped at ROOT_ENUM_MAX_INTERVAL_NS. + u64 rootEnumIntervalFloorNs() const; + + // Whether runPassManualWalk() should run root/stack-ref enumeration this pass: always on the + // search's first pass, otherwise only once rootEnumIntervalFloorNs() above has elapsed since + // _last_root_enum_ns, the last attempt (truncated attempts included - see + // ROOT_ENUM_MIN_INTERVAL_NS's comment for why retries get no fast lane). + bool shouldRunRootEnum(u64 now_ns, bool manual_first_pass) const; + // Cheap probe (max=1, not the real poll pollWatchedTargets() makes) into LivenessTracker's // population-trend table: true if at least one klass shows a positive population slope worth // chasing. Always false when no leak-signal source (generations or liveheap tracking) is @@ -863,7 +900,9 @@ class ReferenceChainTracker { bool applyOomRamp(double seconds_to_oom); // Search restart gate: true once _safepoint_pain_budget has - // drained back to zero (canStartNow()) *and* hasLeakSignal() above reports at least one leak + // drained back to zero (canStartNow()) *and* the root-enum interval floor + // (rootEnumIntervalFloorNs(), measured from _last_root_enum_ns; waived while it is still 0) has + // elapsed *and* hasLeakSignal() above reports at least one leak // candidate. The two-arg form consumes the caller's projection; the one-arg form reads // secondsToOOM() once and forwards. bool canAffordNewSearch(u64 now_ns); @@ -1159,6 +1198,10 @@ class ReferenceChainTracker { // (genuinely stuck, not just slow). static constexpr int NO_PROGRESS_PASS_LIMIT = 30; + // Consecutive errored root-enum attempts the ladder tolerates before abandoning (ROOT_ENUM_ERROR) + // - bounds a persistently failing IterateOverReachableObjects even with ttl=0. + static constexpr u32 MAX_CONSECUTIVE_ENUM_ERRORS = 3; + // Base limit for the canary-specific stuck detector: candidate-discovery must show no progress // (no change to _candidate_found_bits, no new candidate admitted into a slot) for this many // consecutive passes AND the whole-graph frontier must also have stalled for @@ -1231,7 +1274,63 @@ class ReferenceChainTracker { int passesSinceLastCandidateProgressForTest() const { return _passes_since_last_candidate_progress; } int canaryStuckRestartCountForTest() const { return _canary_stuck_restart_count; } void setCandidateFoundBitsForTest(u64 bits) { _candidate_found_bits = bits; } - void setSearchStartedForTest(bool v) { _search_started = v; } + // Engine lock serializes with shouldRunPass()/runPass(); the BFS thread also touches + // _effective_cadence_ns outside the lock (threadLoop()'s urgency ramp and logs), so both fields + // use store()/load() everywhere they can overlap. While urgent the ramp overwrites a seeded + // cadence, which is why tests retry their seeds. + void setPassSpacingForTest(u64 ms_since_last_pass, u64 cadence_ms) { + MutexLocker engine_guard(_engine_lock); + store(_last_pass_ns, OS::nanotime() - ms_since_last_pass * 1000000ULL); + store(_effective_cadence_ns, cadence_ms * 1000000ULL); + } + // Raw-value sibling of setPassSpacingForTest() (same lock/store discipline - see its comment): + // places the previous pass's end at an absolute OS::nanotime() value. + void setLastPassNsForTest(u64 ns) { + MutexLocker engine_guard(_engine_lock); + store(_last_pass_ns, ns); + } + // Seeds shouldRunRootEnum()'s inputs. Engine lock held: both fields are otherwise + // BFS-thread-owned (runPassManualWalk()). Negative ms_since_last_enum clamps to 0 (never ran). + void setRootEnumStateForTest(long ms_since_last_enum, u64 last_enum_cost_ms) { + MutexLocker engine_guard(_engine_lock); + _last_root_enum_ns = ms_since_last_enum < 0 + ? 0 + : OS::nanotime() - (u64)ms_since_last_enum * 1000000ULL; + _last_root_enum_safepoint_ms = last_enum_cost_ms; + } + bool shouldRunRootEnumForTest(u64 now_ns, bool manual_first_pass) const { + MutexLocker engine_guard(_engine_lock); + return shouldRunRootEnum(now_ns, manual_first_pass); + } + u64 rootEnumIntervalFloorNsForTest() const { + MutexLocker engine_guard(_engine_lock); + return rootEnumIntervalFloorNs(); + } + // Engine lock is the happens-before edge: every reader of _candidate_count/_candidate_found_bits + // runs under _engine_lock (shouldRunPass(), runPass()/runPassSerialized(), pollWatchedTargets()), + // so the plain writes need no atomics. + void setCanaryChaseStateForTest(int candidate_count, u64 found_bits) { + MutexLocker engine_guard(_engine_lock); + _candidate_count = candidate_count; + _candidate_found_bits = found_bits; + } + void setSearchStartedForTest(bool v) { + MutexLocker engine_guard(_engine_lock); + _search_started = v; + } + // Forces a started, RUNNING search with fresh pain budgets so shouldRunPassForTest() evaluates + // the running-search gates deterministically; a concurrently passing BFS thread can still drive + // the real search terminal in the seed-to-call window - tests retry the seed. + // _tags_released stays untouched: the terminal transition recomputes it from releaseSearchTags(); + // forcing it true could let restartSearch() rewind _next_tag over live tags. + void forceRunningSearchForTest() { + MutexLocker engine_guard(_engine_lock); + _search_started = true; + store(_search_state, (u8)SearchState::RUNNING); + store(_abandon_reason, (u8)SearchAbandonReason::NONE); + _cpu_pain_budget = PainBudget(_pain_budget_refill_rate); + _search_pain_ms = 0; + } // OOM-urgency hysteresis state (isUrgent()'s latch/release machine, // referenceChains.cpp): tests drive it via isUrgentForTest() and assert on // the raw latch/tick reads. diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java index 9e406c75c0..306966d983 100644 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java +++ b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java @@ -799,6 +799,63 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l */ static native boolean isUrgentForTest0(); + /** + * Test seam (debug native builds only): backdates the previous pass and sets the effective + * cadence so {@link #shouldRunPassForTest0()} evaluates its minimum-pass-spacing gate without + * sleeping. Values persist until the next call or the next real BFS pass overwrites them. + */ + static native void setReferenceChainPassSpacingForTest0(long msSinceLastPass, + long cadenceMs); + + /** + * Test seam (debug native builds only): sets ReferenceChainTracker's root-enumeration + * scheduling state - the last attempt {@code msSinceLastEnum} milliseconds in the past + * (negative means "never ran") and its measured in-safepoint cost + * {@code lastEnumCostMs}, which the adaptive interval floor extends. Feeds + * {@link #shouldRunRootEnumForTest0(boolean)}. + */ + static native void setReferenceChainRootEnumStateForTest0(long msSinceLastEnum, + long lastEnumCostMs); + + /** + * Test seam (debug native builds only): evaluates whether ReferenceChainTracker's next pass + * would run root/stack-ref enumeration ({@code shouldRunRootEnum()}) with the current + * scheduling state; {@code manualFirstPass} forces the search's always-enumerate first pass. + * Pure predicate - no JVMTI call is made. + */ + static native boolean shouldRunRootEnumForTest0(boolean manualFirstPass); + + /** + * Test seam (debug native builds only): the current adaptive root-enumeration interval floor + * in nanoseconds ({@code ROOT_ENUM_MIN_INTERVAL_NS} extended by the last attempt's measured + * in-safepoint cost, capped) - the value {@link #shouldRunRootEnumForTest0(boolean)} compares + * against. Zero in release builds. + */ + static native long rootEnumIntervalFloorNsForTest0(); + + /** + * Test seam (debug native builds only): the live GC-finish epoch ({@code gcFinishEpoch()}, + * advanced by every JVMTI GarbageCollectionFinish), so a test can verify a real GC was + * observed. Zero in release builds. + */ + static native long referenceChainGcFinishEpochForTest0(); + + /** + * Test seam (debug native builds only): forces the tracker into a started, RUNNING search + * with fresh pain budgets, so {@link #shouldRunPassForTest0()} evaluates the running-search + * scheduling gates deterministically even when the live BFS thread is concurrently running + * (and possibly terminating) its own real search. + */ + static native void forceRunningReferenceChainSearchForTest0(); + + /** + * Test seam (debug native builds only): sets the canary chase's candidate count and + * found-bits mask so {@link #shouldRunPassForTest0()} evaluates the canary lane - with at + * least one candidate and not all found, the gate takes the candidate-chase backoff path. + */ + static native void setReferenceChainCanaryStateForTest0(int candidateCount, + long foundBits); + // ---- Test-only reads of the current thread's OTEP record ---------------------------------- // Each resolves the current carrier's record directly (like the write primitives above) with // no cached buffer and no per-thread Java object; introspection/test use only. diff --git a/ddprof-lib/src/test/cpp/referenceChainsBfsTests.inc b/ddprof-lib/src/test/cpp/referenceChainsBfsTests.inc index 34696f137e..56191a6492 100644 --- a/ddprof-lib/src/test/cpp/referenceChainsBfsTests.inc +++ b/ddprof-lib/src/test/cpp/referenceChainsBfsTests.inc @@ -187,6 +187,245 @@ TEST_F(ReferenceChainsBfsTest, BudgetExhaustionTruncatesAndIsReported) { tracker->stop(); } +// A truncated root enumeration must keep the search alive until a later attempt COMPLETES: the +// truncated attempt records _root_enum_incomplete, which runPass()'s termination ladder treats as +// pending frontier, so neither the plain COMPLETED exit (frontier drained, no watched-leak +// klasses) nor the no-progress abandon (frontier flat while the pass waits out +// rootEnumIntervalFloorNs()) may fire while roots remain unenumerated. This FAILS on unfixed +// code: expansion-only passes see truncated == false and drain/flatline the frontier, so the +// search completes (or abandons after NO_PROGRESS_PASS_LIMIT flat passes) with roots lost. +TEST_F(ReferenceChainsBfsTest, TruncatedRootEnumKeepsSearchAliveUntilEnumCompletes) { + Arguments args; + // firstpassbudget=1: the first enumeration admits one root (nodeA) and truncates. budget=1000: + // the later expansion-only passes drain the frontier quickly, so the flat passes below are + // pure floor-wait passes. + ASSERT_FALSE(args.parse( + "referencechains=true:hops=64:budget=1000:firstpassbudget=1:ttl=0")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int nodeA = addNode(); + int nodeB = addNode(); + int nodeC = addNode(); + // A second root the 1-unit enum budget cannot reach: the first enumeration admits nodeA, + // truncates on nodeD, and records _root_enum_incomplete (a single-root graph would complete + // the enumeration even at budget 1 - ALREADY_ADMITTED roots are uncharged on retry). + int nodeD = addNode(); + + script = { + {JVMTI_HEAP_REFERENCE_JNI_GLOBAL, -1, nodeA, -1}, // admitted via root enum + {JVMTI_HEAP_REFERENCE_JNI_GLOBAL, -1, nodeD, -1}, // unenumerated - truncates the attempt + {JVMTI_HEAP_REFERENCE_FIELD, nodeA, nodeB, -1}, // admitted via expansion + {JVMTI_HEAP_REFERENCE_FIELD, nodeA, nodeC, -1}, + }; + + // The fixture reset does not touch the root-enumeration state: clear any residue. + std::pair saved = ReferenceChainsTestAccessor::setRootEnumRawStateForTest(0, 0); + + // Pass 1 (manual first pass - enumeration runs): budget truncation ends the pass with the + // root picture incomplete. + bool truncated = true; + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_TRUE(truncated); + EXPECT_EQ(SearchState::RUNNING, tracker->searchState()); + + // Freeze the enumeration clock inside the interval floor: every pass below is expansion-only + // (shouldRunRootEnum() holds the enum back) while the flag stays set. + ReferenceChainsTestAccessor::setRootEnumRawStateForTest(OS::nanotime(), 0); + + // Pass 2: expansion-only, admits nodeB/nodeC - the frontier grows, and the search must stay + // RUNNING even though this pass's own truncated flag is false (unfixed code: has_pending_ + // frontier was pass-local, so a drained frontier could complete the search here). + truncated = false; + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_FALSE(truncated); + EXPECT_EQ(SearchState::RUNNING, tracker->searchState()); + + // Passes 3+: expansion-only over a drained, then flat frontier. The no-progress counter must + // be HELD (neither reset nor incremented) on every floor-wait pass - 30+ of them would + // otherwise reach NO_PROGRESS_PASS_LIMIT and abandon the search before the next enumeration + // chunk could run (the second exit the flag closes). + const int kFloorWaitPasses = ReferenceChainTracker::NO_PROGRESS_PASS_LIMIT + 5; + for (int i = 0; i < kFloorWaitPasses; i++) { + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_FALSE(truncated); + EXPECT_EQ(SearchState::RUNNING, tracker->searchState()) + << "pass " << i + << ": a floor-wait pass must not let the search complete or abandon"; + EXPECT_EQ(0, tracker->passesSinceLastProgressForTest()) + << "pass " << i + << ": the no-progress counter must be held while enumeration is incomplete"; + } + + // Let the interval floor elapse: the next pass runs the enumeration to completion (nodeA is + // already admitted and uncharged, so the 1-unit budget reaches nodeD). + ReferenceChainsTestAccessor::setRootEnumRawStateForTest( + OS::nanotime() - ReferenceChainsTestAccessor::rootEnumMinIntervalNs() - 1, 0); + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_FALSE(truncated); + // The completed attempt clears the flag, so the ladder may now reach COMPLETED - and does, + // since the frontier was drained and no watched-leak klass remains. + EXPECT_EQ(SearchState::COMPLETED, tracker->searchState()); + + // Mandatory: the fixture reset does not restore the root-enumeration state. + ReferenceChainsTestAccessor::setRootEnumRawStateForTest(saved.first, saved.second); + tracker->stop(); +} + +// An expired TTL must not abandon a search whose root enumeration is still incomplete: abandoning +// mid-enumeration loses the unenumerated roots, so the ladder defers the TTL abandon until the +// enumeration completes (the clock keeps running - no free window after completion). This FAILS on +// unfixed code: the TTL branch ran first and abandoned while nodeD was still unenumerated. +TEST_F(ReferenceChainsBfsTest, TtlDeferredUntilRootEnumCompletes) { + Arguments args; + ASSERT_FALSE(args.parse( + "referencechains=true:hops=64:budget=1000:firstpassbudget=1:ttl=50")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int nodeA = addNode(); + int nodeB = addNode(); + int nodeC = addNode(); + int nodeD = addNode(); + + script = { + {JVMTI_HEAP_REFERENCE_JNI_GLOBAL, -1, nodeA, -1}, // admitted via root enum + {JVMTI_HEAP_REFERENCE_JNI_GLOBAL, -1, nodeD, -1}, // unenumerated - truncates the attempt + {JVMTI_HEAP_REFERENCE_FIELD, nodeA, nodeB, -1}, // admitted via expansion + {JVMTI_HEAP_REFERENCE_FIELD, nodeA, nodeC, -1}, + }; + + std::pair saved = ReferenceChainsTestAccessor::setRootEnumRawStateForTest(0, 0); + + // Pass 1 (manual first pass): the 1-unit budget truncates on nodeD - _root_enum_incomplete + // set, search RUNNING. + bool truncated = true; + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_TRUE(truncated); + EXPECT_EQ(SearchState::RUNNING, tracker->searchState()); + + // Expire the TTL and freeze the enumeration clock: every pass below is expansion-only + // floor-wait with the TTL long past. + ReferenceChainsTestAccessor::setSearchStartNsForTest(OS::nanotime() - 100000000ULL); + ReferenceChainsTestAccessor::setRootEnumRawStateForTest(OS::nanotime(), 0); + + const int kDeferredPasses = ReferenceChainTracker::NO_PROGRESS_PASS_LIMIT + 5; + for (int i = 0; i < kDeferredPasses; i++) { + truncated = false; + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_EQ(SearchState::RUNNING, tracker->searchState()) + << "pass " << i + << ": an expired TTL must not abandon a search with incomplete enumeration"; + } + + // Unfreeze the enumeration clock: the next pass completes the enumeration, the deferral ends + // (no unbounded extension), and the search settles - COMPLETED here, since the frontier was + // drained during the floor-wait passes and no watched-leak klass remains. + ReferenceChainsTestAccessor::setRootEnumRawStateForTest( + OS::nanotime() - ReferenceChainsTestAccessor::rootEnumMinIntervalNs() - 1, 0); + truncated = false; + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_FALSE(truncated); + EXPECT_EQ(SearchState::COMPLETED, tracker->searchState()); + EXPECT_EQ(SearchAbandonReason::NONE, ReferenceChainsTestAccessor::searchAbandonReasonForTest()); + + ReferenceChainsTestAccessor::setRootEnumRawStateForTest(saved.first, saved.second); + tracker->stop(); +} + +// A persistently failing enumerator must not run forever when ttl=0: the ladder abandons with +// ROOT_ENUM_ERROR once MAX_CONSECUTIVE_ENUM_ERRORS consecutive attempts return a JVMTI error. +// This FAILS on unfixed code: the errored attempts only set _root_enum_incomplete, so with ttl=0 +// no termination path ever fires. +TEST_F(ReferenceChainsBfsTest, ConsecutiveEnumErrorsAbandonSearch) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000:ttl=0")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int nodeA = addNode(); + script = { + {JVMTI_HEAP_REFERENCE_JNI_GLOBAL, -1, nodeA, -1}, + }; + + std::pair saved = ReferenceChainsTestAccessor::setRootEnumRawStateForTest(0, 0); + active_fixture->ior_error = JVMTI_ERROR_OUT_OF_MEMORY; + + // Pass 1 (manual): the enumeration attempt errors - incomplete, counter 1, still RUNNING. + bool truncated = true; + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_TRUE(truncated); + EXPECT_EQ(SearchState::RUNNING, tracker->searchState()); + + // Passes 2..MAX: backdate the floor so each pass runs a real enumeration attempt that errors + // again; the ladder must abandon when the counter reaches MAX_CONSECUTIVE_ENUM_ERRORS. + for (u32 i = 2; i < ReferenceChainTracker::MAX_CONSECUTIVE_ENUM_ERRORS; i++) { + ReferenceChainsTestAccessor::setRootEnumRawStateForTest( + OS::nanotime() - ReferenceChainsTestAccessor::rootEnumMinIntervalNs() - 1, 0); + truncated = false; + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_EQ(SearchState::RUNNING, tracker->searchState()) + << "error " << i << ": below the cap the search must keep running"; + } + ReferenceChainsTestAccessor::setRootEnumRawStateForTest( + OS::nanotime() - ReferenceChainsTestAccessor::rootEnumMinIntervalNs() - 1, 0); + truncated = false; + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_EQ(SearchState::ABANDONED, tracker->searchState()); + EXPECT_EQ(SearchAbandonReason::ROOT_ENUM_ERROR, ReferenceChainsTestAccessor::searchAbandonReasonForTest()); + + ReferenceChainsTestAccessor::setRootEnumRawStateForTest(saved.first, saved.second); + tracker->stop(); +} + +// A successful enumeration attempt - truncated or complete - must reset the consecutive-error +// counter: the cap bounds consecutive FAILURES, not total failures over the search's life. +TEST_F(ReferenceChainsBfsTest, SuccessfulEnumAttemptResetsErrorCounter) { + Arguments args; + ASSERT_FALSE(args.parse( + "referencechains=true:hops=64:budget=1000:firstpassbudget=1:ttl=0")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + int nodeA = addNode(); + int nodeB = addNode(); + int nodeC = addNode(); + int nodeD = addNode(); + + script = { + {JVMTI_HEAP_REFERENCE_JNI_GLOBAL, -1, nodeA, -1}, // admitted; truncates on nodeD + {JVMTI_HEAP_REFERENCE_JNI_GLOBAL, -1, nodeD, -1}, + {JVMTI_HEAP_REFERENCE_FIELD, nodeA, nodeB, -1}, + {JVMTI_HEAP_REFERENCE_FIELD, nodeA, nodeC, -1}, + }; + + std::pair saved = ReferenceChainsTestAccessor::setRootEnumRawStateForTest(0, 0); + + // Seed the counter to one below the cap, then let a successful (truncated) attempt run: it + // resets the counter, so the next single error must NOT abandon (unfixed-on-reset code: the + // seeded 2 + this error would reach the cap). + ReferenceChainsTestAccessor::setConsecutiveEnumErrorsForTest( + ReferenceChainTracker::MAX_CONSECUTIVE_ENUM_ERRORS - 1); + bool truncated = true; + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_TRUE(truncated); // successful JVMTI call, budget truncation - resets the counter + EXPECT_EQ(SearchState::RUNNING, tracker->searchState()); + + // One errored attempt after the reset: counter back to 1 - below the cap, still RUNNING. + ReferenceChainsTestAccessor::setRootEnumRawStateForTest( + OS::nanotime() - ReferenceChainsTestAccessor::rootEnumMinIntervalNs() - 1, 0); + active_fixture->ior_error = JVMTI_ERROR_OUT_OF_MEMORY; + truncated = false; + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + EXPECT_TRUE(truncated); + EXPECT_EQ(SearchState::RUNNING, tracker->searchState()) + << "one error after a successful reset attempt must not abandon"; + + active_fixture->ior_error = JVMTI_ERROR_NONE; + ReferenceChainsTestAccessor::setRootEnumRawStateForTest(saved.first, saved.second); + tracker->stop(); +} + TEST_F(ReferenceChainsBfsTest, PreTaggedClassObjectsAreNeverExpandedOrAdmitted) { Arguments args; ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); @@ -477,8 +716,10 @@ TEST_F(ReferenceChainsBfsTest, MultiPassResumptionReconstructsChainAcrossPasses) Arguments args; // budget=1 forces each pass to admit at most one new frontier entry, so this 3-hop chain cannot // be discovered within a single pass - exercising expandFrontier() (resumed passes), not just - // the first pass's root walk. - ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1:firstpassbudget=1")); + // the first pass's root walk. firstpassbudget is ample so the first enumeration completes + // (clearing _root_enum_incomplete): this test pins expansion resumption, not the truncated- + // enum hold (see TruncatedRootEnumKeepsSearchAliveUntilEnumCompletes). + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1:firstpassbudget=1000")); ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); ASSERT_FALSE(tracker->start(args)); @@ -816,6 +1057,11 @@ TEST_F(ReferenceChainsBfsTest, CanaryStuckLimitDoublesPerRestartAndResetsOnClean LivenessTracker::instance()->setGcGenerationsForTest(false); // 0 = "never activated"; the next start() re-stamps it through the CAS from 0. LivenessTracker::instance()->setLeakSignalActivatedNsForTest(0); + // The seeded rising heap-floor trajectory must not leak into later tests: residual samples + // make heapFloorRising() true and suppress hasLeakSignal() for the next test that arms a + // leak signal (heapFloorResetForTest()'s documented purpose; recording was disabled above, + // the seam's required precondition). + LivenessTracker::instance()->heapFloorResetForTest(); tracker->stop(); ReferenceChainsTestAccessor::resetUrgencyForTest(); // The per-iteration publish (publishUrgencyStateForTest above) set the ramp flag; the latch @@ -858,6 +1104,11 @@ TEST_F(ReferenceChainsBfsTest, CanaryLaneBacksOffWithoutProgressAndResetsOnProgr // progress and the multiplier stays at 1 - back-to-back. ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); EXPECT_EQ(1, ReferenceChainsTestAccessor::canaryBackoffMult()); + // Backdate the previous pass's end past the minimum-spacing gate (shouldRunPass() enforces one + // effective cadence between passes on every lane) so the assertions below exercise the canary + // lane's own backoff, not the spacing gate. + ReferenceChainsTestAccessor::setLastPassNsForTest( + OS::nanotime() - ReferenceChainsTestAccessor::baselineCadenceNs() - 1); EXPECT_TRUE(ReferenceChainsTestAccessor::shouldRunPass(OS::nanotime())) << "a fresh chase must be allowed to run back-to-back"; @@ -866,16 +1117,26 @@ TEST_F(ReferenceChainsBfsTest, CanaryLaneBacksOffWithoutProgressAndResetsOnProgr EXPECT_EQ(2, ReferenceChainsTestAccessor::canaryBackoffMult()); ReferenceChainsTestAccessor::setCanaryBackoffForTest( /*mult=*/2, /*ema_ms=*/100, OS::nanotime()); + ReferenceChainsTestAccessor::setLastPassNsForTest( + OS::nanotime() - ReferenceChainsTestAccessor::baselineCadenceNs() - 1); EXPECT_FALSE(ReferenceChainsTestAccessor::shouldRunPass(OS::nanotime())) << "a no-progress canary pass must hold off the next one"; // Beyond the spacing, the chase is allowed again - the backoff paces, it never abandons. - EXPECT_TRUE(ReferenceChainsTestAccessor::shouldRunPass( - ReferenceChainsTestAccessor::lastCanaryPassNs() + - 2ULL * 100ULL * 1000000ULL + 1)) + u64 resume_ns = ReferenceChainsTestAccessor::lastCanaryPassNs() + + 2ULL * 100ULL * 1000000ULL + 1; + ReferenceChainsTestAccessor::setLastPassNsForTest( + resume_ns - ReferenceChainsTestAccessor::baselineCadenceNs() - 1); + EXPECT_TRUE(ReferenceChainsTestAccessor::shouldRunPass(resume_ns)) << "elapsed spacing must re-admit the canary pass"; - // The OOM urgency ramp overrides the backoff gate entirely. + // The OOM urgency ramp overrides the backoff gate entirely. Mirror the ramp's production + // effect on the cadence (threadLoop() drives _effective_cadence_ns down with urgency) so the + // minimum-spacing gate opens the way it does under a live ramp. ReferenceChainsTestAccessor::setOomRampActive(true); + ReferenceChainsTestAccessor::setEffectiveCadenceNs( + ReferenceChainsTestAccessor::minEffectiveCadenceNs()); + ReferenceChainsTestAccessor::setLastPassNsForTest( + OS::nanotime() - ReferenceChainsTestAccessor::minEffectiveCadenceNs() - 1); EXPECT_TRUE(ReferenceChainsTestAccessor::shouldRunPass(OS::nanotime())) << "urgency must bypass the canary backoff"; ReferenceChainsTestAccessor::setOomRampActive(false); @@ -898,11 +1159,227 @@ TEST_F(ReferenceChainsBfsTest, CanaryLaneBacksOffWithoutProgressAndResetsOnProgr ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); EXPECT_EQ(1, ReferenceChainsTestAccessor::canaryBackoffMult()) << "candidate progress must reset the spacing multiplier to 1"; + ReferenceChainsTestAccessor::setLastPassNsForTest( + OS::nanotime() - ReferenceChainsTestAccessor::baselineCadenceNs() - 1); EXPECT_TRUE(ReferenceChainsTestAccessor::shouldRunPass(OS::nanotime())); tracker->stop(); } +// forceRunningSearchForTest() forces RUNNING for the scheduling tests but must leave +// _tags_released at its real value: that flag is only consulted once the search is terminal, and +// a fabricated "true" would hide a still-pending release and let restartSearch() rewind _next_tag +// over live tags in this process-wide singleton. +TEST_F(ReferenceChainsBfsTest, ForceRunningSearchLeavesTagsReleasedUntouched) { + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + + // Fixture default (_tags_released = true): the seam must not clear it either. + ASSERT_TRUE(ReferenceChainsTestAccessor::tagsReleased()); + tracker->forceRunningSearchForTest(); + EXPECT_TRUE(ReferenceChainsTestAccessor::tagsReleased()); + EXPECT_EQ(SearchState::RUNNING, tracker->searchState()); + + // A RUNNING search with unreleased tags. + ReferenceChainsTestAccessor::setInFlightSearchForTest(OS::nanotime()); + ASSERT_FALSE(ReferenceChainsTestAccessor::tagsReleased()); + tracker->forceRunningSearchForTest(); + EXPECT_FALSE(ReferenceChainsTestAccessor::tagsReleased()) + << "a test seam must never fabricate a completed tag release"; + EXPECT_EQ(SearchState::RUNNING, tracker->searchState()); + // The next test's fixture reset restores _tags_released / the search state. +} + +// _effective_cadence_ns and _last_pass_ns are written by the spacing seams from a test thread +// while shouldRunPass() and updatePacing() read (and write) them on another thread without +// _engine_lock - the shape of the JNI/gtest seam paths. Every such access must be an atomic +// load()/store(); under the tsan native config a plain access reports a data race and fails the +// run. Without TSan this is a smoke test (the cadence must stay sane). +TEST_F(ReferenceChainsBfsTest, PassSpacingSeamsDoNotRaceSchedulingReaders) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); // start() spawns no BFS thread + tracker->forceRunningSearchForTest(); + ReferenceChainsTestAccessor::setCandidateCountForTest(0); + const u64 saved_cadence = ReferenceChainsTestAccessor::effectiveCadenceNs(); + const int saved_budget = ReferenceChainsTestAccessor::effectiveBudget(); + + const u64 deadline = OS::nanotime() + 200ULL * 1000000ULL; + std::atomic writer_done(false); + std::thread writer([&]() { + for (u64 i = 0; i < 20000 && OS::nanotime() < deadline; i++) { + if ((i & 1) == 0) { + tracker->setPassSpacingForTest(i % 3, 1 + i % 5); + } else { + ReferenceChainsTestAccessor::setLastPassNsForTest(OS::nanotime() - i); + } + } + writer_done.store(true); + }); + for (u64 i = 0; !writer_done.load() || i < 100; i++) { + ReferenceChainsTestAccessor::shouldRunPass(OS::nanotime()); + ReferenceChainsTestAccessor::updatePacing(1000000ULL * (i % 50)); + if (OS::nanotime() >= deadline && writer_done.load()) { + break; + } + } + writer.join(); + + EXPECT_GT(ReferenceChainsTestAccessor::effectiveCadenceNs(), 0u); + + ReferenceChainsTestAccessor::setEffectiveCadenceNs(saved_cadence); + ReferenceChainsTestAccessor::setEffectiveBudget(saved_budget); + ReferenceChainsTestAccessor::resetPacingController(); + tracker->stop(); +} + +// runPassManualWalk() converts the root-enumeration call's measured OS::nanotime() delta to ms and +// stores it as the next attempt's cost (_last_root_enum_safepoint_ms); rootEnumIntervalFloorNs() +// extends ROOT_ENUM_MIN_INTERVAL_NS by ROOT_ENUM_INTERVAL_NS_PER_COST_MS per cost-ms, capped at +// ROOT_ENUM_MAX_INTERVAL_NS. A 150ms enumeration must therefore land the floor in [3s, 30s): an +// unscaled ns value saturates the cap, a wrong scale or a dropped assignment leaves it at MIN. +TEST_F(ReferenceChainsBfsTest, RootEnumMeasuredCostExtendsIntervalFloor) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + // The fixture reset does not touch the root-enumeration state: clear any residue. + std::pair saved = ReferenceChainsTestAccessor::setRootEnumRawStateForTest(0, 0); + // Vacuity guard: before any attempt the floor sits at the minimum. + ASSERT_EQ(ReferenceChainsTestAccessor::rootEnumMinIntervalNs(), + tracker->rootEnumIntervalFloorNsForTest()); + + int n = addNode(); + script = { + {JVMTI_HEAP_REFERENCE_JNI_GLOBAL, -1, n, -1}, + }; + + const u64 kRootEnumDelayMs = 150; + ior_delay_ns = kRootEnumDelayMs * 1000000ULL; + bool truncated = true; + // First pass of the search: root enumeration runs (the only path that tags n). + ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); + ior_delay_ns = 0; + EXPECT_NE(0, tags_ever_assigned[n]) << "root enumeration must have run"; + + u64 floor_ns = tracker->rootEnumIntervalFloorNsForTest(); + // >= 150 x ROOT_ENUM_INTERVAL_NS_PER_COST_MS (3s): the ns->ms conversion reached the floor. + EXPECT_GE(floor_ns, kRootEnumDelayMs * ReferenceChainsTestAccessor::rootEnumNsPerCostMs()) + << "measured root-enum cost must extend the interval floor (ns->ms conversion)"; + // > ROOT_ENUM_MIN_INTERVAL_NS: a /1e9 scale or a dropped assignment leaves it at MIN. + EXPECT_GT(floor_ns, ReferenceChainsTestAccessor::rootEnumMinIntervalNs()); + // < ROOT_ENUM_MAX_INTERVAL_NS: an unscaled ns cost would saturate the cap. + EXPECT_LT(floor_ns, ReferenceChainsTestAccessor::rootEnumMaxIntervalNs()) + << "a 150ms attempt must not saturate the cap (unscaled ns would)"; + + ReferenceChainsTestAccessor::setRootEnumRawStateForTest(saved.first, saved.second); + tracker->stop(); +} + +// shouldRunRootEnum()'s interval check is inclusive (now - _last_root_enum_ns >= floor), both at +// the base floor and at a cost-extended floor; the manual first pass bypasses it. +TEST_F(ReferenceChainsBfsTest, RootEnumIntervalBoundaryIsInclusive) { + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + const u64 kMin = ReferenceChainsTestAccessor::rootEnumMinIntervalNs(); + // Far from 0 (the never-ran value) and from wrapping. + const u64 kLast = 100 * kMin; + std::pair saved = ReferenceChainsTestAccessor::setRootEnumRawStateForTest(kLast, 0); + ASSERT_EQ(kMin, tracker->rootEnumIntervalFloorNsForTest()); + + EXPECT_TRUE(tracker->shouldRunRootEnumForTest(kLast + kMin, false)) + << "the base interval boundary must be inclusive (>=, not >)"; + EXPECT_FALSE(tracker->shouldRunRootEnumForTest(kLast + kMin - 1, false)) + << "one ns short of the base interval must still hold enumeration off"; + EXPECT_TRUE(tracker->shouldRunRootEnumForTest(kLast + kMin - 1, true)) + << "the search's first pass must bypass the interval"; + + const u64 kCostMs = 500; + ReferenceChainsTestAccessor::setRootEnumRawStateForTest(kLast, kCostMs); + const u64 kExtended = kCostMs * ReferenceChainsTestAccessor::rootEnumNsPerCostMs(); + ASSERT_EQ(kExtended, tracker->rootEnumIntervalFloorNsForTest()); + EXPECT_TRUE(tracker->shouldRunRootEnumForTest(kLast + kExtended, false)) + << "the cost-extended boundary must be inclusive"; + EXPECT_FALSE(tracker->shouldRunRootEnumForTest(kLast + kExtended - 1, false)) + << "one ns short of the cost-extended interval must still hold enumeration off"; + + // Mandatory: the fixture reset does not restore these fields. + ReferenceChainsTestAccessor::setRootEnumRawStateForTest(saved.first, saved.second); +} + +// canAffordNewSearch() must apply the root-enumeration interval floor to a restarted search's +// first enumeration: restartSearch() deliberately does not reset _last_root_enum_ns, so a +// restart inherits the finished search's last attempt timestamp and pays the same +// rootEnumIntervalFloorNs() as any other IterateOverReachableObjects attempt - closing the +// enumerate -> early-terminate -> restart -> enumerate loop the pain budget alone permitted. +// The brand-new-tracker exemption (_last_root_enum_ns == 0) must survive: an agent starting +// early in the JVM's lifetime must not have its only mandatory enumeration delayed. +TEST_F(ReferenceChainsBfsTest, RestartGateHonorsRootEnumIntervalFloor) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + + // Arm a live leak signal (generations tracking + a growing klass population) so the gate + // reaches the floor check with the leak-signal source active; restored below. The seeded + // population belongs to the CURRENT class map: earlier suite tests mutate the class map and + // leave _last_class_map_generation stale, which makes selectLeakCandidates() select nothing + // (same seam SearchRestartTest::SetUp applies). + LivenessTracker::instance()->setGcGenerationsForTest(true); + LivenessTracker::instance()->classMapGenerationSetForTest( + Profiler::instance()->classMap()->generation()); + int fake_object_storage = 0; + { + int slot; + bool created; + for (u16 i = 1; i <= 20; i++) { + LivenessTracker::instance()->klassPopulationRecordForTest( + /*klass_id=*/1, i, i, &slot, &created); + LivenessTracker::instance()->tidTrendRecordForTest( + /*klass_id=*/1, /*tid=*/4242, (u32)i, (u64)i); + } + LivenessTracker::instance()->klassPopulationSetRepresentativeForTest( + nullptr, /*klass_id=*/1, (jweak)&fake_object_storage); + } + ASSERT_TRUE(ReferenceChainsTestAccessor::hasLeakSignalForTest(3600.0)); + + // The fixture reset does not touch the root-enumeration state, and the suite's earlier + // runPass-driven tests leave a real _last_root_enum_ns behind - clear it so the + // brand-new-tracker premise below actually holds (same seam + // TruncatedRootEnumKeepsSearchAliveUntilEnumCompletes uses). + std::pair saved = + ReferenceChainsTestAccessor::setRootEnumRawStateForTest(0, 0); + + const u64 now = OS::nanotime(); + + // Brand-new tracker (never enumerated, _last_root_enum_ns == 0): the floor is waived. + EXPECT_TRUE(ReferenceChainsTestAccessor::canAffordNewSearch(now)); + + // A recent enumeration attempt inside the floor blocks the restart... + ReferenceChainsTestAccessor::setRootEnumRawStateForTest(now, 0); + EXPECT_FALSE(ReferenceChainsTestAccessor::canAffordNewSearch(now + 1)); + + // ...including at a cost-extended floor (a 150ms last attempt extends the 2s base). + const u64 kCostMs = 150; + ReferenceChainsTestAccessor::setRootEnumRawStateForTest(now, kCostMs); + const u64 kExtendedFloor = + ReferenceChainsTestAccessor::rootEnumMinIntervalNs() > + kCostMs * ReferenceChainsTestAccessor::rootEnumNsPerCostMs() + ? ReferenceChainsTestAccessor::rootEnumMinIntervalNs() + : kCostMs * ReferenceChainsTestAccessor::rootEnumNsPerCostMs(); + EXPECT_FALSE(ReferenceChainsTestAccessor::canAffordNewSearch( + now + kExtendedFloor - 1)); + + // Once the floor elapses, the gate opens again. + EXPECT_TRUE(ReferenceChainsTestAccessor::canAffordNewSearch( + now + kExtendedFloor + 1)); + + // Restore the seeded root-enum state and the no-signal state. + ReferenceChainsTestAccessor::setRootEnumRawStateForTest(saved.first, saved.second); + LivenessTracker::instance()->klassPopulationResetForTest(); + LivenessTracker::instance()->setGcGenerationsForTest(false); + tracker->stop(); +} + // With no leak-signal source active (neither generations nor liveheap // tracking), the collector must stay completely idle: no first pass and no // restart, ever. Searching without a signal runs whole-heap BFS passes with @@ -931,8 +1408,10 @@ TEST_F(ReferenceChainsBfsTest, NoSearchWithoutLeakSignalSource) { TEST_F(ReferenceChainsBfsTest, ChainCompletesWithoutAbandonment) { Arguments args; // budget=1 on a graph where each pass admits exactly one new edge until the chain is exhausted, - // then the frontier stops growing. - ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1:ttl=0:firstpassbudget=1")); + // then the frontier stops growing. firstpassbudget is ample so the first enumeration completes + // (clearing _root_enum_incomplete): this test pins natural completion, not the truncated-enum + // hold (see TruncatedRootEnumKeepsSearchAliveUntilEnumCompletes). + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1:ttl=0:firstpassbudget=1000")); ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); ASSERT_FALSE(tracker->start(args)); @@ -1154,9 +1633,12 @@ TEST_F(ReferenceChainsBfsTest, ResolveOrDropPrunesDeadFrontierEntries) { ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); // pass 2: resolve-or-drop EXPECT_FALSE(truncated); - // The dead branch was pruned for free - with nothing else pending, the search completes rather - // than staying RUNNING or being ABANDONED. - EXPECT_EQ(SearchState::COMPLETED, tracker->searchState()); + // The dead branch was pruned for free - but the search stays RUNNING, not COMPLETED: the + // first enumeration truncated (firstpassbudget=1 never reached decoyRoot), so + // _root_enum_incomplete is still set and the unenumerated root keeps the search alive until + // the next enumeration attempt completes. Completing here instead would silently drop every + // chain reachable only from decoyRoot - the premature-termination defect this flag closes. + EXPECT_EQ(SearchState::RUNNING, tracker->searchState()); EXPECT_EQ(2, tracker->passesRun()); FrontierEntry entry{}; diff --git a/ddprof-lib/src/test/cpp/referenceChainsLifecycleTests.inc b/ddprof-lib/src/test/cpp/referenceChainsLifecycleTests.inc index e1b2e8a68d..3f22f46d0d 100644 --- a/ddprof-lib/src/test/cpp/referenceChainsLifecycleTests.inc +++ b/ddprof-lib/src/test/cpp/referenceChainsLifecycleTests.inc @@ -46,11 +46,16 @@ protected: TEST_F(ReferenceChainsLifecycleTest, ChurnDuringConcurrentPassesStaysConsistent) { Arguments args; - // budget=2 with a first-pass budget of 2 keeps every search multi-pass - // (the graph does not fit in one pass's expand budget), so passes are - // actually in flight while the churn threads run. + // budget=2 keeps every search multi-pass (the graph does not fit in one + // pass's expand budget), so passes are actually in flight while the churn + // threads run. firstpassbudget must be ample so the root enumeration can + // COMPLETE: under the R4/R5 termination law a search may not terminate + // while roots remain unenumerated (_root_enum_incomplete), and with a + // 2-unit enum budget every attempt truncates - no terminal outcome could + // ever be reached (same adaptation as MultiPassResumptionReconstructsChain + // AcrossPasses / ChainCompletesWithoutAbandonment in referenceChainsBfsTests.inc). ASSERT_FALSE(args.parse( - "referencechains=true:hops=64:budget=2:firstpassbudget=2")); + "referencechains=true:hops=64:budget=2:firstpassbudget=1000")); ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); ASSERT_FALSE(tracker->start(args)); @@ -80,6 +85,11 @@ TEST_F(ReferenceChainsLifecycleTest, ChurnDuringConcurrentPassesStaysConsistent) constexpr int kGcCycles = 200; constexpr int kAbortCycles = 30; + // Consecutive RUNNING passes before the pass thread sleeps out the root-enumeration + // interval floor (see the RUNNING branch below for why). + constexpr int kRunningStreakBeforeEnumFloorWait = 200; + int running_streak = 0; + std::atomic pass_done{false}; std::atomic restarts{0}; std::atomic aborted_passes{0}; @@ -136,8 +146,22 @@ TEST_F(ReferenceChainsLifecycleTest, ChurnDuringConcurrentPassesStaysConsistent) } u8 state = ReferenceChainsTestAccessor::searchStateForTest(); if (state == SearchState::RUNNING) { + // R4/R5: while _root_enum_incomplete, a search may not terminate until a + // root-enumeration attempt completes. The floor is wall-clock + // (rootEnumMinIntervalNs()) and this loop drives runPass() back-to-back, so + // once pass 1's window-aborted enumeration set the flag, only a later pass + // run after the floor elapses can clear it. Sleep the floor out periodically + // (only while no abort window is open) so the search can reach the terminal + // outcome the restarts guard below requires; the abort windows exhaust after + // kAbortCycles, so the retries are bounded. + if (++running_streak >= kRunningStreakBeforeEnumFloorWait && + !ReferenceChainsTestAccessor::abortPassRequested()) { + running_streak = 0; + usleep(ReferenceChainsTestAccessor::rootEnumMinIntervalNs() / 1000 + 1000); + } continue; } + running_streak = 0; // Terminal outcome. runPass()'s terminal branch retries the tag // release on the NEXT call - mirror threadLoop()/shouldRunPass(): // only restart once the release is confirmed. diff --git a/ddprof-lib/src/test/cpp/referenceChainsMockHeap.h b/ddprof-lib/src/test/cpp/referenceChainsMockHeap.h index bd0c4b2d95..62bb0a4e09 100644 --- a/ddprof-lib/src/test/cpp/referenceChainsMockHeap.h +++ b/ddprof-lib/src/test/cpp/referenceChainsMockHeap.h @@ -97,6 +97,15 @@ class ReferenceChainsMockHeap { // When non-zero, mock_GetObjectsWithTags() below busy-waits this many nanoseconds. u64 gotw_delay_ns = 0; + // When non-zero, mock_IterateOverReachableObjects() below busy-waits this many nanoseconds + // before reporting roots - a deterministic in-safepoint cost for root enumeration, measured + // by runPassManualWalk() into _last_root_enum_safepoint_ms. + u64 ior_delay_ns = 0; + + // When set to a JVMTI error, mock_IterateOverReachableObjects() returns it instead of running + // the script - drives runPassManualWalk()'s root_err != JVMTI_ERROR_NONE path. + jvmtiError ior_error = JVMTI_ERROR_NONE; + // Synthetic frontier-holder arrays for expandFrontier()'s array-holder walk: // mock_NewObjectArray() hands back an opaque handle, mock_SetObjectArrayElement() records its // elements here, and mock_FollowReferences() treats every recorded element as an expansion seed @@ -428,6 +437,15 @@ class ReferenceChainsMockHeap { jvmtiEnv *, jvmtiHeapRootCallback heap_root_cb, jvmtiStackReferenceCallback, jvmtiObjectReferenceCallback, const void *user_data) { + if (active_fixture->ior_delay_ns != 0) { + u64 until = OS::nanotime() + active_fixture->ior_delay_ns; + while (OS::nanotime() < until) { + // busy-wait (see gotw_delay_ns): the measured duration must not undershoot. + } + } + if (active_fixture->ior_error != JVMTI_ERROR_NONE) { + return active_fixture->ior_error; + } for (auto &e : active_fixture->script) { if (e.referrer_idx != -1) { continue; diff --git a/ddprof-lib/src/test/cpp/referenceChainsOomTests.inc b/ddprof-lib/src/test/cpp/referenceChainsOomTests.inc index f65a361a28..8b6deb409e 100644 --- a/ddprof-lib/src/test/cpp/referenceChainsOomTests.inc +++ b/ddprof-lib/src/test/cpp/referenceChainsOomTests.inc @@ -579,6 +579,111 @@ TEST_F(OomUrgencyTest, EmergencyBandEntrySeedsGrowthBaseline) { EXPECT_FALSE(ReferenceChainsTestAccessor::updateUrgentAdmissionForTest(400.0, true)); } +#ifdef DEBUG +// Context for the heap-floor read hook (setHeapFloorReadHookForTest()): each +// invocation records one full ring wrap of samples between the reader's +// payload load and its validating seq re-load; fire_once limits that to the +// first invocation. +struct HeapFloorReadHookCtx { + int calls = 0; // hook invocations (one per reader attempt) + int records = 0; // invocations that actually recorded a ring wrap + bool fire_once = false; + u64 clock = 0; +}; + +static void heapFloorReadHookThunk(void *arg) { + HeapFloorReadHookCtx *ctx = static_cast(arg); + int n = ctx->calls++; + if (ctx->fire_once && n > 0) { + return; + } + ctx->records++; + LivenessTracker *lt = LivenessTracker::instance(); + // A full wrap returns the head to the slot the reader is holding - the + // torn-read interleaving the gate must survive. + constexpr int RING_SIZE = LivenessTracker::heapFloorRingSizeForTest(); + constexpr u64 WRAP_BASE = 1100ULL << 20; + for (int i = 0; i < RING_SIZE; i++) { + lt->heapFloorRecordForTest(WRAP_BASE + ((u64)i << 20), ctx->clock, 0); + ctx->clock += 1; + } +} +#endif + +// Emergency-band entry while newestHeapFloorSample() exhausts its retry bound +// (an unstable read returns 0): the exhausted read's 0 must NOT be consumed as +// a seed - the baseline stays the unseeded sentinel 0, so the next +// emergency-band iteration retries and seeds from a good read. The emergency +// path additionally never touches the freshness pair +// (_admission_sample_seq / _admission_decision). +TEST_F(OomUrgencyTest, EmergencyBandEntryWithUnstableReadLeavesBaselineUnseeded) { +#ifdef DEBUG + LivenessTracker *lt = LivenessTracker::instance(); + // The fixture's SetUp does NOT clear the heap-floor ring: residual samples + // would poison this scenario's trajectory and leak into later ones. + lt->heapFloorResetForTest(); + // The hook must never outlive this test, on any exit path. + struct ClearHook { + ~ClearHook() { LivenessTracker::setHeapFloorReadHookForTest(nullptr, nullptr); } + } clear_hook; + + // Record one good sample BEFORE installing the hook: with fill == 0 + // newestHeapFloorSample() returns 0 on attempt 0 before the DEBUG hook + // fires (an empty-ring early return consumes no retry attempts), so an + // empty ring would silently exercise the empty-ring path instead of the + // retry-bound exhaustion this test pins. + lt->heapFloorRecordForTest(1000 * MiB, _clock_ns, 0); + _clock_ns += SAMPLE_SPACING_S * SEC_NS; + + // Every reader attempt now wraps the ring, so the seed read exhausts + // HEAP_FLOOR_READ_ATTEMPTS and reports no consistent sample. + HeapFloorReadHookCtx ctx; + LivenessTracker::setHeapFloorReadHookForTest(heapFloorReadHookThunk, &ctx); + + // The counter is global and monotone within the binary: always capture a + // local baseline, never assume 0. + long long unstable0 = Counters::getCounter(HEAP_FLOOR_READ_UNSTABLE); + + // The emergency band must admit without growth evidence even when the + // seed read fails. + EXPECT_TRUE(ReferenceChainsTestAccessor::updateUrgentAdmissionForTest(299.0, true)); + ASSERT_EQ(LivenessTracker::HEAP_FLOOR_READ_ATTEMPTS, ctx.records) + << "the seed read must have been attempted and exhausted its retry bound"; + EXPECT_EQ(unstable0 + 1, Counters::getCounter(HEAP_FLOOR_READ_UNSTABLE)) + << "the exhausted read must be counted exactly once, even with seq_out == nullptr"; + // The exhausted read's 0 is not seeded: a readable 1000 * MiB sample sits + // in the ring while the baseline stays the unseeded sentinel 0, so the + // observed 0 cannot be the seed of any readable sample. + EXPECT_EQ(0u, ReferenceChainsTestAccessor::admissionGrowthBaselineForTest()) + << "the exhausted read's 0 must not be seeded: the baseline stays the unseeded " + "sentinel so the next emergency iteration retries"; + // Emergency admission is not growth evidence: the freshness pair stays untouched. + EXPECT_EQ(0u, ReferenceChainsTestAccessor::admissionSampleSeqForTest()); + + // Next emergency-band iteration retries: with a coherent sample available, + // the seed succeeds - proving the earlier 0 read was never counted as a seed. + LivenessTracker::setHeapFloorReadHookForTest(nullptr, nullptr); + lt->heapFloorResetForTest(); + lt->heapFloorRecordForTest(1000 * MiB, _clock_ns, 0); + _clock_ns += SAMPLE_SPACING_S * SEC_NS; + EXPECT_TRUE(ReferenceChainsTestAccessor::updateUrgentAdmissionForTest(299.0, true)); + EXPECT_EQ(1000 * MiB, ReferenceChainsTestAccessor::admissionGrowthBaselineForTest()) + << "the retry must seed from the good read, proving the earlier 0 was never a seed"; + + // The seed drives the ramp-band gate: growth past it admits, comparing + // against the seeded baseline rather than re-snapshotting (same semantics + // EmergencyBandEntrySeedsGrowthBaseline pins). + LivenessTracker::instance()->heapFloorRecordForTest(1000 * MiB + RATE, _clock_ns, 0); + _clock_ns += SAMPLE_SPACING_S * SEC_NS; + EXPECT_TRUE(ReferenceChainsTestAccessor::updateUrgentAdmissionForTest(400.0, true)) + << "ramp-band observation must compare against the seeded baseline, not re-snapshot"; + + lt->heapFloorResetForTest(); +#else + GTEST_SKIP() << "the heap-floor read hook is DEBUG-only"; +#endif +} + // Boundary of the emergency band: a projection of exactly OOM_URGENT_THRESHOLD_S // (300s) is ramp band, not emergency. updateUrgentAdmission() keeps the // growth-evidence hold (first observation snapshots and does not admit; only a @@ -925,37 +1030,6 @@ TEST_F(OomUrgencyTest, HeapFloorSequenceIsMonotonicAcrossResets) { lt->heapFloorResetForTest(); } -#ifdef DEBUG -// Context for the heap-floor read hook (setHeapFloorReadHookForTest()): each -// invocation records one full ring wrap of samples between the reader's -// payload load and its validating seq re-load; fire_once limits that to the -// first invocation. -struct HeapFloorReadHookCtx { - int calls = 0; // hook invocations (one per reader attempt) - int records = 0; // invocations that actually recorded a ring wrap - bool fire_once = false; - u64 clock = 0; -}; - -static void heapFloorReadHookThunk(void *arg) { - HeapFloorReadHookCtx *ctx = static_cast(arg); - int n = ctx->calls++; - if (ctx->fire_once && n > 0) { - return; - } - ctx->records++; - LivenessTracker *lt = LivenessTracker::instance(); - // A full wrap returns the head to the slot the reader is holding - the - // torn-read interleaving the gate must survive. - constexpr int RING_SIZE = LivenessTracker::heapFloorRingSizeForTest(); - constexpr u64 WRAP_BASE = 1100ULL << 20; - for (int i = 0; i < RING_SIZE; i++) { - lt->heapFloorRecordForTest(WRAP_BASE + ((u64)i << 20), ctx->clock, 0); - ctx->clock += 1; - } -} -#endif - // When the read hook interleaves a full ring wrap inside newestHeapFloorSample() // (the torn-read window), the admission gate must evaluate - and consume the // seq of - only a coherent pair. A torn read would pair sample 1's payload @@ -1357,10 +1431,15 @@ TEST_F(ReferenceChainsCanaryRefillTest, RefillRaiseTracksOpenChaseExactly) { // The refill logic lives past shouldRunPass()'s first-pass branch, so // mark the search started the way a first runPass() would have. ReferenceChainsTestAccessor::setSearchStartedForTest(true); + // Synthetic now values below start just past the fixed MIN_PASS_SPACING_NS floor; shrink the + // cadence too, so the minimum-spacing gate (now_ns - _last_pass_ns >= max(_effective_cadence_ns, + // MIN_PASS_SPACING_NS), with _last_pass_ns = 0 after reset) stays open and the refill + // assertions exercise the canary lane, not the spacing gate. + ReferenceChainsTestAccessor::setEffectiveCadenceNs(100); double base = ReferenceChainsTestAccessor::basePainBudgetRefillRateForTest(); EXPECT_DOUBLE_EQ(0.01, base); // default painbudget=1 percent - u64 now = 1000; + u64 now = ReferenceChainsTestAccessor::minPassSpacingNs() + 1000; // No candidates: the applied rate is the configured one. The return // value here is cadence-dependent, not part of the invariant - only the // applied refill rate is. @@ -1391,6 +1470,41 @@ TEST_F(ReferenceChainsCanaryRefillTest, RefillRaiseTracksOpenChaseExactly) { ReferenceChainsTestAccessor::reset(); } +// The refill raise must be applied BEFORE the minimum-pass-spacing gate +// returns: the gate's induced wait drains _cpu_pain_budget, so a spacing-closed +// evaluation with a chase open must still apply the chase's rate (reverting the +// ordering leaves the rate at its base here). +TEST_F(ReferenceChainsCanaryRefillTest, RefillRaiseAppliesBeforeSpacingGate) { + ReferenceChainsTestAccessor::reset(); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true")); + ASSERT_FALSE(tracker->start(args)); + + ReferenceChainsTestAccessor::setSearchStartedForTest(true); + double base = ReferenceChainsTestAccessor::basePainBudgetRefillRateForTest(); + + // Chase open, spacing gate closed: the gate holds the pass, but the rate + // must already be raised - the wait it induces drains at the current rate. + ReferenceChainsTestAccessor::setCandidateCountForTest(2); + ReferenceChainsTestAccessor::setCandidateFoundBitsForTest(0x1); + u64 now = 1000; + ReferenceChainsTestAccessor::setLastPassNsForTest(now); // now - last = 0 < cadence + EXPECT_FALSE(ReferenceChainsTestAccessor::shouldRunPass(now)) + << "the spacing gate must hold this pass"; + EXPECT_DOUBLE_EQ(std::min(base * ReferenceChainTracker::CANARY_PAIN_BUDGET_REFILL_MULTIPLIER, 1.0), + ReferenceChainsTestAccessor::cpuPainBudgetRefillRateForTest()); + + // Sanity: with the gate reopened the chase lane authorizes and the rate stays raised. + now += 1000000000ULL; + EXPECT_TRUE(ReferenceChainsTestAccessor::shouldRunPass(now)); + EXPECT_DOUBLE_EQ(std::min(base * ReferenceChainTracker::CANARY_PAIN_BUDGET_REFILL_MULTIPLIER, 1.0), + ReferenceChainsTestAccessor::cpuPainBudgetRefillRateForTest()); + + tracker->stop(); + ReferenceChainsTestAccessor::reset(); +} + // stop() ends the urgency episode at a recording boundary, so it must clear // updateUrgentAdmission()'s growth baseline along with the latch: the // heap-floor ring survives the boundary, and a baseline left over from the diff --git a/ddprof-lib/src/test/cpp/referenceChainsTestAccessors.h b/ddprof-lib/src/test/cpp/referenceChainsTestAccessors.h index 37082defe2..04ed4ceab6 100644 --- a/ddprof-lib/src/test/cpp/referenceChainsTestAccessors.h +++ b/ddprof-lib/src/test/cpp/referenceChainsTestAccessors.h @@ -14,6 +14,7 @@ #include #include +#include #include #include "classTagAllocator.h" @@ -53,6 +54,8 @@ class ReferenceChainsTestAccessor { ClassTagAllocator::resetForTest(); t->_search_started = false; t->_tags_released = true; + t->_root_enum_incomplete = false; + t->_consecutive_enum_errors = 0; t->_search_state = SearchState::RUNNING; t->_abandon_reason = SearchAbandonReason::NONE; t->_search_start_ns = 0; @@ -69,7 +72,6 @@ class ReferenceChainsTestAccessor { t->_static_anchor_fifo_set.clear(); t->_static_anchor_fifo_klass_counts.clear(); t->_static_anchor_fresh_queue.clear(); - t->_last_pass_gc_finish_epoch = 0; t->_last_pass_ns = 0; t->_passes_run = 0; t->_passes_since_last_progress = 0; @@ -200,6 +202,12 @@ class ReferenceChainsTestAccessor { static bool admissionDecisionForTest() { return ReferenceChainTracker::instance()->_admission_decision; } + // updateUrgentAdmission()'s emergency-band seed anchor: the newest heap-floor + // sample captured at emergency-band entry (referenceChains.cpp), or 0 (the + // unseeded sentinel) while no seed has been taken. + static u64 admissionGrowthBaselineForTest() { + return ReferenceChainTracker::instance()->_admission_growth_baseline; + } // True while the BFS thread (startThread()/threadLoop()) is live. start() never spawns it // (only startThread() does - profiler.cpp and the JNI-only resetSearchStateForTest()), so // gtests may drive BFS-thread-only seams after start(); tests that rely on that assert this @@ -237,6 +245,14 @@ class ReferenceChainsTestAccessor { ReferenceChainTracker::instance()->setSearchStartedForTest(v); } + // Backdates the previous pass's end so the minimum-spacing gate stays open while a test + // exercises one authorization lane (canary backoff, GC epoch) in isolation. Goes through the + // tracker seam (engine lock + store(), ordered with any pass); a real pass can still overwrite + // the seed between seed and assert - tests retry, same as forceRunningSearchForTest(). + static void setLastPassNsForTest(u64 ns) { + ReferenceChainTracker::instance()->setLastPassNsForTest(ns); + } + static u64 searchPainMs() { return ReferenceChainTracker::instance()->_search_pain_ms; } @@ -417,6 +433,10 @@ class ReferenceChainsTestAccessor { return ReferenceChainTracker::instance()->_search_start_ns; } + static void setConsecutiveEnumErrorsForTest(u32 n) { + ReferenceChainTracker::instance()->_consecutive_enum_errors = n; + } + static size_t pendingAbandonedEventCountForTest() { ReferenceChainTracker *t = ReferenceChainTracker::instance(); t->_pending_abandoned_events_lock.lock(); @@ -673,6 +693,16 @@ class ReferenceChainsTestAccessor { t->_effective_pause_target_ms = configured_ms; } + static u64 minEffectiveCadenceNs() { + return ReferenceChainTracker::MIN_EFFECTIVE_CADENCE_NS; + } + + // shouldRunPass()'s fixed minimum pass-spacing floor (referenceChains.h) - see the + // shouldRunPass pacing law comment there. + static u64 minPassSpacingNs() { + return ReferenceChainTracker::MIN_PASS_SPACING_NS; + } + // Start the controller from a chosen below-ceiling/above-baseline point with a fresh // _pause_pid, so a prior sequence's integral windup cannot leak into per-step assertions. static void setEffectiveBudget(int v) { @@ -687,6 +717,37 @@ class ReferenceChainsTestAccessor { ReferenceChainTracker::instance()->_pause_pid.reset(); } + // Root-enumeration raw state (shouldRunRootEnum()'s inputs) at exact values - unlike the + // tracker's setRootEnumStateForTest(), which derives the timestamp from OS::nanotime() and + // so cannot pin a boundary. Engine lock held (static accessor: take it explicitly). Returns + // the previous pair so the caller can restore it - the fixture reset does not zero these. + static std::pair setRootEnumRawStateForTest(u64 last_enum_ns, u64 cost_ms) { + ReferenceChainTracker *t = ReferenceChainTracker::instance(); + MutexLocker engine_guard(t->_engine_lock); + std::pair prev(t->_last_root_enum_ns, t->_last_root_enum_safepoint_ms); + t->_last_root_enum_ns = last_enum_ns; + t->_last_root_enum_safepoint_ms = cost_ms; + return prev; + } + + static std::pair rootEnumRawStateForTest() { + ReferenceChainTracker *t = ReferenceChainTracker::instance(); + MutexLocker engine_guard(t->_engine_lock); + return std::pair(t->_last_root_enum_ns, t->_last_root_enum_safepoint_ms); + } + + static u64 rootEnumMinIntervalNs() { return ReferenceChainTracker::ROOT_ENUM_MIN_INTERVAL_NS; } + static u64 rootEnumNsPerCostMs() { + return ReferenceChainTracker::ROOT_ENUM_INTERVAL_NS_PER_COST_MS; + } + static u64 rootEnumMaxIntervalNs() { return ReferenceChainTracker::ROOT_ENUM_MAX_INTERVAL_NS; } + + // Live GC-finish epoch (relaxed atomic) - save/restore it; the fixture reset does not touch + // _gc_finish_epoch (only GC callbacks advance it). + static void setGcFinishEpochForTest(u64 v) { + store(ReferenceChainTracker::instance()->_gc_finish_epoch, v); + } + // Budget-borrowing (see _borrowed_budget in referenceChains.h). static int borrowCeilingMultiplier() { return ReferenceChainTracker::BORROW_CEILING_MULTIPLIER; diff --git a/ddprof-stresstest/src/repro/java/com/datadoghq/profiler/repro/ReferenceChainLeakDemo.java b/ddprof-stresstest/src/repro/java/com/datadoghq/profiler/repro/ReferenceChainLeakDemo.java index 9415bb57a1..5547ff4ac9 100644 --- a/ddprof-stresstest/src/repro/java/com/datadoghq/profiler/repro/ReferenceChainLeakDemo.java +++ b/ddprof-stresstest/src/repro/java/com/datadoghq/profiler/repro/ReferenceChainLeakDemo.java @@ -56,6 +56,13 @@ * anywhere near this app's {@code CachedEntry} population. Pass {@code firstpassbudget=N} * explicitly only to override that auto-scaled default. * + *

Optional escape-analysis spin workers: setting {@code REFCHAINS_EASPIN=} + * (e.g. {@code REFCHAINS_EASPIN=4}) starts worker threads that hold compiled frames with + * scalar-replaced (EA-eliminated) locals on stack while the tracker runs, so the root + * enumeration's forced deoptimizations (see {@link #spinLoop}) become observable in the + * recording. Workers are full-core busy loops - size the count well below the core count when + * comparing safepoint/GC timings (exit semantics in {@link #startEaSpinWorkers}). + * *

Grows an ever-referenced, never-evicted {@code HashMap}-backed cache (same shape as * ddprof-test's {@code LeakingCacheScenario}). Two deliberate choices, both found by running * without them and never seeing a chain get discovered: @@ -181,6 +188,8 @@ private static void run(Map cache, Path recordingPath, Stri long startNanos = System.nanoTime(); long deadlineNanos = durationSeconds >= 0 ? startNanos + durationSeconds * 1_000_000_000L : Long.MAX_VALUE; + + startEaSpinWorkers(deadlineNanos); long startHeapUsedMb = heapUsedMb(); long nextKey = 300; @@ -241,6 +250,9 @@ private static void run(Map cache, Path recordingPath, Stri Thread.sleep(roundIntervalMs); } + // Wind down the EA spin workers (no-op if none started) before the final dump/metrics. + spinStop = true; + if (durationSeconds >= 0) { profiler.dump(snapshotPath); System.out.println("[repro] dumped " + snapshotPath); @@ -309,6 +321,89 @@ private static String pct(double fraction) { return String.format("%.1f%%", fraction * 100.0); } + /** + * Escape-analysis spin workers ({@code REFCHAINS_EASPIN=} threads, default 0): while a + * worker sits in {@link #spinLoop}, the root enumeration's + * {@code EscapeBarrier::deoptimize_objects_all_threads} deoptimizes its compiled frame + * ({@code has_ea_local_in_scope()}). A forced deopt makes the nmethod not-entrant; + * re-invoking the method re-compiles it and re-applies EA, so deopts recur for the whole run. + * + *

Each worker is a full-core busy loop. In a bounded run it exits within one + * {@code spinLoop} trip once {@code deadlineNanos} passes or the main loop ends + * ({@link #spinStop}), so the final dump/metrics are not perturbed; in the default + * run-forever mode ({@code deadlineNanos == Long.MAX_VALUE}) it runs until JVM exit (daemon), + * matching the main loop. + */ + private static void startEaSpinWorkers(final long deadlineNanos) { + int threads; + String raw = System.getenv("REFCHAINS_EASPIN"); + if (raw == null || raw.trim().isEmpty()) { + return; + } + try { + threads = Integer.parseInt(raw.trim()); + } catch (NumberFormatException e) { + System.err.println("[repro] invalid REFCHAINS_EASPIN='" + raw + "' - starting no spin workers"); + return; + } + for (int t = 0; t < threads; t++) { + final long seed = 0x51ED270B + t * 0x9E3779B97F4A7C15L; + Thread th = new Thread(() -> { + // Re-checked once per spinLoop trip, never inside spinLoop - its body shape is + // load-bearing for the EA deopt target - so exit latency is at most one trip. + while (!spinStop && System.nanoTime() < deadlineNanos) { + spinLoop(seed, SPIN_TRIP_COUNT, SPIN_SALT); + } + }, "easpin-" + t); + th.setDaemon(true); + th.start(); + } + System.out.println("[repro] started " + threads + " EA spin workers (one busy core each, " + + (deadlineNanos == Long.MAX_VALUE ? "until JVM exit" : "until the run deadline") + + ") - root enumerations should now deoptimize their compiled frames"); + } + + private static final long SPIN_TRIP_COUNT = 300_000_000L; + private static final long SPIN_SALT = 0x27D4EB2F165667C5L; + + // Set once run()'s main loop exits; spin workers check it between spinLoop trips. In bounded + // mode the main loop exits at the same deadline the workers check, so the flag only shortens + // the window by at most one trip; in run-forever mode the main loop never exits, so the flag + // is never set and the workers stop at JVM exit. + private static volatile boolean spinStop; + + /** + * One field, one constructor argument, so C2 scalar-replaces the instance - the shape + * {@code has_ea_local_in_scope()} reports. Identity hashCode, monitors, or a non-inlined + * call taking the object force a heap allocation and remove the deopt target. + */ + private static final class EaLocal { + long f; + + EaLocal(long f) { + this.f = f; + } + } + + /** + * Loop body shaped for a reliable deopt target: one {@code EaLocal} live across every + * iteration (at any safepoint inside the loop the compiled scope holds the scalar-replaced + * local), a conditional {@code p.f} write so the read cannot be hoisted away, and a salt-xor + * so the accumulator cannot be evaluated symbolically and the loop must actually run. + */ + private static long spinLoop(long seed, long iterations, long salt) { + EaLocal p = new EaLocal(seed ^ salt); + long acc = seed; + for (long i = 0; i < iterations; i++) { + acc += p.f; + acc ^= salt; + if ((i & 0xFFFF) == 0) { + p.f = acc; + } + } + return acc + p.f; + } + /** * Same shape as {@code LeakingCacheScenario.CachedPayload}/{@code * ReferenceChainLeakAntagonist.CachedEntry}: plain long fields, not a nested array, so a diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/JavaProfilerApiSurfaceTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/JavaProfilerApiSurfaceTest.java index 0a9883b2f7..b469de741b 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/JavaProfilerApiSurfaceTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/JavaProfilerApiSurfaceTest.java @@ -40,7 +40,11 @@ public void ownedBlockHooksAreNotPublicApiBeforeTaskBlockInstrumentation() throw "heapFloorResetForTest0", "heapFloorRisingForTest0", "shouldRunPassForTest0", "referenceChainPassesRunForTest0", "backdateLeakSignalActivationForTest0", "setLeakSignalActivatedNsForTest0", - "secondsToOOMForTest0", "urgentLatchedForTest0", "isUrgentForTest0")); + "secondsToOOMForTest0", "urgentLatchedForTest0", "isUrgentForTest0", + "setReferenceChainPassSpacingForTest0", + "setReferenceChainRootEnumStateForTest0", "shouldRunRootEnumForTest0", + "rootEnumIntervalFloorNsForTest0", "referenceChainGcFinishEpochForTest0", + "setReferenceChainCanaryStateForTest0", "forceRunningReferenceChainSearchForTest0")); @Test public void referenceChainTestSeamsAreNotPublicApi() { diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/ReferenceChainTestHooks.java b/ddprof-test/src/test/java/com/datadoghq/profiler/ReferenceChainTestHooks.java index 9a96808345..dd8f055e1b 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/ReferenceChainTestHooks.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/ReferenceChainTestHooks.java @@ -50,6 +50,34 @@ public static boolean runReferenceChainPass0() { return JavaProfiler.runReferenceChainPass0(); } + public static void setReferenceChainPassSpacingForTest0(long msSinceLastPass, long cadenceMs) { + JavaProfiler.setReferenceChainPassSpacingForTest0(msSinceLastPass, cadenceMs); + } + + public static void setReferenceChainRootEnumStateForTest0(long msSinceLastEnum, long lastEnumCostMs) { + JavaProfiler.setReferenceChainRootEnumStateForTest0(msSinceLastEnum, lastEnumCostMs); + } + + public static boolean shouldRunRootEnumForTest0(boolean manualFirstPass) { + return JavaProfiler.shouldRunRootEnumForTest0(manualFirstPass); + } + + public static long rootEnumIntervalFloorNsForTest0() { + return JavaProfiler.rootEnumIntervalFloorNsForTest0(); + } + + public static long referenceChainGcFinishEpochForTest0() { + return JavaProfiler.referenceChainGcFinishEpochForTest0(); + } + + public static void forceRunningReferenceChainSearchForTest0() { + JavaProfiler.forceRunningReferenceChainSearchForTest0(); + } + + public static void setReferenceChainCanaryStateForTest0(int candidateCount, long foundBits) { + JavaProfiler.setReferenceChainCanaryStateForTest0(candidateCount, foundBits); + } + public static void pollReferenceChainTargets0() { JavaProfiler.pollReferenceChainTargets0(); } diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/AggressiveLeakReferenceChainTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/AggressiveLeakReferenceChainTest.java index 38de748a9f..9ce0415a0f 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/AggressiveLeakReferenceChainTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/AggressiveLeakReferenceChainTest.java @@ -15,6 +15,7 @@ import java.util.Arrays; import java.util.function.Supplier; +import static com.datadoghq.profiler.referencechains.ReferenceChainAssertions.assumeDebugBuild; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -85,12 +86,6 @@ protected boolean isPlatformSupported() { return !(Platform.isJavaVersion(8) || Platform.isJ9() || Platform.isZing()); } - private static void assumeDebugBuild() { - assumeTrue(ReferenceChainAssertions.isDebugInstrumentedBuild(), - "heapFloorRecordForTest0/setMaxHeapBytesForTest0/shouldRunPassForTest0 only exist in a " - + "-DDEBUG native build (debug/asan/tsan - javaApi.cpp's #ifdef DEBUG guard)"); - } - /** * Undoes this class's synthetic seeding so later tests in this shared, no-forkEvery JVM never * see it. This helper is the only place the cleanup ordering lives; every test's finally block @@ -140,14 +135,21 @@ private static GateEvaluation evaluateGateAfterSearchReset() { // candidateSeen bracket honest. int[] candidatesBefore = ReferenceChainTestHooks.selectLeakCandidateKlassIds0(); ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); - int passesBefore = ReferenceChainTestHooks.referenceChainPassesRunForTest0(); + // Sampled AFTER the reset, so not a pre-reset baseline: a value > 0 means a pass ENDED + // between the reset and this read (the reset zeroes the counter only at pass END, so a + // long in-flight pass is still invisible to this read - the candidate bracket plus + // checkNoCandidateFreeActivity's retry compensates for that blind spot). + int passesAfterReset = ReferenceChainTestHooks.referenceChainPassesRunForTest0(); GateEvaluation e = new GateEvaluation(); e.gateOpen = ReferenceChainTestHooks.shouldRunPassForTest0(); int[] candidatesAfter = ReferenceChainTestHooks.selectLeakCandidateKlassIds0(); + // Sampled after the gate call: a BFS pass racing between the gate evaluation and this read + // is conservatively attributed to the evaluation window (safe direction: it only widens + // the observed activity window, never masks a gate violation). e.passesAfter = ReferenceChainTestHooks.referenceChainPassesRunForTest0(); e.candidateSeen = !isEmpty(candidatesBefore) || !isEmpty(candidatesAfter); e.detail = "candidatesBefore=" + Arrays.toString(candidatesBefore) + " candidatesAfter=" - + Arrays.toString(candidatesAfter) + " passesBefore=" + passesBefore + + Arrays.toString(candidatesAfter) + " passesAfterReset=" + passesAfterReset + " passesAfter=" + e.passesAfter; return e; } diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainAssertions.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainAssertions.java index aa6f0ba7b1..0eb121e087 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainAssertions.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainAssertions.java @@ -22,6 +22,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import static org.junit.jupiter.api.Assumptions.assumeTrue; /** * Shared {@code datadog.ReferenceChain} JFR-parsing helpers. @@ -32,6 +33,17 @@ * via JMC's {@code JfrLoaderToolkit} ({@link #findMatchForClass(IItemCollection, Class)}). */ public final class ReferenceChainAssertions { + + /** + * Skips the calling test unless the native library is a debug-instrumented build - the + * reference-chain {@code *ForTest0} seams exist only under javaApi.cpp's {@code #ifdef DEBUG}. + */ + public static void assumeDebugBuild() { + assumeTrue(isDebugInstrumentedBuild(), + "reference-chain test seams only exist in a -DDEBUG native build (debug/asan/tsan - " + + "javaApi.cpp's #ifdef DEBUG guard)"); + } + private ReferenceChainAssertions() {} /** diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainSchedulingTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainSchedulingTest.java new file mode 100644 index 0000000000..c532a283b9 --- /dev/null +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainSchedulingTest.java @@ -0,0 +1,417 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.datadoghq.profiler.referencechains; + +import com.datadoghq.profiler.AbstractProfilerTest; +import com.datadoghq.profiler.Platform; +import com.datadoghq.profiler.ReferenceChainTestHooks; +import org.junit.jupiter.api.Test; + +import java.util.function.BooleanSupplier; + +import static com.datadoghq.profiler.referencechains.ReferenceChainAssertions.assumeDebugBuild; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/** + * Pins the two scheduling gates that bound the reference-chain tracker's JVMTI heap-walk + * frequency on services with a high young-GC rate: + *

    + *
  • Minimum pass spacing ({@code shouldRunPass()}): every authorization lane waits out + * {@code max(_effective_cadence_ns, MIN_PASS_SPACING_NS)} since the previous pass - a fixed + * floor independent of the pacing controller's output, with the OOM ramp as the only + * exemption. Passes run on a pure cadence; a completed GC neither bypasses the spacing nor + * earns an earlier pass.
    • + *
  • Adaptive root-enumeration interval ({@code shouldRunRootEnum()}): re-enumeration + * waits out {@code ROOT_ENUM_MIN_INTERVAL_NS} extended by the last attempt's measured + * in-safepoint cost.
    • + *
+ * The mechanisms these gates bound are documented on the constants in referenceChains.h. + * + *

Only runs under the debug native build - the backing native methods do not exist in a + * release build, same pattern as {@link ReferenceChainTestSeamsTest}. + */ +public class ReferenceChainSchedulingTest extends AbstractProfilerTest { + + // A cadence far above any timing jitter in these tests: with the spacing gate seeded to "the + // previous pass just ended", the gate stays closed for the whole assertion window, and the BFS + // thread - which reads the same cadence for its idle sleep - stays quiesced while the seeded + // state is in force. + private static final long CADENCE_MS = 10_000; + // Long enough for any BFS iteration already in flight when a test seeds its state to finish + // (post-reset the thread runs no pass: the search is not started and there is no leak signal, + // so an iteration is two atomic loads and a sleep decision). + private static final long QUIESCE_MS = 1_200; + // Java mirrors of referenceChains.h's root-enumeration constants - no seam exposes them: + // ROOT_ENUM_MIN_INTERVAL_NS, ROOT_ENUM_INTERVAL_NS_PER_COST_MS, ROOT_ENUM_MAX_INTERVAL_NS. + // A native retune must update these mirrors. rootEnumBaseIntervalMatchesNative and the + // floor-equality probes below pin all three against the live library, so a stale mirror fails + // loudly instead of silently re-seeding the wrong regime. + private static final long NS_PER_MS = 1_000_000L; + private static final long ROOT_ENUM_MIN_INTERVAL_NS = 2_000_000_000L; + private static final long ROOT_ENUM_NS_PER_COST_MS = 20_000_000L; + private static final long ROOT_ENUM_MAX_INTERVAL_NS = 30_000_000_000L; + // Seed ages either side of the base interval, and costs that extend / cap the floor. + private static final long ROOT_ENUM_BEFORE_BASE_MS = + ROOT_ENUM_MIN_INTERVAL_NS / NS_PER_MS - 1_000; + private static final long ROOT_ENUM_PAST_BASE_MS = + ROOT_ENUM_MIN_INTERVAL_NS / NS_PER_MS + 1_000; + private static final long ROOT_ENUM_EXTENDING_COST_MS = 500; + private static final long ROOT_ENUM_CAPPING_COST_MS = 5_000; + // Seed age just past the cap: with the capping cost the floor is ROOT_ENUM_MAX_INTERVAL_NS, so + // this age must open the gate (kills an uncapped floor). + private static final long ROOT_ENUM_PAST_CAP_MS = + ROOT_ENUM_MAX_INTERVAL_NS / NS_PER_MS + 1_000; + + @Override + protected String getProfilerCommand() { + return "generations=true,referencechains=true:hops=32:budget=500:ttl=60000:framecap=2000000"; + } + + @Override + protected boolean isPlatformSupported() { + return !(Platform.isJavaVersion(8) || Platform.isJ9() || Platform.isZing()); + } + + /** + * Evaluates {@code probe} against re-seeded scheduling state, retrying while a concurrently + * passing BFS thread invalidates the seed: a real pass overwrites {@code _last_pass_ns} + * and the root-enumeration state with live values, and can + * drive the real search terminal, which can only ever flip a seeded-open gate to the closed + * reading - so a closed result is final and only an open/matching result needs the retry. + * The same real pass also produces the closed reading on its own, so a closed-direction probe + * (a negated gate reading) must be bracketed by a check that the seeded state is still in force + * (see {@link #rootEnumClosedWithSeedIntact(long, Runnable)}) - otherwise a closed result + * caused by a real pass would be accepted instead of retried. + */ + private static boolean eventuallyTrue(String message, Runnable seed, + BooleanSupplier probe) + throws InterruptedException { + for (int i = 0; i < 50; i++) { + seed.run(); + if (probe.getAsBoolean()) { + return true; + } + Thread.sleep(100); + } + assertFalse(true, message + " (state re-seeded 50 times over ~5s, never observed)"); + return false; + } + + /** + * Closed-direction root-enum probe (see eventuallyTrue): a closed gate counts only while the + * seeded state is still in force, verified by a floor + pass-counter bracket around the gate + * read. A concurrent real enumeration also closes the gate (it refreshes _last_root_enum_ns to + * "now"), so a bare closed reading cannot tell the seeded floor apart from that unrelated + * close - and a floor-only bracket is not enough either: a real enumeration measuring a cost + * that re-derives the same floor (e.g. ~0 ms on a small graph vs the seeded 0 cost) would + * reproduce the seeded floor while the closed reading was actually caused by that enumeration. + * + *

The bracket is structurally airtight for pass-driven taint: real root enumeration runs + * ONLY inside a BFS pass ({@code runPassSerialized()} holds {@code _engine_lock} across the + * whole {@code runPass()}), and {@code _passes_run} increments only after the manual walk + * returns - so enumeration state and the pass counter flip together, and the probe's + * engine-lock-acquiring gate/floor reads can never observe one without the other: a completed + * pass always flips {@code p0 != p1}, and an enumeration whose measured cost shifts the floor + * flips {@code f1 != expectedFloorNs}. The probe also re-seeds immediately before reading, + * undoing any real enumeration that already completed and clobbered the seeded state, so only + * an enumeration landing DURING the read sequence can taint it - a microsecond window instead + * of the seed-to-probe gap. Premise: if a future change ever runs root enumeration outside a + * pass (outside {@code runPassSerialized()}'s lock hold), the pass-counter premise breaks and + * this bracket degrades to probabilistic - revisit it then. + */ + private static boolean rootEnumClosedWithSeedIntact(long expectedFloorNs, Runnable seed) { + seed.run(); // re-seed immediately before reading: undoes any real enumeration that already + // completed and clobbered the seeded state. + int p0 = ReferenceChainTestHooks.referenceChainPassesRunForTest0(); + long f0 = ReferenceChainTestHooks.rootEnumIntervalFloorNsForTest0(); + boolean closed = !ReferenceChainTestHooks.shouldRunRootEnumForTest0(false); + long f1 = ReferenceChainTestHooks.rootEnumIntervalFloorNsForTest0(); + int p1 = ReferenceChainTestHooks.referenceChainPassesRunForTest0(); + return closed && p0 == p1 && f0 == expectedFloorNs && f1 == expectedFloorNs; + } + + /** Seed relations the root-enum tests rely on, checked on the Java mirrors themselves. */ + private static void assertRootEnumSeedAssumptions() { + long pastBaseNs = ROOT_ENUM_PAST_BASE_MS * NS_PER_MS; + long extendedNs = ROOT_ENUM_EXTENDING_COST_MS * ROOT_ENUM_NS_PER_COST_MS; + assertTrue(ROOT_ENUM_BEFORE_BASE_MS > 0, "before-base age must be positive"); + assertTrue(pastBaseNs < extendedNs && extendedNs < ROOT_ENUM_MAX_INTERVAL_NS, + "extending cost must land strictly between the past-base age and the cap"); + assertTrue(ROOT_ENUM_CAPPING_COST_MS * ROOT_ENUM_NS_PER_COST_MS > ROOT_ENUM_MAX_INTERVAL_NS, + "capping cost must exceed the cap"); + assertTrue(ROOT_ENUM_PAST_CAP_MS * NS_PER_MS > ROOT_ENUM_MAX_INTERVAL_NS, + "past-cap age must exceed the cap"); + } + + private static void seedRunningSearch() { + // forceRunningSearchForTest (not just "search started"): the live BFS thread may be running + // its own real search (leftover leak signal from earlier tests in this shared JVM) and can + // drive it terminal between this seed and the gate call - the forced RUNNING state with fresh + // pain budgets keeps the gate evaluating the running-search lanes deterministically. + ReferenceChainTestHooks.forceRunningReferenceChainSearchForTest0(); + // Zero candidates: the canary lane is off and the plain cadence lane is what runs. + ReferenceChainTestHooks.setReferenceChainCanaryStateForTest0(0, 0); + } + + /** + * Pins the fixed 100ms pass-spacing floor (MIN_PASS_SPACING_NS): even when updatePacing()'s + * controller relaxes the effective cadence to 10ms, a pass less than 100ms after the previous + * one must stay held off - the controller paces each pass's pause LENGTH, not the pass rate. + * Once the fixed floor has elapsed (with the cadence still relaxed), the pass is authorized. + */ + @Test + public void passSpacingHoldsAtFixedFloorEvenWithRelaxedCadence() throws InterruptedException { + assumeDebugBuild(); + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); + try { + seedRunningSearch(); + // Quiesce first with a wide cadence (same pattern as the canary test below): the live BFS + // thread - which sleeps for the seeded cadence - must be parked before the tight seeds land. + ReferenceChainTestHooks.setReferenceChainPassSpacingForTest0(0, CADENCE_MS); + Thread.sleep(QUIESCE_MS); + // Seed a relaxed 10ms cadence (what updatePacing() can reach on cheap passes) with the + // previous pass 50ms ago - inside the fixed floor but well past the relaxed cadence. + ReferenceChainTestHooks.setReferenceChainPassSpacingForTest0(50, 10); + assertFalse(ReferenceChainTestHooks.shouldRunPassForTest0(), + "A pass inside the fixed 100ms floor must stay held off even though the relaxed " + + "10ms cadence has elapsed"); + + // Past the fixed floor (500ms, cadence still relaxed to 10ms): the pass is authorized. + eventuallyTrue("A pass past the fixed floor must be authorized even with a relaxed cadence", + () -> { + seedRunningSearch(); + ReferenceChainTestHooks.setReferenceChainPassSpacingForTest0(500, 10); + }, + ReferenceChainTestHooks::shouldRunPassForTest0); + } finally { + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); + } + } + + /** + * Kills the canary lane's unbounded back-to-back rate: a productive chase (backoff multiplier + * 1) would otherwise skip its spacing gate entirely, chaining passes with no gap and making + * every pass pay its JVM VM-operation cost. The minimum-pass-spacing gate must hold the chase + * off even on that lane. + */ + @Test + public void canaryBackToBackPassWaitsForMinimumPassSpacing() throws InterruptedException { + assumeDebugBuild(); + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); + try { + seedRunningSearch(); + ReferenceChainTestHooks.setReferenceChainPassSpacingForTest0(0, CADENCE_MS); + Thread.sleep(QUIESCE_MS); + // One unresolved candidate: the gate takes the canary lane with backoff multiplier 1. + // Seeded after the quiesce so the BFS thread's per-iteration pollWatchedTargets() (which + // overwrites the candidate set from real leak signals) cannot clobber it. + ReferenceChainTestHooks.setReferenceChainCanaryStateForTest0(1, 0); + assertFalse(ReferenceChainTestHooks.shouldRunPassForTest0(), + "A canary chase with fresh backoff (multiplier 1) must still wait out the minimum " + + "pass spacing before starting the next pass"); + } finally { + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); + } + } + + /** The canary floor must only defer the chase, not starve it: once the spacing has elapsed, + * the chase resumes at multiplier-1 rate. */ + @Test + public void canaryPassRunsOnceSpacingElapses() throws InterruptedException { + assumeDebugBuild(); + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); + try { + seedRunningSearch(); + eventuallyTrue("A canary chase with fresh backoff must run a pass once the minimum " + + "spacing has elapsed", + () -> { + seedRunningSearch(); + ReferenceChainTestHooks.setReferenceChainCanaryStateForTest0(1, 0); + ReferenceChainTestHooks.setReferenceChainPassSpacingForTest0( + CADENCE_MS * 2, CADENCE_MS); + }, + ReferenceChainTestHooks::shouldRunPassForTest0); + } finally { + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); + } + } + + /** + * The reset contract (resetSearchStateForTest()): it zeroes {@code _last_pass_ns} along with + * the pass counter, so the minimum-spacing gate is open again right after a reset - a stale + * previous-pass timestamp must not hold the next test's first pass off. A closed reading with + * zero passes run since the reset is definitive (no real pass refreshed the timestamp); a + * closed reading after a real pass is retried. + */ + @Test + public void resetReopensMinimumPassSpacingGate() throws InterruptedException { + assumeDebugBuild(); + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); + try { + for (int attempt = 0; attempt < 10; attempt++) { + // A closed gate: the previous pass "just ended" under a long cadence. + ReferenceChainTestHooks.setReferenceChainPassSpacingForTest0(0, CADENCE_MS); + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); + seedRunningSearch(); + if (ReferenceChainTestHooks.shouldRunPassForTest0()) { + return; + } + assertTrue(ReferenceChainTestHooks.referenceChainPassesRunForTest0() != 0, + "the spacing gate must be open right after a reset - _last_pass_ns/" + + "_effective_cadence_ns were not reset"); + Thread.sleep(100); + } + assertFalse(true, "an open spacing gate right after a reset was never observed"); + } finally { + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); + } + } + + /** Root/stack-ref enumeration waits out its base interval before re-running. */ + @Test + public void rootEnumWaitsForMinimumInterval() throws InterruptedException { + assumeDebugBuild(); + assertRootEnumSeedAssumptions(); + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); + try { + Runnable seed = () -> ReferenceChainTestHooks.setReferenceChainRootEnumStateForTest0( + ROOT_ENUM_BEFORE_BASE_MS, 0); + eventuallyTrue("Re-enumeration must wait out the base interval since the last attempt", + seed, + () -> rootEnumClosedWithSeedIntact(ROOT_ENUM_MIN_INTERVAL_NS, seed)); + } finally { + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); + } + } + + /** The base interval must only defer enumeration, not disable it. */ + @Test + public void rootEnumRunsAfterBaseInterval() throws InterruptedException { + assumeDebugBuild(); + assertRootEnumSeedAssumptions(); + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); + try { + eventuallyTrue("Re-enumeration must run once the base interval has elapsed", + () -> ReferenceChainTestHooks.setReferenceChainRootEnumStateForTest0( + ROOT_ENUM_PAST_BASE_MS, 0), + () -> ReferenceChainTestHooks.shouldRunRootEnumForTest0(false)); + } finally { + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); + } + } + + /** + * Pins the ROOT_ENUM_MIN_INTERVAL_NS mirror against the live library: with no measured cost + * the native floor is exactly the base interval. + */ + @Test + public void rootEnumBaseIntervalMatchesNative() throws InterruptedException { + assumeDebugBuild(); + assertRootEnumSeedAssumptions(); + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); + try { + eventuallyTrue("The base root-enumeration interval must match ROOT_ENUM_MIN_INTERVAL_NS", + () -> ReferenceChainTestHooks.setReferenceChainRootEnumStateForTest0( + ROOT_ENUM_PAST_BASE_MS, 0), + () -> ReferenceChainTestHooks.rootEnumIntervalFloorNsForTest0() + == ROOT_ENUM_MIN_INTERVAL_NS); + } finally { + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); + } + } + + /** + * Kills the fixed 2s interval: an attempt that held the VM at a safepoint for a known cost + * must push the next attempt out proportionally - on a busy service the cost recurs (the JVM + * re-creates frames with scalar-replaced locals continuously), so only a cost-scaled floor + * bounds the deopt-all rate. + */ + @Test + public void rootEnumIntervalExtendsWithMeasuredSafepointCost() throws InterruptedException { + assumeDebugBuild(); + assertRootEnumSeedAssumptions(); + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); + try { + Runnable seed = () -> ReferenceChainTestHooks.setReferenceChainRootEnumStateForTest0( + ROOT_ENUM_PAST_BASE_MS, ROOT_ENUM_EXTENDING_COST_MS); + // Positive control: the past-base seed alone opens the gate, so the closed reading below + // is attributable to the cost-extended floor. + eventuallyTrue("A past-base seed with no measured cost must open the gate", + () -> ReferenceChainTestHooks.setReferenceChainRootEnumStateForTest0( + ROOT_ENUM_PAST_BASE_MS, 0), + () -> ReferenceChainTestHooks.shouldRunRootEnumForTest0(false)); + // ROOT_ENUM_EXTENDING_COST_MS of measured in-safepoint cost: floor = max(MIN, cost x + // PER_COST) - strictly between the past-base age and the cap. + eventuallyTrue("The interval floor must extend by ROOT_ENUM_INTERVAL_NS_PER_COST_MS per " + + "measured in-safepoint cost millisecond", + () -> ReferenceChainTestHooks.setReferenceChainRootEnumStateForTest0( + ROOT_ENUM_PAST_BASE_MS, ROOT_ENUM_EXTENDING_COST_MS), + () -> ReferenceChainTestHooks.rootEnumIntervalFloorNsForTest0() + == ROOT_ENUM_EXTENDING_COST_MS * ROOT_ENUM_NS_PER_COST_MS); + eventuallyTrue("Re-enumeration must wait out the cost-extended interval even though the " + + "base interval has elapsed", + seed, + () -> rootEnumClosedWithSeedIntact( + ROOT_ENUM_EXTENDING_COST_MS * ROOT_ENUM_NS_PER_COST_MS, seed)); + } finally { + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); + } + } + + /** The cost-scaled interval must stay capped so root refresh never stops entirely. */ + @Test + public void rootEnumIntervalCapsAtMaximum() throws InterruptedException { + assumeDebugBuild(); + assertRootEnumSeedAssumptions(); + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); + try { + Runnable seed = () -> ReferenceChainTestHooks.setReferenceChainRootEnumStateForTest0( + ROOT_ENUM_PAST_BASE_MS, ROOT_ENUM_CAPPING_COST_MS); + // Positive control, as in rootEnumIntervalExtendsWithMeasuredSafepointCost. + eventuallyTrue("A past-base seed with no measured cost must open the gate", + () -> ReferenceChainTestHooks.setReferenceChainRootEnumStateForTest0( + ROOT_ENUM_PAST_BASE_MS, 0), + () -> ReferenceChainTestHooks.shouldRunRootEnumForTest0(false)); + // ROOT_ENUM_CAPPING_COST_MS scales past ROOT_ENUM_MAX_INTERVAL_NS uncapped; the cap holds + // it there. + eventuallyTrue("The cost-scaled interval must be capped at ROOT_ENUM_MAX_INTERVAL_NS", + () -> ReferenceChainTestHooks.setReferenceChainRootEnumStateForTest0( + ROOT_ENUM_PAST_BASE_MS, ROOT_ENUM_CAPPING_COST_MS), + () -> ReferenceChainTestHooks.rootEnumIntervalFloorNsForTest0() + == ROOT_ENUM_MAX_INTERVAL_NS); + eventuallyTrue("Re-enumeration must wait out the capped interval", + seed, + () -> rootEnumClosedWithSeedIntact(ROOT_ENUM_MAX_INTERVAL_NS, seed)); + // An uncapped floor would still hold this age off. + eventuallyTrue("Re-enumeration must run once the capped interval has elapsed", + () -> ReferenceChainTestHooks.setReferenceChainRootEnumStateForTest0( + ROOT_ENUM_PAST_CAP_MS, ROOT_ENUM_CAPPING_COST_MS), + () -> ReferenceChainTestHooks.shouldRunRootEnumForTest0(false)); + } finally { + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); + } + } + + /** The search's first enumeration is mandatory and must ignore the interval entirely. */ + @Test + public void firstPassAlwaysRunsRootEnum() { + assumeDebugBuild(); + assertRootEnumSeedAssumptions(); + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); + try { + // Worst case: an attempt "just ran" at maximum measured cost - the first pass of a fresh + // search must still enumerate. + ReferenceChainTestHooks.setReferenceChainRootEnumStateForTest0(0, ROOT_ENUM_CAPPING_COST_MS); + assertTrue(ReferenceChainTestHooks.shouldRunRootEnumForTest0(true), + "The search's first pass must always run root/stack-ref enumeration"); + } finally { + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); + } + } +} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTestSeamsTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTestSeamsTest.java index 4fe1d3ff16..6a7917d4ad 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTestSeamsTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTestSeamsTest.java @@ -10,10 +10,10 @@ import com.datadoghq.profiler.ReferenceChainTestHooks; import org.junit.jupiter.api.Test; +import static com.datadoghq.profiler.referencechains.ReferenceChainAssertions.assumeDebugBuild; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assumptions.assumeTrue; /** * PROF-15341 follow-up: {@code ReferenceChainTrackingTest} drives slope detection and chain @@ -68,10 +68,6 @@ protected boolean isPlatformSupported() { return !(Platform.isJavaVersion(8) || Platform.isJ9() || Platform.isZing()); } - private static void assumeDebugBuild() { - assumeTrue(ReferenceChainAssertions.isDebugInstrumentedBuild()); - } - /** Seeds twenty epochs of strictly increasing population counts, then asserts * {@code selectLeakCandidates()} ranks the klass as a leak candidate - proving the ranking * logic works without the real JVMTI heap sampler ever surfacing this klass. */ diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTrackingTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTrackingTest.java index 9bdbfe26b7..2d1ad31f1f 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTrackingTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTrackingTest.java @@ -123,6 +123,15 @@ public class ReferenceChainTrackingTest extends AbstractProfilerTest { private static final long ABANDON_DUMP_RETRY_SPACING_MS = 1_000; private static final long ABANDON_LAST_DUMP_MIN_DELAY_MS = 8_000; + // Seam-driven passes are front-loaded onto the first GRACE_DRIVE_ATTEMPTS grace-loop + // attempts (8 passes each, 24 total); the remaining grace-loop attempts are poll-only + // (organic BFS cadence + 1s sleep + dump + match), so CI load does not scale with the loop + // width. Each driven pass serializes against the BFS thread via _engine_lock and feeds the + // shared pause-controller cost model (_effective_cadence_ns, _canary_pass_ema_ms, + // _cpu_pain_budget), and the deterministic-frontier claim needs only cumulative progress, + // which the front-loaded burst far exceeds. + private static final int GRACE_DRIVE_ATTEMPTS = 3; + /** * The {@code datadog.ReferenceChain} ActiveSetting's {@code id} (flightRecorder.cpp's * T_REFERENCE_CHAIN). ActiveSetting events from cpu/wall/memory/nativemem also read @@ -467,10 +476,15 @@ public void shouldReconstructReferrerChainToGcRoot() throws Exception { // for why that is load-bearing; the seam stubs are no-ops in release builds). Ten attempts, // matching the cache-leak test's loop: this fixture's ArrayList resizes churn the walked // element path just like the cache's HashMap resizes, so late-round discovery needs the - // same scheduling-independent margin. + // same scheduling-independent margin. The seam burst is capped at the first + // GRACE_DRIVE_ATTEMPTS attempts to bound synchronous heap-walk load on shared CI runners + // (each pass serializes against the BFS thread via _engine_lock and feeds the + // pause-controller cost model); the remaining attempts are poll-only on the organic cadence. JfrEvents lastEvents = null; for (int attempt = 0; match == null && attempt < 10; attempt++) { - driveReferenceChainPasses(8); + if (attempt < GRACE_DRIVE_ATTEMPTS) { + driveReferenceChainPasses(8); + } Thread.sleep(1000); dump(scratchDumpPath); lastEvents = verifyEvents(scratchDumpPath, "datadog.ReferenceChain", false); @@ -691,10 +705,16 @@ public void shouldReconstructReferrerChainThroughUnboundedCacheLeak() throws Exc // duration). Driving passes here makes discovery depend on cumulative frontier progress, // which is deterministic, instead of on that scheduling. pollReferenceChainTargets0() after // each pass is what tags the candidate's tracked instances and polls the representative - - // the same per-iteration tail the BFS thread runs. + // the same per-iteration tail the BFS thread runs. The burst is capped at the first + // GRACE_DRIVE_ATTEMPTS attempts to bound synchronous heap-walk load on shared CI runners + // (each pass serializes against the BFS thread via _engine_lock and feeds the + // pause-controller cost model); the remaining attempts are poll-only on the organic + // cadence, which keeps this loop's load within the pre-seam 5-attempt shape. JfrEvents lastEvents = null; for (int attempt = 0; match == null && attempt < 10; attempt++) { - driveReferenceChainPasses(8); + if (attempt < GRACE_DRIVE_ATTEMPTS) { + driveReferenceChainPasses(8); + } Thread.sleep(1000); dump(scratchDumpPath); lastEvents = verifyEvents(scratchDumpPath, "datadog.ReferenceChain", false); @@ -764,7 +784,13 @@ public void shouldReconstructReferrerChainThroughUnboundedCacheLeak() throws Exc * flakiness, observed in practice), the shared search can still be {@code RUNNING} with a large, * non-fresh frontier when this method starts - the fresh-search precondition is something this * method establishes, not something it presumes. Debug-only: the seam does not exist in a - * release build. + * release build, and the {@code assumeTrue} at the top of the method body skips this test + * there - a JUnit assumption failure is reported as SKIPPED, not passed, so coverage reports + * must not count it. By design, release-config runs (the {@code test:release} label) + * therefore exercise none of the {@code buildAbandonedEvent()} → + * {@code FlightRecorder::recordReferenceChainAbandoned()} write path; if the candidate-gate + * contract ever changes so a release build could reach {@code ABANDONED} without the debug + * population seams, this assumption must be revisited. * *

Why this method also seeds a klass population trend, despite requesting no * {@code generations=true} of its own: {@code LivenessTracker::_gc_generations} is only ever @@ -860,13 +886,17 @@ public void shouldReportAbandonedSearchOnTinyFrontierCap() throws Exception { // the frontier cap/ttl cutoff inside whatever wall-clock time this shared JVM's scheduling // grants it. The abandoned event only reaches the recording through a dump(), so each check // needs a fresh dump - and each dump rewrites the whole shared recording. Dumps are - // therefore capped at ABANDON_MAX_DUMPS and paced so the last one lands at least - // ABANDON_LAST_DUMP_MIN_DELAY_MS after the grace period started; the passesRun probe above - // absorbs most of the scheduling slack without any I/O. + // therefore capped at ABANDON_MAX_DUMPS and paced so that, when the loop runs to + // completion, the last one lands at or after ABANDON_LAST_DUMP_MIN_DELAY_MS after the + // grace period started (on the final attempt attemptsLeft == 1, so the sleep is + // Math.max(SPACING, full remaining); an early success break can land earlier, which is + // fine because the event was already found and the floor is moot); the passesRun probe + // above absorbs most of the scheduling slack without any I/O. long lastDumpEarliestNs = graceStartNs + TimeUnit.MILLISECONDS.toNanos(ABANDON_LAST_DUMP_MIN_DELAY_MS); JfrEvents abandoned = null; int dumpsTaken = 0; + long lastDumpAtNs = 0; for (int attempt = 0; attempt < ABANDON_MAX_DUMPS; attempt++) { if (attempt > 0) { int attemptsLeft = ABANDON_MAX_DUMPS - attempt; // includes the dump about to be taken @@ -874,6 +904,7 @@ public void shouldReportAbandonedSearchOnTinyFrontierCap() throws Exception { TimeUnit.NANOSECONDS.toMillis(lastDumpEarliestNs - System.nanoTime()); Thread.sleep(Math.max(ABANDON_DUMP_RETRY_SPACING_MS, remainingMs / attemptsLeft)); } + lastDumpAtNs = System.nanoTime(); dump(dumpPath); dumpsTaken++; abandoned = verifyEvents(dumpPath, "datadog.ReferenceChainAbandoned", false); @@ -885,8 +916,9 @@ public void shouldReportAbandonedSearchOnTinyFrontierCap() throws Exception { "Expected a datadog.ReferenceChainAbandoned event within " + dumpsTaken + " of at most " + ABANDON_MAX_DUMPS + " dumps (after 3 GCs and a 1500ms grace period; the pass " + "probe waited up to " + ABANDON_PASS_WAIT_MS + "ms, passesRun=" - + ReferenceChainTestHooks.referenceChainPassesRunForTest0() + "; last dump >= " - + ABANDON_LAST_DUMP_MIN_DELAY_MS + "ms after grace start)"); + + ReferenceChainTestHooks.referenceChainPassesRunForTest0() + "; last dump " + + TimeUnit.NANOSECONDS.toMillis(lastDumpAtNs - graceStartNs) + + "ms after grace start, nominal floor " + ABANDON_LAST_DUMP_MIN_DELAY_MS + "ms)"); } finally { Files.deleteIfExists(dumpPath); } diff --git a/doc/ReferenceChainsCollectionSummary.md b/doc/ReferenceChainsCollectionSummary.md index 0cf16f5f20..90fbd82aaa 100644 --- a/doc/ReferenceChainsCollectionSummary.md +++ b/doc/ReferenceChainsCollectionSummary.md @@ -40,7 +40,7 @@ Candidate selection uses sampled objects. It can miss a leak when sampling does Each pass can perform these operations: -1. Enumerate roots and stack references. The first pass always does this. Later passes retry a truncated enumeration or wait at least two seconds. +1. Enumerate roots and stack references. The first pass always does this. Later passes wait at least two seconds since the last attempt - extended by the last attempt's measured in-safepoint cost - truncated attempts included; a pass whose enumeration truncated ends after the enumeration. 2. Walk candidate thread objects when the candidate data identifies qualifying threads. 3. Scan static fields in chunks when the loaded-class count changes. 4. Expand a batch of frontier entries. diff --git a/doc/architecture/LiveHeapReferenceChainsImplementation.md b/doc/architecture/LiveHeapReferenceChainsImplementation.md index d6ad2d2c7e..be2235fd3d 100644 --- a/doc/architecture/LiveHeapReferenceChainsImplementation.md +++ b/doc/architecture/LiveHeapReferenceChainsImplementation.md @@ -69,7 +69,7 @@ The tracker measures total pass time and in-safepoint time separately. It charge `IterateOverReachableObjects()` calls `heapRootCallback()` and `stackRefCallback()`. These callbacks admit root-attached objects. They do not traverse all children of each root. -The first pass always performs root enumeration. A later pass performs it after a truncated enumeration or an interval of at least two seconds. +The first pass always performs root enumeration. A later pass performs it after an interval of at least two seconds since the last attempt, extended by the last attempt's measured in-safepoint cost (20 milliseconds of interval per millisecond of cost, capped at 30 seconds). Each enumeration deoptimizes, on every thread, every compiled frame whose scope holds a scalar-replaced (escape-analysis-eliminated) object (one `VM_DeoptimizeFrame` operation per frame), and the JIT re-creates those frames continuously on a busy service, so the interval grows with the cost that recurs per attempt. A truncated attempt retries after the same interval as a completed one: with a first-pass budget too small for the live heap, every attempt truncates, and a next-pass retry would turn every pass into a deopt-all enumeration. A truncated pass ends after the enumeration - the next pass's expansion phases proceed on the roots already admitted. Root enumeration uses `firstpassbudget`. The name is historical: the implementation can use this budget again during later root-enumeration attempts. @@ -171,6 +171,8 @@ The base candidate-stall limit is 30 passes. Consecutive `CANARY_STUCK` restarts The implementation controls different costs separately: +- The minimum pass spacing (the effective cadence) applies to every pass authorization - GC epoch, canary back-to-back, and fallback cadence alike. +- The root-enumeration interval floor extends with the last attempt's measured in-safepoint cost. - `_effective_budget` limits edge admission for ordinary work. - `_pause_pid` adjusts that budget from measured in-safepoint time. - `_effective_cadence_ns` changes when the budget controller reaches a limit and while the OOM urgency ramp is active. When an urgency episode ends, the ramp restores the one-second fallback in the same iteration. diff --git a/doc/architecture/ReferenceChainsSignalsExplained.md b/doc/architecture/ReferenceChainsSignalsExplained.md index 80f731cd36..270d6bae6c 100644 --- a/doc/architecture/ReferenceChainsSignalsExplained.md +++ b/doc/architecture/ReferenceChainsSignalsExplained.md @@ -32,13 +32,20 @@ The population-trend signal is constructed by the liveness tracker, which fills A running search uses the following order: -1. Check the non-safepoint CPU cost budget. -2. If unresolved candidates exist, apply candidate-search backoff. -3. Otherwise, run after a new GC epoch or after the adaptive fallback cadence expires. +1. Enforce the minimum pass spacing: no pass starts sooner than the effective cadence after the previous pass ended, on every authorization lane below. +2. Check the non-safepoint CPU cost budget. +3. If unresolved candidates exist, apply candidate-search backoff. +4. Otherwise, run after a new GC epoch or after the adaptive fallback cadence expires. + +### Minimum pass spacing + +A completed GC authorizes a pass, and a productive canary chase (backoff multiplier one) may run its next pass as soon as scheduling allows - but neither may start sooner than one effective cadence interval after the previous pass ended. Without this floor, a high young-GC rate re-triggers passes at the GC rate even after the pacing controller has widened the cadence to shed in-safepoint load, and the canary lane runs passes with no gap at all. Each pass costs several JVM VM operations (root enumeration, static-field sweep chunks, expansion batches), so unbounded pass rate multiplies into a safepoint storm. + +The OOM ramp needs no exemption: while urgent it owns the effective cadence outright, driving this floor down with urgency. ### Candidate-search backoff -The source calls the focused candidate search the *canary* path. A new candidate or a resolved candidate resets its spacing multiplier to one. With this value, the next pass can start after the previous pass finishes. +The source calls the focused candidate search the *canary* path. A new candidate or a resolved candidate resets its spacing multiplier to one. With this value, the next pass can start after the previous pass finishes, but never sooner than the minimum pass spacing above. If a pass makes no candidate progress, the multiplier doubles up to its cap. The required spacing is: @@ -50,9 +57,9 @@ The OOM ramp disables this backoff. The non-safepoint CPU cost budget still appl ### GC epoch and fallback cadence -When no unresolved candidate controls scheduling, a completed GC authorizes a pass. The tracker compares the current GC-finish epoch with the epoch recorded by the previous pass. +When no unresolved candidate controls scheduling, a completed GC authorizes a pass. The tracker compares the current GC-finish epoch with the epoch recorded by the previous pass. The authorization remains subject to the minimum pass spacing: when young GCs are more frequent than the effective cadence, the pass rate follows the cadence, not the GC rate. -The fallback cadence also authorizes a pass. It starts at one second. Two writers adjust it between 10 milliseconds and four seconds: the pause controller (`updatePacing()`) and the OOM urgency ramp. When an urgency episode ends, the ramp restores the one-second fallback in the same iteration. +The fallback cadence also authorizes a pass. It starts at one second. Two writers adjust it between 10 milliseconds and four seconds: the pause controller (`updatePacing()`) and the OOM urgency ramp. When an urgency episode ends, the ramp restores the one-second fallback in the same iteration. The minimum pass spacing uses the same value, so the two authorizations share one rate bound. ## Leak signal diff --git a/utils/run-refchains-repro.sh b/utils/run-refchains-repro.sh index 642f1b431a..ab539a7e07 100755 --- a/utils/run-refchains-repro.sh +++ b/utils/run-refchains-repro.sh @@ -42,6 +42,14 @@ # bare name ("Serial", "G1", "Parallel", "Shenandoah", # "Z", "Epsilon") or the full flag name ("SerialGC"). # e.g. REFCHAINS_GC=Serial or REFCHAINS_GC=ZGC +# REFCHAINS_EASPIN number of escape-analysis spin worker threads the demo +# app starts (default: 0 - passed through to the app via +# the env var of the same name; see ReferenceChainLeakDemo. +# startEaSpinWorkers). With workers running, root enumerations +# deoptimize their compiled frames, making VM_DeoptimizeFrame +# safepoints observable in the recording. Each worker +# busy-spins one core; workers stop at the run's deadline +# (run-forever: at JVM exit). set -euo pipefail