From fe21d4095b8ee5f6502439812ceab43909c2c545 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 28 Aug 2026 11:23:10 +0000 Subject: [PATCH 1/2] perf(gc): the dirty-page cache is mirrored in a process global tagged by the writer's TSD base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The write barrier's dirty-page compare reached its per-thread cell through the hot-TLS chain — a global slot-index load, the pthread key, `mrs`, the TSD slot, then the cell: four dependent loads on every barrier call, and after the leaf entry the profile put the barrier's single hottest instruction on that chain (2.3% of an ECS frame on `mrs` and what waits on it). The cell stays the authority; every path that writes or clears it also writes a (page, owner) pair of process globals, owner being the writing thread's TSD base. A reader identifies itself with one `mrs` and two loads that do not depend on each other: if the owner is the calling thread the page word is its own last write, so the compare is exactly the cell's; otherwise it falls back to the cell. A torn read can only answer "not cached" for a page the reader owns (heaps are per thread, so another thread's page is never this thread's slot page) — the conservative direction. Darwin/aarch64 only; other targets keep the cell. Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- .../perry-runtime/src/gc/dirty_page_cache.rs | 77 +++++++++++++++++++ crates/perry-runtime/src/tls_hot.rs | 20 +++++ 2 files changed, 97 insertions(+) diff --git a/crates/perry-runtime/src/gc/dirty_page_cache.rs b/crates/perry-runtime/src/gc/dirty_page_cache.rs index 0c235c79c4..1df671273b 100644 --- a/crates/perry-runtime/src/gc/dirty_page_cache.rs +++ b/crates/perry-runtime/src/gc/dirty_page_cache.rs @@ -85,6 +85,75 @@ use std::cell::Cell; /// `usize::MAX` would need a 76-bit address. const NO_PAGE: usize = usize::MAX; +/// The process-global mirror of the cache, tagged with the TSD base of the +/// thread that wrote it. +/// +/// The per-thread cell stays the authority, but reaching it costs the hot-TLS +/// chain — a global slot-index load, the pthread key, `mrs`, the TSD slot, +/// then the cell — four *dependent* loads on every barrier call, and the +/// profile put the barrier entry's single hottest instruction on that chain. +/// This mirror is read with one `mrs` and two loads that do not depend on +/// each other: if the owner word names the calling thread, the page word is +/// that thread's own most recent write (every path that writes or clears the +/// cell also writes here), so the compare is exactly the cell's; otherwise +/// another thread wrote last and the reader falls back to its cell. +/// +/// Why a torn read across the two words is still harmless: a reader can only +/// mis-see a page another thread cached, and heaps are per thread — a slot +/// this thread stores into is never on another thread's page — so the +/// mismatch cannot answer "already dirty" for a page this thread owns. +#[cfg(all( + target_vendor = "apple", + target_arch = "aarch64", + target_pointer_width = "64" +))] +mod mirror { + use std::sync::atomic::{AtomicUsize, Ordering}; + + static OWNER: AtomicUsize = AtomicUsize::new(0); + static PAGE: AtomicUsize = AtomicUsize::new(super::NO_PAGE); + + /// `Some(cached == page)` when the mirror is this thread's, else `None`. + #[inline(always)] + pub(super) fn probe(page: usize) -> Option { + let me = crate::tls_hot::darwin_tsd::base(); + if OWNER.load(Ordering::Relaxed) == me { + Some(PAGE.load(Ordering::Relaxed) == page) + } else { + None + } + } + + #[inline(always)] + pub(super) fn publish(page: usize) { + PAGE.store(page, Ordering::Relaxed); + OWNER.store(crate::tls_hot::darwin_tsd::base(), Ordering::Relaxed); + } + + #[inline(always)] + pub(super) fn clear() { + if OWNER.load(Ordering::Relaxed) == crate::tls_hot::darwin_tsd::base() { + PAGE.store(super::NO_PAGE, Ordering::Relaxed); + } + } +} + +#[cfg(not(all( + target_vendor = "apple", + target_arch = "aarch64", + target_pointer_width = "64" +)))] +mod mirror { + #[inline(always)] + pub(super) fn probe(_page: usize) -> Option { + None + } + #[inline(always)] + pub(super) fn publish(_page: usize) {} + #[inline(always)] + pub(super) fn clear() {} +} + /// The cache cell: an inline value in this thread's [`crate::tls_hot::HotTls`] /// — not a `std::thread_local!` (whose `_tlv_get_addr` was ~1% of a 5k-entity /// ECS frame by itself) and not a generic hot slot either: this is the HIT @@ -101,6 +170,12 @@ fn cell() -> &'static Cell { #[inline] pub(super) fn dirty_old_page_already_marked(page: usize) -> bool { debug_assert_ne!(page, NO_PAGE, "page number collides with the empty marker"); + // A stale mirror read (another thread published between the two loads) + // can only answer "not cached" for a page this thread owns — the + // conservative direction — so the cell is not re-consulted on a miss. + if let Some(hit) = mirror::probe(page) { + return hit; + } cell().get() == page } @@ -109,6 +184,7 @@ pub(super) fn dirty_old_page_already_marked(page: usize) -> bool { #[inline] pub(super) fn note_dirty_old_page_marked(page: usize) { cell().set(page); + mirror::publish(page); } /// Drop the cached page. Called from every path that can remove a page from @@ -117,6 +193,7 @@ pub(super) fn note_dirty_old_page_marked(page: usize) { /// not check whether the page they touched is the cached one. pub(crate) fn invalidate() { cell().set(NO_PAGE); + mirror::clear(); } /// Test-only: is the cache currently empty? Lets the #7187 Phase B tests assert diff --git a/crates/perry-runtime/src/tls_hot.rs b/crates/perry-runtime/src/tls_hot.rs index 0e42ffd6bb..74269aebe4 100644 --- a/crates/perry-runtime/src/tls_hot.rs +++ b/crates/perry-runtime/src/tls_hot.rs @@ -320,6 +320,26 @@ pub(crate) mod darwin_tsd { /// # Safety /// `slot` must be a key returned by `pthread_key_create`, so that the index /// lands inside the thread's TSD array. + /// This thread's TSD base — the per-thread constant [`get`] indexes from, + /// exposed so a hot reader can *identify* the calling thread with one + /// `mrs` and no memory access at all (the write barrier's dirty-page + /// cache mirrors its value under the writing thread's base). Same asm and + /// the same NOT-`pure` discipline as [`get`]: the value must be re-read + /// wherever execution can resume on another thread. + #[inline(always)] + pub(crate) fn base() -> usize { + let base: usize; + // SAFETY: reads a user-readable system register; no memory touched. + unsafe { + core::arch::asm!( + "mrs {b}, tpidrro_el0", + b = out(reg) base, + options(nomem, nostack, preserves_flags) + ); + } + base & !0b111 + } + #[inline(always)] pub(super) unsafe fn get(slot: usize) -> *mut u8 { let base: usize; From d66d9493c260967acdd72455d6b4d2ddcb440d2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 28 Aug 2026 11:24:26 +0000 Subject: [PATCH 2/2] changelog: fragment for #8949 Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- changelog.d/8949-barrier-cache-mirror.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/8949-barrier-cache-mirror.md diff --git a/changelog.d/8949-barrier-cache-mirror.md b/changelog.d/8949-barrier-cache-mirror.md new file mode 100644 index 0000000000..3f7cd97244 --- /dev/null +++ b/changelog.d/8949-barrier-cache-mirror.md @@ -0,0 +1 @@ +- **gc:** the write barrier's one-entry dirty-page cache is mirrored in a process global tagged with the writing thread's TSD base, so the owning thread's hit test is one `mrs` and two independent loads instead of the four-load hot-TLS chain (Darwin/aarch64; other targets keep the per-thread cell, which stays the authority).