From 4fa1ed2b8db8a5a6fbb5d00d330fdae0a981208f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 28 Aug 2026 10:28:44 +0200 Subject: [PATCH 1/5] perf(runtime): the hottest small thread-local values live inline in HotTls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A hot-TLS slot and a named pointer field both resolve as TSD base → HotTls → slot pointer → value. On the three probes that run on nearly every store or boxed read — the write barrier's one-entry dirty-page cache, the memoized Array.prototype / Object.prototype rows consulted by every indexed array write, and the direct-mapped box-pointer caches — the profile put the barrier's remaining self time on that dependent chain rather than on anything it computed. Small Copy values with a const initial state can live in HotTls itself (TSD base → HotTls → value), so these five now do; the generic slot mechanism is unchanged for everything else, and the collector's root rewrite of the prototype rows walks the inline cells exactly as it walked the slot. Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- .../perry-runtime/src/array/prototype_addr.rs | 96 +++++------ crates/perry-runtime/src/box.rs | 156 +++++++++--------- .../perry-runtime/src/gc/dirty_page_cache.rs | 24 +-- crates/perry-runtime/src/tls_hot.rs | 32 ++++ 4 files changed, 176 insertions(+), 132 deletions(-) diff --git a/crates/perry-runtime/src/array/prototype_addr.rs b/crates/perry-runtime/src/array/prototype_addr.rs index 18401932d1..2de23a26c7 100644 --- a/crates/perry-runtime/src/array/prototype_addr.rs +++ b/crates/perry-runtime/src/array/prototype_addr.rs @@ -64,51 +64,55 @@ use std::cell::Cell; /// defect — is not representable. Adding a third memoized intrinsic address /// means bumping this and adding a row to [`PROTOTYPE_ADDR_BUILTINS`]; it is /// then covered by both halves automatically. -const PROTOTYPE_ADDR_CACHE_COUNT: usize = 2; +const PROTOTYPE_ADDR_CACHE_COUNT: usize = crate::tls_hot::INLINE_PROTOTYPE_ADDR_ROWS; /// Row index of the `Array.prototype` cell. const ARRAY_PROTO_CACHE: usize = 0; /// Row index of the `Object.prototype` cell. const OBJECT_PROTO_CACHE: usize = 1; -crate::perry_thread_local! { - /// **THIS THREAD's** lazily-memoized intrinsic prototype addresses, indexed - /// by [`ARRAY_PROTO_CACHE`] / [`OBJECT_PROTO_CACHE`]. `usize::MAX` marks a - /// row as not-yet-computed. - /// - /// Row 0 is `Array.prototype`. An out-of-bounds element read on an ordinary - /// array must fall through to `Array.prototype[index]` (ECMA-262 - /// OrdinaryGet → prototype chain), but in real code nobody adds numeric - /// indices to `Array.prototype`, so the hot OOB path stays one load until - /// the (rare) write flips `ARRAY_PROTO_HAS_INDEX`. - /// - /// Row 1 is `Object.prototype`: a numeric index installed there - /// (`Object.prototype[2] = 2`, or a defineProperty accessor) shows through - /// array HOLES and OOB reads (chain: arr → Array.prototype → - /// Object.prototype; test262 concat/S15.4.4.4_A3_T3). Consulted by the - /// typed-feedback guards and the hole/OOB read fallbacks. - /// - /// ***THESE ARE RAW ADDRESSES OF MOVABLE OBJECTS*** (#6981). - /// `Array.prototype` relocates two different ways, and BOTH leave the cache - /// pointing at a `GC_FLAG_FORWARDED` stub while every reader resolves its - /// own receiver through `clean_arr_ptr` (which follows forwarding): - /// - /// 1. `js_array_grow` — an indexed write past the dense capacity - /// (`Array.prototype[300] = v`) reallocates and forwards the old head; - /// 2. the copying young-gen minor — it evacuates the prototype and - /// forwards. - /// - /// A stale cache is not merely a wrong value: `array_oob_prototype_get`'s - /// self-recursion guard is `proto != receiver`, and after a move those are - /// two different addresses **for the same object**, so the guard stops - /// firing and `js_array_get_f64` ⇄ `array_oob_prototype_get` recurse until - /// the stack guard page (SIGSEGV, "excessive recursion"). Hence the two - /// defences below: [`memoized_prototype_addr`] resolves the forwarding - /// chain and self-heals, and [`scan_prototype_addr_cache_roots_mut`] lets - /// the collector rewrite the slot so the address stays live even once the - /// from-space stub is recycled. - static PROTOTYPE_ADDRS: [Cell; PROTOTYPE_ADDR_CACHE_COUNT] = - const { [const { Cell::new(usize::MAX) }; PROTOTYPE_ADDR_CACHE_COUNT] }; +/// **THIS THREAD's** lazily-memoized intrinsic prototype addresses, indexed +/// by [`ARRAY_PROTO_CACHE`] / [`OBJECT_PROTO_CACHE`]. `usize::MAX` marks a +/// row as not-yet-computed. +/// +/// Row 0 is `Array.prototype`. An out-of-bounds element read on an ordinary +/// array must fall through to `Array.prototype[index]` (ECMA-262 +/// OrdinaryGet → prototype chain), but in real code nobody adds numeric +/// indices to `Array.prototype`, so the hot OOB path stays one load until +/// the (rare) write flips `ARRAY_PROTO_HAS_INDEX`. +/// +/// Row 1 is `Object.prototype`: a numeric index installed there +/// (`Object.prototype[2] = 2`, or a defineProperty accessor) shows through +/// array HOLES and OOB reads (chain: arr → Array.prototype → +/// Object.prototype; test262 concat/S15.4.4.4_A3_T3). Consulted by the +/// typed-feedback guards and the hole/OOB read fallbacks. +/// +/// ***THESE ARE RAW ADDRESSES OF MOVABLE OBJECTS*** (#6981). +/// `Array.prototype` relocates two different ways, and BOTH leave the cache +/// pointing at a `GC_FLAG_FORWARDED` stub while every reader resolves its +/// own receiver through `clean_arr_ptr` (which follows forwarding): +/// +/// 1. `js_array_grow` — an indexed write past the dense capacity +/// (`Array.prototype[300] = v`) reallocates and forwards the old head; +/// 2. the copying young-gen minor — it evacuates the prototype and +/// forwards. +/// +/// A stale cache is not merely a wrong value: `array_oob_prototype_get`'s +/// self-recursion guard is `proto != receiver`, and after a move those are +/// two different addresses **for the same object**, so the guard stops +/// firing and `js_array_get_f64` ⇄ `array_oob_prototype_get` recurse until +/// the stack guard page (SIGSEGV, "excessive recursion"). Hence the two +/// defences below: [`memoized_prototype_addr`] resolves the forwarding +/// chain and self-heals, and [`scan_prototype_addr_cache_roots_mut`] lets +/// the collector rewrite the slot so the address stays live even once the +/// from-space stub is recycled. +/// +/// The rows live INLINE in this thread's [`crate::tls_hot::HotTls`]: they are +/// consulted on every indexed array write, and a generic hot slot cost one +/// more dependent load than the value itself. +#[inline(always)] +fn prototype_addrs() -> &'static [Cell; PROTOTYPE_ADDR_CACHE_COUNT] { + &crate::tls_hot::hot().prototype_addrs } /// The `globalThis` builtin whose `.prototype` fills each row of @@ -136,11 +140,9 @@ static PROTOTYPE_ADDR_BUILTINS: [&[u8]; PROTOTYPE_ADDR_CACHE_COUNT] = [b"Array", /// thread's to-space address into a cell that could be naming another agent's /// heap. pub fn scan_prototype_addr_cache_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { - PROTOTYPE_ADDRS.with(|cells| { - for cell in cells { - rewrite_prototype_addr_slot(cell, visitor); - } - }); + for cell in prototype_addrs() { + rewrite_prototype_addr_slot(cell, visitor); + } } /// The per-cell half of [`scan_prototype_addr_cache_roots_mut`]. @@ -217,7 +219,7 @@ fn heal_prototype_addr(cache: &Cell, cached: usize) -> usize { /// `globalThis` bootstrap, memoized. #[inline] fn resolve_prototype_addr(slot: usize) -> usize { - if let Some(addr) = PROTOTYPE_ADDRS.with(|cells| memoized_prototype_addr(&cells[slot])) { + if let Some(addr) = memoized_prototype_addr(&prototype_addrs()[slot]) { return addr; } bootstrap_prototype_addr(slot) @@ -252,7 +254,7 @@ fn bootstrap_prototype_addr(slot: usize) -> usize { // call into here via `note_array_proto_iterator_write`). Re-derive until it // resolves. if addr != 0 { - PROTOTYPE_ADDRS.with(|cells| cells[slot].set(addr)); + prototype_addrs()[slot].set(addr); } addr } @@ -296,7 +298,7 @@ pub(crate) fn test_prototype_addr_cache_wiring() -> [(usize, &'static [u8]); 2] /// than assumed. #[cfg(test)] pub(crate) fn test_prototype_addr_cell_count() -> usize { - PROTOTYPE_ADDRS.with(|cells| cells.len()) + prototype_addrs().len() } /// The two halves of the #6981 defences, exported so the tests can drive them diff --git a/crates/perry-runtime/src/box.rs b/crates/perry-runtime/src/box.rs index de741ae20f..5657352685 100644 --- a/crates/perry-runtime/src/box.rs +++ b/crates/perry-runtime/src/box.rs @@ -124,48 +124,58 @@ crate::perry_thread_local! { /// re-reads the same handful of boxes (`__gen_state`, `__gen_done`, /// `__gen_executing`, plus the activation's body locals) on every step, and /// activations run one at a time. -const BOX_PTR_CACHE_SLOTS: usize = 8; +const BOX_PTR_CACHE_SLOTS: usize = crate::tls_hot::INLINE_BOX_PTR_CACHE_SLOTS; -type BoxPtrCache = crate::tls_hot::HotKey<[std::cell::Cell; BOX_PTR_CACHE_SLOTS]>; +type BoxPtrCache = [std::cell::Cell; BOX_PTR_CACHE_SLOTS]; -crate::perry_thread_local! { - /// Direct-mapped **positive** cache over `BOX_REGISTRY`. - /// - /// `js_box_get`/`js_box_set` validate their operand against the registry on - /// every access (perry#4898), and that hash probe is the single largest leaf - /// in Perry's async machinery — the transform boxes every body local of an - /// `async` function, so a state machine pays one probe per local read and - /// one per write. Measured on a promise-only kernel (24 000 activations, - /// 48 000 awaits): `is_registered_{,i32_,bool_}box_ptr` were 8.2 % + 5.9 % - /// + 5.5 % of leaf samples. - /// - /// ## Why caching only positives is sound - /// - /// Box-cell memory is **never returned to the allocator**: an address - /// minted by `js_*box_alloc*` is a box cell for the life of the thread — - /// live in the registry, or (since the #7933 follow-up) parked in the - /// release quarantine/free pool, but never recycled into a non-box - /// allocation. `js_*box_release` removes a cell from the registry AND - /// evicts it from this cache (`box_ptr_cache_evict`), so a cache hit - /// still implies "currently registered": the only writer that removes a - /// registry entry clears the matching cache slot in the same call, on - /// the same thread. A hit is therefore exactly as authoritative as the - /// probe it replaces. - /// - /// A **negative** cache would NOT be sound — an address that is not a box - /// today can be minted as one tomorrow — so a miss always falls through to - /// the hash set, and only a confirmed positive is recorded. That keeps the - /// perry#4898 rejection (a read-only `__TEXT.__cstring` address that passes - /// every structural check) exactly as strict as before. - /// - /// Thread-local like the registry it fronts: a box minted on another thread - /// is not in this thread's registry, and never enters this thread's cache. - static BOX_PTR_CACHE: [std::cell::Cell; BOX_PTR_CACHE_SLOTS] = - const { [const { std::cell::Cell::new(0) }; BOX_PTR_CACHE_SLOTS] }; - static I32_BOX_PTR_CACHE: [std::cell::Cell; BOX_PTR_CACHE_SLOTS] = - const { [const { std::cell::Cell::new(0) }; BOX_PTR_CACHE_SLOTS] }; - static BOOL_BOX_PTR_CACHE: [std::cell::Cell; BOX_PTR_CACHE_SLOTS] = - const { [const { std::cell::Cell::new(0) }; BOX_PTR_CACHE_SLOTS] }; +/// Direct-mapped **positive** cache over `BOX_REGISTRY`. +/// +/// `js_box_get`/`js_box_set` validate their operand against the registry on +/// every access (perry#4898), and that hash probe is the single largest leaf +/// in Perry's async machinery — the transform boxes every body local of an +/// `async` function, so a state machine pays one probe per local read and +/// one per write. Measured on a promise-only kernel (24 000 activations, +/// 48 000 awaits): `is_registered_{,i32_,bool_}box_ptr` were 8.2 % + 5.9 % +/// + 5.5 % of leaf samples. +/// +/// ## Why caching only positives is sound +/// +/// Box-cell memory is **never returned to the allocator**: an address +/// minted by `js_*box_alloc*` is a box cell for the life of the thread — +/// live in the registry, or (since the #7933 follow-up) parked in the +/// release quarantine/free pool, but never recycled into a non-box +/// allocation. `js_*box_release` removes a cell from the registry AND +/// evicts it from this cache (`box_ptr_cache_evict`), so a cache hit +/// still implies "currently registered": the only writer that removes a +/// registry entry clears the matching cache slot in the same call, on +/// the same thread. A hit is therefore exactly as authoritative as the +/// probe it replaces. +/// +/// A **negative** cache would NOT be sound — an address that is not a box +/// today can be minted as one tomorrow — so a miss always falls through to +/// the hash set, and only a confirmed positive is recorded. That keeps the +/// perry#4898 rejection (a read-only `__TEXT.__cstring` address that passes +/// every structural check) exactly as strict as before. +/// +/// Thread-local like the registry it fronts: a box minted on another thread +/// is not in this thread's registry, and never enters this thread's cache. +/// +/// The three caches live INLINE in this thread's [`crate::tls_hot::HotTls`]: +/// a boxed-local read probes one on every access, and a generic hot slot +/// cost one more dependent load than the value itself. +#[inline(always)] +fn box_ptr_cache() -> &'static BoxPtrCache { + &crate::tls_hot::hot().box_ptr_cache +} + +#[inline(always)] +fn i32_box_ptr_cache() -> &'static BoxPtrCache { + &crate::tls_hot::hot().i32_box_ptr_cache +} + +#[inline(always)] +fn bool_box_ptr_cache() -> &'static BoxPtrCache { + &crate::tls_hot::hot().bool_box_ptr_cache } crate::perry_thread_local! { @@ -428,7 +438,7 @@ fn publish_box_cell(addr: usize, tag: usize) { BOX_REGISTRY.with(|r| { r.borrow_mut().remove(&addr); }); - box_ptr_cache_evict(&BOX_PTR_CACHE, addr); + box_ptr_cache_evict(box_ptr_cache(), addr); unsafe { (*(addr as *mut Box)).value = crate::value::TAG_UNDEFINED }; push_free_cell(addr, &BOX_FREE_HEAD); } @@ -436,7 +446,7 @@ fn publish_box_cell(addr: usize, tag: usize) { I32_BOX_REGISTRY.with(|r| { r.borrow_mut().remove(&addr); }); - box_ptr_cache_evict(&I32_BOX_PTR_CACHE, addr); + box_ptr_cache_evict(i32_box_ptr_cache(), addr); unsafe { (*(addr as *mut I32Box)).value = -1 }; push_free_cell(addr, &I32_BOX_FREE_HEAD); } @@ -444,7 +454,7 @@ fn publish_box_cell(addr: usize, tag: usize) { BOOL_BOX_REGISTRY.with(|r| { r.borrow_mut().remove(&addr); }); - box_ptr_cache_evict(&BOOL_BOX_PTR_CACHE, addr); + box_ptr_cache_evict(bool_box_ptr_cache(), addr); unsafe { (*(addr as *mut BoolBox)).value = true }; push_free_cell(addr, &BOOL_BOX_FREE_HEAD); } @@ -628,13 +638,13 @@ fn box_ptr_cache_index(addr: usize) -> usize { } #[inline(always)] -fn box_ptr_cache_hit(cache: &'static BoxPtrCache, addr: usize) -> bool { - cache.with(|slots| slots[box_ptr_cache_index(addr)].get() == addr) +fn box_ptr_cache_hit(cache: &BoxPtrCache, addr: usize) -> bool { + cache[box_ptr_cache_index(addr)].get() == addr } #[inline(always)] -fn box_ptr_cache_record(cache: &'static BoxPtrCache, addr: usize) { - cache.with(|slots| slots[box_ptr_cache_index(addr)].set(addr)); +fn box_ptr_cache_record(cache: &BoxPtrCache, addr: usize) { + cache[box_ptr_cache_index(addr)].set(addr); } /// Evict `addr` from its direct-mapped cache slot if it currently occupies @@ -643,13 +653,11 @@ fn box_ptr_cache_record(cache: &'static BoxPtrCache, addr: usize) { /// every `js_box_get`/`js_box_set` on a parked address falling through to /// the registry probe and missing. #[inline(always)] -fn box_ptr_cache_evict(cache: &'static BoxPtrCache, addr: usize) { - cache.with(|slots| { - let slot = &slots[box_ptr_cache_index(addr)]; - if slot.get() == addr { - slot.set(0); - } - }); +fn box_ptr_cache_evict(cache: &BoxPtrCache, addr: usize) { + let slot = &cache[box_ptr_cache_index(addr)]; + if slot.get() == addr { + slot.set(0); + } } /// Allocate a new box with an initial JSValue bit pattern. @@ -671,7 +679,7 @@ pub extern "C" fn js_box_alloc_bits(initial_bits: i64) -> *mut Box { BOX_REGISTRY.with(|r| { r.borrow_mut().insert(addr); }); - box_ptr_cache_record(&BOX_PTR_CACHE, addr); + box_ptr_cache_record(box_ptr_cache(), addr); return ptr; } unsafe { @@ -690,7 +698,7 @@ pub extern "C" fn js_box_alloc_bits(initial_bits: i64) -> *mut Box { BOX_REGISTRY.with(|r| { r.borrow_mut().insert(ptr as usize); }); - box_ptr_cache_record(&BOX_PTR_CACHE, ptr as usize); + box_ptr_cache_record(box_ptr_cache(), ptr as usize); ptr } } @@ -715,7 +723,7 @@ pub extern "C" fn js_i32_box_alloc(initial_value: i32) -> *mut I32Box { I32_BOX_REGISTRY.with(|r| { r.borrow_mut().insert(addr); }); - box_ptr_cache_record(&I32_BOX_PTR_CACHE, addr); + box_ptr_cache_record(i32_box_ptr_cache(), addr); return ptr; } unsafe { @@ -731,7 +739,7 @@ pub extern "C" fn js_i32_box_alloc(initial_value: i32) -> *mut I32Box { I32_BOX_REGISTRY.with(|r| { r.borrow_mut().insert(ptr as usize); }); - box_ptr_cache_record(&I32_BOX_PTR_CACHE, ptr as usize); + box_ptr_cache_record(i32_box_ptr_cache(), ptr as usize); ptr } } @@ -750,7 +758,7 @@ pub extern "C" fn js_bool_box_alloc(initial_value: i32) -> *mut BoolBox { BOOL_BOX_REGISTRY.with(|r| { r.borrow_mut().insert(addr); }); - box_ptr_cache_record(&BOOL_BOX_PTR_CACHE, addr); + box_ptr_cache_record(bool_box_ptr_cache(), addr); return ptr; } unsafe { @@ -766,7 +774,7 @@ pub extern "C" fn js_bool_box_alloc(initial_value: i32) -> *mut BoolBox { BOOL_BOX_REGISTRY.with(|r| { r.borrow_mut().insert(ptr as usize); }); - box_ptr_cache_record(&BOOL_BOX_PTR_CACHE, ptr as usize); + box_ptr_cache_record(bool_box_ptr_cache(), ptr as usize); ptr } } @@ -808,7 +816,7 @@ pub extern "C" fn js_box_release(ptr: *mut Box) { if !was_registered { return; } - box_ptr_cache_evict(&BOX_PTR_CACHE, addr); + box_ptr_cache_evict(box_ptr_cache(), addr); unsafe { // Cleared BEFORE parking: a parked cell must read as `undefined` // through any stale path, and must retain nothing for the GC (the @@ -852,7 +860,7 @@ pub extern "C" fn js_i32_box_release(ptr: *mut I32Box) { if !was_registered { return; } - box_ptr_cache_evict(&I32_BOX_PTR_CACHE, addr); + box_ptr_cache_evict(i32_box_ptr_cache(), addr); unsafe { (*ptr).value = -1; } @@ -894,7 +902,7 @@ pub extern "C" fn js_bool_box_release(ptr: *mut BoolBox) { if !was_registered { return; } - box_ptr_cache_evict(&BOOL_BOX_PTR_CACHE, addr); + box_ptr_cache_evict(bool_box_ptr_cache(), addr); unsafe { (*ptr).value = true; } @@ -1272,12 +1280,12 @@ fn is_registered_box_ptr(ptr: *mut Box) -> bool { return false; } let addr = ptr as usize; - if box_ptr_cache_hit(&BOX_PTR_CACHE, addr) { + if box_ptr_cache_hit(box_ptr_cache(), addr) { return true; } let present = BOX_REGISTRY.with(|r| r.borrow().contains(&addr)); if present { - box_ptr_cache_record(&BOX_PTR_CACHE, addr); + box_ptr_cache_record(box_ptr_cache(), addr); } present } @@ -1319,12 +1327,12 @@ fn is_registered_i32_box_ptr(ptr: *mut I32Box) -> bool { return false; } let addr = ptr as usize; - if box_ptr_cache_hit(&I32_BOX_PTR_CACHE, addr) { + if box_ptr_cache_hit(i32_box_ptr_cache(), addr) { return true; } let present = I32_BOX_REGISTRY.with(|r| r.borrow().contains(&addr)); if present { - box_ptr_cache_record(&I32_BOX_PTR_CACHE, addr); + box_ptr_cache_record(i32_box_ptr_cache(), addr); } present } @@ -1335,12 +1343,12 @@ fn is_registered_bool_box_ptr(ptr: *mut BoolBox) -> bool { return false; } let addr = ptr as usize; - if box_ptr_cache_hit(&BOOL_BOX_PTR_CACHE, addr) { + if box_ptr_cache_hit(bool_box_ptr_cache(), addr) { return true; } let present = BOOL_BOX_REGISTRY.with(|r| r.borrow().contains(&addr)); if present { - box_ptr_cache_record(&BOOL_BOX_PTR_CACHE, addr); + box_ptr_cache_record(bool_box_ptr_cache(), addr); } present } @@ -1423,12 +1431,10 @@ pub(crate) fn test_clear_box_registry() { // only for tests — and it must drop the caches for the same reason a single // release evicts one slot: otherwise a later test would see a stale "yes" // for an address this call just un-registered. - for cache in [&BOX_PTR_CACHE, &I32_BOX_PTR_CACHE, &BOOL_BOX_PTR_CACHE] { - cache.with(|slots| { - for slot in slots { - slot.set(0); - } - }); + for cache in [box_ptr_cache(), i32_box_ptr_cache(), bool_box_ptr_cache()] { + for slot in cache { + slot.set(0); + } } } diff --git a/crates/perry-runtime/src/gc/dirty_page_cache.rs b/crates/perry-runtime/src/gc/dirty_page_cache.rs index da33d1a6d2..0c235c79c4 100644 --- a/crates/perry-runtime/src/gc/dirty_page_cache.rs +++ b/crates/perry-runtime/src/gc/dirty_page_cache.rs @@ -85,26 +85,30 @@ use std::cell::Cell; /// `usize::MAX` would need a 76-bit address. const NO_PAGE: usize = usize::MAX; -// Hot TLS, not `std::thread_local!`: this is the HIT path of every old→young -// store the barrier remembers (an old bucket taking a young command each -// push), and the `_tlv_get_addr` resolution a plain thread-local pays per -// probe was ~1% of a 5k-entity ECS frame by itself. -crate::perry_thread_local! { - static LAST_DIRTY_OLD_PAGE: Cell = const { Cell::new(NO_PAGE) }; +/// The cache cell: an inline value in this thread's [`crate::tls_hot::HotTls`] +/// — not a `std::thread_local!` (whose `_tlv_get_addr` was ~1% of a 5k-entity +/// ECS frame by itself) and not a generic hot slot either: this is the HIT +/// path of every store the barrier consults (an old bucket taking a young +/// command each push), and the slot's extra dependent load was the measurable +/// part of what the barrier still cost after the parent/child classifications +/// were skipped on a hit. +#[inline(always)] +fn cell() -> &'static Cell { + &crate::tls_hot::hot().last_dirty_old_page } /// Is `page` known to be recorded already? See the module invariant. #[inline] pub(super) fn dirty_old_page_already_marked(page: usize) -> bool { debug_assert_ne!(page, NO_PAGE, "page number collides with the empty marker"); - LAST_DIRTY_OLD_PAGE.with(Cell::get) == page + cell().get() == page } /// Record that `page` is now in `DIRTY_OLD_PAGES` **and** stamped dirty in the /// arena page metadata. Callers must have established both immediately before. #[inline] pub(super) fn note_dirty_old_page_marked(page: usize) { - LAST_DIRTY_OLD_PAGE.with(|cell| cell.set(page)); + cell().set(page); } /// Drop the cached page. Called from every path that can remove a page from @@ -112,12 +116,12 @@ pub(super) fn note_dirty_old_page_marked(page: usize) { /// the module doc. Cheap enough (one thread-local store) that these callers do /// not check whether the page they touched is the cached one. pub(crate) fn invalidate() { - LAST_DIRTY_OLD_PAGE.with(|cell| cell.set(NO_PAGE)); + cell().set(NO_PAGE); } /// Test-only: is the cache currently empty? Lets the #7187 Phase B tests assert /// that an invalidation really happened rather than that nothing broke. #[cfg(test)] pub(super) fn is_empty_for_tests() -> bool { - LAST_DIRTY_OLD_PAGE.with(Cell::get) == NO_PAGE + cell().get() == NO_PAGE } diff --git a/crates/perry-runtime/src/tls_hot.rs b/crates/perry-runtime/src/tls_hot.rs index 16e279ed7c..0e42ffd6bb 100644 --- a/crates/perry-runtime/src/tls_hot.rs +++ b/crates/perry-runtime/src/tls_hot.rs @@ -153,12 +153,39 @@ pub(crate) struct HotTls { pub(crate) learned_inline_fields: *mut u8, // gc/roots/temp_roots.rs pub(crate) temp_roots: *mut u8, + // ------------------------------------------------------------------ + // Inline hot VALUES. A named pointer field and a generic slot both cost + // TSD base → `HotTls` → slot pointer → value; a value that lives here is + // one dependent load shorter (TSD base → `HotTls` → value), and on the + // hottest probes — the write barrier's dirty-page cache on every + // remembered store, the prototype rows on every indexed array write, the + // box-pointer caches on every boxed-local read — that load was the + // measurable part. Only small `Copy` values with a `const` initial state + // belong here; anything needing `Drop` stays a slot. + // ------------------------------------------------------------------ + /// `gc::dirty_page_cache` — the one-entry dirty-page cache + /// (`usize::MAX` = nothing cached). + pub(crate) last_dirty_old_page: Cell, + /// `array::prototype_addr` — this thread's memoized intrinsic prototype + /// addresses, `usize::MAX` = not yet computed. Rewritten by the + /// collector's root scan like the slot it replaced. + pub(crate) prototype_addrs: [Cell; INLINE_PROTOTYPE_ADDR_ROWS], + /// `box` — direct-mapped positive caches over the three box registries. + pub(crate) box_ptr_cache: [Cell; INLINE_BOX_PTR_CACHE_SLOTS], + pub(crate) i32_box_ptr_cache: [Cell; INLINE_BOX_PTR_CACHE_SLOTS], + pub(crate) bool_box_ptr_cache: [Cell; INLINE_BOX_PTR_CACHE_SLOTS], /// Generic slots, one per [`crate::perry_thread_local`] declaration that /// this thread has resolved at least once. Last, so the named fields above /// keep their small fixed offsets. slots: [Cell<*mut u8>; HOT_SLOT_CAPACITY], } +/// Rows of [`HotTls::prototype_addrs`]; `array::prototype_addr` sizes its +/// builtin-name table from this. +pub(crate) const INLINE_PROTOTYPE_ADDR_ROWS: usize = 2; +/// Slots of each [`HotTls`] box-pointer cache; `box` indexes with this. +pub(crate) const INLINE_BOX_PTR_CACHE_SLOTS: usize = 8; + impl HotTls { /// Read a claimed slot. `idx` must have passed the `< HOT_SLOT_CAPACITY` /// test that both sentinels fail. @@ -195,6 +222,11 @@ impl HotTls { shape_install_memo: std::ptr::null_mut(), learned_inline_fields: std::ptr::null_mut(), temp_roots: std::ptr::null_mut(), + last_dirty_old_page: Cell::new(usize::MAX), + prototype_addrs: [const { Cell::new(usize::MAX) }; INLINE_PROTOTYPE_ADDR_ROWS], + box_ptr_cache: [const { Cell::new(0) }; INLINE_BOX_PTR_CACHE_SLOTS], + i32_box_ptr_cache: [const { Cell::new(0) }; INLINE_BOX_PTR_CACHE_SLOTS], + bool_box_ptr_cache: [const { Cell::new(0) }; INLINE_BOX_PTR_CACHE_SLOTS], slots: [const { Cell::new(std::ptr::null_mut()) }; HOT_SLOT_CAPACITY], }; } From beed26ac210f538e40a69cd27dd9d1f12b56231e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 28 Aug 2026 10:55:18 +0200 Subject: [PATCH 2/5] perf(gc): the write barrier's dirty-page hit returns from a leaf entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every pointer store into an old object reached js_write_barrier_slot_validated_parent, which made two out-of-line calls before anything was decided — decode_heap_addr for the child, and incremental_mark_barrier_value, whose "no cycle anywhere" test sat inside the callee — and then entered the outlined write_barrier_decoded_parent, whose frame saves six registers, to run the one-entry dirty-page cache compare that answers the second and third push into the same bucket. The tag decode and the idle test now inline (their slow arms are cold, out of line), and the cache test is hoisted into the entry ahead of the outlined body, so a hit is a leaf path: a tag test, two static loads, the hot-TLS page compare, return. The counters and the remembered set built are unchanged; the decoded-parent body keeps its own copy of the test for its other callers. Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- crates/perry-runtime/src/gc/barrier/leaf.rs | 40 +++++++++++++++++ crates/perry-runtime/src/gc/barrier/mod.rs | 35 ++++++++------- crates/perry-runtime/src/gc/barrier_store.rs | 9 ++++ .../src/gc/tests/barrier_decoded_parent.rs | 43 +++++++++++++++++++ 4 files changed, 109 insertions(+), 18 deletions(-) create mode 100644 crates/perry-runtime/src/gc/barrier/leaf.rs diff --git a/crates/perry-runtime/src/gc/barrier/leaf.rs b/crates/perry-runtime/src/gc/barrier/leaf.rs new file mode 100644 index 0000000000..3dc63f732f --- /dev/null +++ b/crates/perry-runtime/src/gc/barrier/leaf.rs @@ -0,0 +1,40 @@ +//! The write barrier's leaf-path helpers — the pieces every entry point +//! inlines ahead of the outlined body. A sibling of `barrier/mod.rs` for the +//! 2000-line file-size gate; same module tree, same visibility semantics. + +use super::*; + +/// The authoritative arena lookup for an address-shaped raw word — the arm of +/// [`decode_heap_addr`] that a subnormal double reaches. Cold and out of line: +/// it is the only part of the decode that is more than a few compares. +#[cold] +#[inline(never)] +pub(in crate::gc) fn decode_raw_pointer_candidate(addr: usize) -> usize { + if matches!( + crate::arena::classify_heap_generation(addr), + crate::arena::HeapGeneration::Unknown + ) { + 0 + } else { + addr + } +} + +/// The barrier's cheapest exit, as an inlinable test the entry points run +/// BEFORE calling into [`write_barrier_decoded_parent`]: an inline slot whose +/// page is the one the dirty-page cache names owes the remembered set nothing +/// (the cache's invariant — see `dirty_page_cache` — is exactly what +/// `remember_old_to_young_inline_slot` would establish for this slot). Hoisted +/// so the second and third push into the same bucket return from a leaf entry, +/// paying neither the outlined function's frame nor either classification. +#[inline(always)] +pub(in crate::gc) fn inline_slot_store_on_cached_dirty_page( + parent_addr: usize, + slot_addr: usize, +) -> bool { + slot_addr != 0 + && slot_addr >= parent_addr + && super::dirty_page_cache::dirty_old_page_already_marked( + crate::arena::generation_page_for_addr(slot_addr), + ) +} diff --git a/crates/perry-runtime/src/gc/barrier/mod.rs b/crates/perry-runtime/src/gc/barrier/mod.rs index fa5ba07323..35b674657a 100644 --- a/crates/perry-runtime/src/gc/barrier/mod.rs +++ b/crates/perry-runtime/src/gc/barrier/mod.rs @@ -990,13 +990,21 @@ fn incremental_mark_barrier_value_with_valid_ptrs( true } +#[inline(always)] pub(super) fn incremental_mark_barrier_value(value_bits: u64) -> bool { // #7469: the overwhelmingly common case is "no cycle anywhere", and // proving it must not cost a thread-local resolution — this runs on every - // heap-pointer store in compiled code. + // heap-pointer store in compiled code. Inlined into every entry point so + // that proof is one static load and a branch there, not a call. if incremental_mark_barrier_globally_idle() { return false; } + incremental_mark_barrier_value_active(value_bits) +} + +/// [`incremental_mark_barrier_value`] once a cycle is known to be active. +#[inline(never)] +fn incremental_mark_barrier_value_active(value_bits: u64) -> bool { let ptr = hot_incremental_mark_valid_ptrs().get(); if ptr.is_null() { return false; @@ -1211,6 +1219,7 @@ pub(super) fn barrier_parent_addr_is_dereferenceable(parent_addr: usize) -> bool /// `*mut ArrayHeader` / `*mut ObjectHeader` / … rather than from JS value /// bits. #[inline] +#[inline(never)] pub(super) fn write_barrier_decoded_parent( parent_addr: usize, slot_addr: usize, @@ -1230,13 +1239,7 @@ pub(super) fn write_barrier_decoded_parent( // which is the whole cost of the barrier on the second and third push into // the same bucket, or on every push into a large array whose tail sits on // one page. - if !external_slot - && slot_addr != 0 - && slot_addr >= parent_addr - && super::dirty_page_cache::dirty_old_page_already_marked( - crate::arena::generation_page_for_addr(slot_addr), - ) - { + if !external_slot && inline_slot_store_on_cached_dirty_page(parent_addr, slot_addr) { bump_write_barrier_trace_counter(BarrierTraceCounter::DirtyPageCacheHits); return; } @@ -1535,7 +1538,7 @@ pub(super) fn malloc_gc_parent_addr(parent_addr: usize) -> bool { /// Accepts POINTER_TAG / STRING_TAG / BIGINT_TAG / SHORT_STRING_TAG; /// SHORT_STRING values return 0 because they're inline data, not /// heap pointers. -#[inline] +#[inline(always)] pub(super) fn decode_heap_addr(bits: u64) -> usize { let tag = bits & TAG_MASK; if tag == POINTER_TAG || tag == STRING_TAG || tag == BIGINT_TAG { @@ -1547,19 +1550,13 @@ pub(super) fn decode_heap_addr(bits: u64) -> usize { // high bits and is rejected here without paying the page-map // classification, which dominated tight numeric store loops. Only // the (rare) subnormal doubles whose bits look address-shaped fall - // through to the authoritative arena lookup. + // through to the authoritative arena lookup — out of line, so the + // tag test above inlines into every barrier entry as a leaf. let addr = bits as usize; if (bits >> 48) != 0 || addr < 0x10000 || addr & 0x7 != 0 { return 0; } - if matches!( - crate::arena::classify_heap_generation(addr), - crate::arena::HeapGeneration::Unknown - ) { - 0 - } else { - addr - } + decode_raw_pointer_candidate(addr) } else { // SHORT_STRING_TAG (0x7FF9), INT32_TAG (0x7FFE), // primitive (0x7FFC), JS_HANDLE (0x7FFB) — none are @@ -1931,6 +1928,7 @@ pub(super) fn remembered_dirty_page_count() -> usize { }) } +mod leaf; /// Gen-GC Phase C: read the current remembered set size — used /// by tests and `PERRY_GC_DIAG=1` output to confirm barrier /// activity. Returns 0 in Phase C1 since no codegen-emitted @@ -1939,6 +1937,7 @@ pub(super) fn remembered_dirty_page_count() -> usize { // module purely for the 2000-line file-size gate; same module tree, same // visibility semantics (the statics they read are pub(super)/pub(crate)). mod maintenance; +pub(super) use leaf::{decode_raw_pointer_candidate, inline_slot_store_on_cached_dirty_page}; pub(super) use super::barrier_store::{barrier_child_prologue, barrier_remembering_active}; pub use maintenance::*; diff --git a/crates/perry-runtime/src/gc/barrier_store.rs b/crates/perry-runtime/src/gc/barrier_store.rs index 9ae974f446..f39ddc4bf6 100644 --- a/crates/perry-runtime/src/gc/barrier_store.rs +++ b/crates/perry-runtime/src/gc/barrier_store.rs @@ -207,6 +207,15 @@ pub extern "C" fn js_write_barrier_slot_validated_parent( bump_write_barrier_trace_counter(BarrierTraceCounter::NonPointerParentSkips); return; } + // Leaf exit for the common repeated store into one old page — see + // `inline_slot_store_on_cached_dirty_page`. + if super::barrier::inline_slot_store_on_cached_dirty_page( + parent_user as usize, + slot_addr as usize, + ) { + bump_write_barrier_trace_counter(BarrierTraceCounter::DirtyPageCacheHits); + return; + } write_barrier_decoded_parent(parent_user as usize, slot_addr as usize, child_addr, false); } diff --git a/crates/perry-runtime/src/gc/tests/barrier_decoded_parent.rs b/crates/perry-runtime/src/gc/tests/barrier_decoded_parent.rs index a3c98de5ba..2308395787 100644 --- a/crates/perry-runtime/src/gc/tests/barrier_decoded_parent.rs +++ b/crates/perry-runtime/src/gc/tests/barrier_decoded_parent.rs @@ -102,6 +102,49 @@ fn inline_slot_store_onto_the_cached_dirty_page_is_a_cache_hit() { reset_remembered_set(); } +/// The same cache hit through the validated-parent entry codegen calls: the +/// hoisted `inline_slot_store_on_cached_dirty_page` test answers before the +/// outlined barrier body is entered — still one remembered page, and a child +/// the classifier would reject still returns through the cache. +#[test] +fn validated_parent_entry_answers_a_cached_dirty_page_store_before_the_body() { + let _guard = GcTestIsolationGuard::new(); + reset_remembered_set(); + + let young = crate::arena::arena_alloc_gc(40, 8, GC_TYPE_OBJECT) as usize; + let (old_obj, fields) = unsafe { alloc_old_test_object(2) }; + let child_bits = ptr_bits(young); + unsafe { + *fields = child_bits; + *fields.add(1) = child_bits; + } + let page = crate::arena::generation_page_for_addr(fields as usize); + assert!(!old_page_dirty_for(page)); + + crate::gc::barrier_store::js_write_barrier_slot_validated_parent( + old_obj as u64, + fields as u64, + child_bits, + ); + assert_eq!(remembered_dirty_page_count(), 1); + assert!(old_page_dirty_for(page)); + + let old_child = crate::arena::arena_alloc_gc_old(40, 8, GC_TYPE_OBJECT) as usize; + crate::gc::barrier_store::js_write_barrier_slot_validated_parent( + old_obj as u64, + (fields as usize + 8) as u64, + ptr_bits(old_child), + ); + assert_eq!( + remembered_dirty_page_count(), + 1, + "a store onto the cached dirty page through the validated-parent entry adds no record" + ); + assert!(old_page_dirty_for(page)); + + reset_remembered_set(); +} + /// The validated-parent entry codegen takes behind its `GC_FLAG_TENURED` gate /// must remember exactly what the tag-dispatching entry remembers. #[test] From 037cb943132a28ebf97cbac2717989889d61ec67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 28 Aug 2026 11:11:09 +0200 Subject: [PATCH 3/5] lint(gc): drop the frontier entries for the box-pointer caches HotTls now holds inline Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- scripts/gc_runtime_root_holders.json | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index f009628ba9..3028ba7d3b 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -1794,10 +1794,6 @@ "file": "crates/perry-runtime/src/box.rs", "name": "BOOL_BOX_FREE_HEAD" }, - { - "file": "crates/perry-runtime/src/box.rs", - "name": "BOOL_BOX_PTR_CACHE" - }, { "file": "crates/perry-runtime/src/box.rs", "name": "BOOL_BOX_REGISTRY" @@ -1810,10 +1806,6 @@ "file": "crates/perry-runtime/src/box.rs", "name": "BOX_FREE_HEAD" }, - { - "file": "crates/perry-runtime/src/box.rs", - "name": "BOX_PTR_CACHE" - }, { "file": "crates/perry-runtime/src/box.rs", "name": "BOX_REGISTRY" @@ -1826,10 +1818,6 @@ "file": "crates/perry-runtime/src/box.rs", "name": "I32_BOX_FREE_HEAD" }, - { - "file": "crates/perry-runtime/src/box.rs", - "name": "I32_BOX_PTR_CACHE" - }, { "file": "crates/perry-runtime/src/box.rs", "name": "I32_BOX_REGISTRY" From feb11f32bf6c86b0deb4e09fe94445426f942ca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 28 Aug 2026 11:17:21 +0200 Subject: [PATCH 4/5] changelog: fragment for #8935 Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- changelog.d/8935-inline-hot-tls-leaf-barrier.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/8935-inline-hot-tls-leaf-barrier.md diff --git a/changelog.d/8935-inline-hot-tls-leaf-barrier.md b/changelog.d/8935-inline-hot-tls-leaf-barrier.md new file mode 100644 index 0000000000..1edc6b29e8 --- /dev/null +++ b/changelog.d/8935-inline-hot-tls-leaf-barrier.md @@ -0,0 +1 @@ +- **runtime/gc:** the hottest small thread-local values (the write barrier's dirty-page cache, the memoized intrinsic prototype rows, the box-pointer caches) live inline in `HotTls`, one dependent load shorter than a hot slot; and the write barrier's dirty-page hit returns from a leaf entry (tag decode and incremental-idle test inlined, the cache compare hoisted ahead of the outlined body). `codehz/ecs` "5k entities: 3 commands each + sync": +1.2% and +1.6% (15/15 paired runs each). From 356ac16a562c925f4d9941afc1a3e4dda785e3f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 28 Aug 2026 12:07:00 +0200 Subject: [PATCH 5/5] fix(gc): drop the unused `use super::*` and duplicate `#[inline]` in the leaf barrier Both are `-D warnings` errors, so the `warnings` job was red on this branch. --- crates/perry-runtime/src/gc/barrier/leaf.rs | 2 -- crates/perry-runtime/src/gc/barrier/mod.rs | 1 - 2 files changed, 3 deletions(-) diff --git a/crates/perry-runtime/src/gc/barrier/leaf.rs b/crates/perry-runtime/src/gc/barrier/leaf.rs index 3dc63f732f..30902a621d 100644 --- a/crates/perry-runtime/src/gc/barrier/leaf.rs +++ b/crates/perry-runtime/src/gc/barrier/leaf.rs @@ -2,8 +2,6 @@ //! inlines ahead of the outlined body. A sibling of `barrier/mod.rs` for the //! 2000-line file-size gate; same module tree, same visibility semantics. -use super::*; - /// The authoritative arena lookup for an address-shaped raw word — the arm of /// [`decode_heap_addr`] that a subnormal double reaches. Cold and out of line: /// it is the only part of the decode that is more than a few compares. diff --git a/crates/perry-runtime/src/gc/barrier/mod.rs b/crates/perry-runtime/src/gc/barrier/mod.rs index 35b674657a..3d4e716899 100644 --- a/crates/perry-runtime/src/gc/barrier/mod.rs +++ b/crates/perry-runtime/src/gc/barrier/mod.rs @@ -1218,7 +1218,6 @@ pub(super) fn barrier_parent_addr_is_dereferenceable(parent_addr: usize) -> bool /// here, because the Rust callers derive `parent_addr` from a live /// `*mut ArrayHeader` / `*mut ObjectHeader` / … rather than from JS value /// bits. -#[inline] #[inline(never)] pub(super) fn write_barrier_decoded_parent( parent_addr: usize,