diff --git a/changelog.d/8935-inline-hot-tls-leaf-barrier.md b/changelog.d/8935-inline-hot-tls-leaf-barrier.md new file mode 100644 index 0000000000..1edc6b29e8 --- /dev/null +++ b/changelog.d/8935-inline-hot-tls-leaf-barrier.md @@ -0,0 +1 @@ +- **runtime/gc:** the hottest small thread-local values (the write barrier's dirty-page cache, the memoized intrinsic prototype rows, the box-pointer caches) live inline in `HotTls`, one dependent load shorter than a hot slot; and the write barrier's dirty-page hit returns from a leaf entry (tag decode and incremental-idle test inlined, the cache compare hoisted ahead of the outlined body). `codehz/ecs` "5k entities: 3 commands each + sync": +1.2% and +1.6% (15/15 paired runs each). diff --git a/crates/perry-runtime/src/array/prototype_addr.rs b/crates/perry-runtime/src/array/prototype_addr.rs index 18401932d1..2de23a26c7 100644 --- a/crates/perry-runtime/src/array/prototype_addr.rs +++ b/crates/perry-runtime/src/array/prototype_addr.rs @@ -64,51 +64,55 @@ use std::cell::Cell; /// defect — is not representable. Adding a third memoized intrinsic address /// means bumping this and adding a row to [`PROTOTYPE_ADDR_BUILTINS`]; it is /// then covered by both halves automatically. -const PROTOTYPE_ADDR_CACHE_COUNT: usize = 2; +const PROTOTYPE_ADDR_CACHE_COUNT: usize = crate::tls_hot::INLINE_PROTOTYPE_ADDR_ROWS; /// Row index of the `Array.prototype` cell. const ARRAY_PROTO_CACHE: usize = 0; /// Row index of the `Object.prototype` cell. const OBJECT_PROTO_CACHE: usize = 1; -crate::perry_thread_local! { - /// **THIS THREAD's** lazily-memoized intrinsic prototype addresses, indexed - /// by [`ARRAY_PROTO_CACHE`] / [`OBJECT_PROTO_CACHE`]. `usize::MAX` marks a - /// row as not-yet-computed. - /// - /// Row 0 is `Array.prototype`. An out-of-bounds element read on an ordinary - /// array must fall through to `Array.prototype[index]` (ECMA-262 - /// OrdinaryGet → prototype chain), but in real code nobody adds numeric - /// indices to `Array.prototype`, so the hot OOB path stays one load until - /// the (rare) write flips `ARRAY_PROTO_HAS_INDEX`. - /// - /// Row 1 is `Object.prototype`: a numeric index installed there - /// (`Object.prototype[2] = 2`, or a defineProperty accessor) shows through - /// array HOLES and OOB reads (chain: arr → Array.prototype → - /// Object.prototype; test262 concat/S15.4.4.4_A3_T3). Consulted by the - /// typed-feedback guards and the hole/OOB read fallbacks. - /// - /// ***THESE ARE RAW ADDRESSES OF MOVABLE OBJECTS*** (#6981). - /// `Array.prototype` relocates two different ways, and BOTH leave the cache - /// pointing at a `GC_FLAG_FORWARDED` stub while every reader resolves its - /// own receiver through `clean_arr_ptr` (which follows forwarding): - /// - /// 1. `js_array_grow` — an indexed write past the dense capacity - /// (`Array.prototype[300] = v`) reallocates and forwards the old head; - /// 2. the copying young-gen minor — it evacuates the prototype and - /// forwards. - /// - /// A stale cache is not merely a wrong value: `array_oob_prototype_get`'s - /// self-recursion guard is `proto != receiver`, and after a move those are - /// two different addresses **for the same object**, so the guard stops - /// firing and `js_array_get_f64` ⇄ `array_oob_prototype_get` recurse until - /// the stack guard page (SIGSEGV, "excessive recursion"). Hence the two - /// defences below: [`memoized_prototype_addr`] resolves the forwarding - /// chain and self-heals, and [`scan_prototype_addr_cache_roots_mut`] lets - /// the collector rewrite the slot so the address stays live even once the - /// from-space stub is recycled. - static PROTOTYPE_ADDRS: [Cell; PROTOTYPE_ADDR_CACHE_COUNT] = - const { [const { Cell::new(usize::MAX) }; PROTOTYPE_ADDR_CACHE_COUNT] }; +/// **THIS THREAD's** lazily-memoized intrinsic prototype addresses, indexed +/// by [`ARRAY_PROTO_CACHE`] / [`OBJECT_PROTO_CACHE`]. `usize::MAX` marks a +/// row as not-yet-computed. +/// +/// Row 0 is `Array.prototype`. An out-of-bounds element read on an ordinary +/// array must fall through to `Array.prototype[index]` (ECMA-262 +/// OrdinaryGet → prototype chain), but in real code nobody adds numeric +/// indices to `Array.prototype`, so the hot OOB path stays one load until +/// the (rare) write flips `ARRAY_PROTO_HAS_INDEX`. +/// +/// Row 1 is `Object.prototype`: a numeric index installed there +/// (`Object.prototype[2] = 2`, or a defineProperty accessor) shows through +/// array HOLES and OOB reads (chain: arr → Array.prototype → +/// Object.prototype; test262 concat/S15.4.4.4_A3_T3). Consulted by the +/// typed-feedback guards and the hole/OOB read fallbacks. +/// +/// ***THESE ARE RAW ADDRESSES OF MOVABLE OBJECTS*** (#6981). +/// `Array.prototype` relocates two different ways, and BOTH leave the cache +/// pointing at a `GC_FLAG_FORWARDED` stub while every reader resolves its +/// own receiver through `clean_arr_ptr` (which follows forwarding): +/// +/// 1. `js_array_grow` — an indexed write past the dense capacity +/// (`Array.prototype[300] = v`) reallocates and forwards the old head; +/// 2. the copying young-gen minor — it evacuates the prototype and +/// forwards. +/// +/// A stale cache is not merely a wrong value: `array_oob_prototype_get`'s +/// self-recursion guard is `proto != receiver`, and after a move those are +/// two different addresses **for the same object**, so the guard stops +/// firing and `js_array_get_f64` ⇄ `array_oob_prototype_get` recurse until +/// the stack guard page (SIGSEGV, "excessive recursion"). Hence the two +/// defences below: [`memoized_prototype_addr`] resolves the forwarding +/// chain and self-heals, and [`scan_prototype_addr_cache_roots_mut`] lets +/// the collector rewrite the slot so the address stays live even once the +/// from-space stub is recycled. +/// +/// The rows live INLINE in this thread's [`crate::tls_hot::HotTls`]: they are +/// consulted on every indexed array write, and a generic hot slot cost one +/// more dependent load than the value itself. +#[inline(always)] +fn prototype_addrs() -> &'static [Cell; PROTOTYPE_ADDR_CACHE_COUNT] { + &crate::tls_hot::hot().prototype_addrs } /// The `globalThis` builtin whose `.prototype` fills each row of @@ -136,11 +140,9 @@ static PROTOTYPE_ADDR_BUILTINS: [&[u8]; PROTOTYPE_ADDR_CACHE_COUNT] = [b"Array", /// thread's to-space address into a cell that could be naming another agent's /// heap. pub fn scan_prototype_addr_cache_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { - PROTOTYPE_ADDRS.with(|cells| { - for cell in cells { - rewrite_prototype_addr_slot(cell, visitor); - } - }); + for cell in prototype_addrs() { + rewrite_prototype_addr_slot(cell, visitor); + } } /// The per-cell half of [`scan_prototype_addr_cache_roots_mut`]. @@ -217,7 +219,7 @@ fn heal_prototype_addr(cache: &Cell, cached: usize) -> usize { /// `globalThis` bootstrap, memoized. #[inline] fn resolve_prototype_addr(slot: usize) -> usize { - if let Some(addr) = PROTOTYPE_ADDRS.with(|cells| memoized_prototype_addr(&cells[slot])) { + if let Some(addr) = memoized_prototype_addr(&prototype_addrs()[slot]) { return addr; } bootstrap_prototype_addr(slot) @@ -252,7 +254,7 @@ fn bootstrap_prototype_addr(slot: usize) -> usize { // call into here via `note_array_proto_iterator_write`). Re-derive until it // resolves. if addr != 0 { - PROTOTYPE_ADDRS.with(|cells| cells[slot].set(addr)); + prototype_addrs()[slot].set(addr); } addr } @@ -296,7 +298,7 @@ pub(crate) fn test_prototype_addr_cache_wiring() -> [(usize, &'static [u8]); 2] /// than assumed. #[cfg(test)] pub(crate) fn test_prototype_addr_cell_count() -> usize { - PROTOTYPE_ADDRS.with(|cells| cells.len()) + prototype_addrs().len() } /// The two halves of the #6981 defences, exported so the tests can drive them diff --git a/crates/perry-runtime/src/box.rs b/crates/perry-runtime/src/box.rs index de741ae20f..5657352685 100644 --- a/crates/perry-runtime/src/box.rs +++ b/crates/perry-runtime/src/box.rs @@ -124,48 +124,58 @@ crate::perry_thread_local! { /// re-reads the same handful of boxes (`__gen_state`, `__gen_done`, /// `__gen_executing`, plus the activation's body locals) on every step, and /// activations run one at a time. -const BOX_PTR_CACHE_SLOTS: usize = 8; +const BOX_PTR_CACHE_SLOTS: usize = crate::tls_hot::INLINE_BOX_PTR_CACHE_SLOTS; -type BoxPtrCache = crate::tls_hot::HotKey<[std::cell::Cell; BOX_PTR_CACHE_SLOTS]>; +type BoxPtrCache = [std::cell::Cell; BOX_PTR_CACHE_SLOTS]; -crate::perry_thread_local! { - /// Direct-mapped **positive** cache over `BOX_REGISTRY`. - /// - /// `js_box_get`/`js_box_set` validate their operand against the registry on - /// every access (perry#4898), and that hash probe is the single largest leaf - /// in Perry's async machinery — the transform boxes every body local of an - /// `async` function, so a state machine pays one probe per local read and - /// one per write. Measured on a promise-only kernel (24 000 activations, - /// 48 000 awaits): `is_registered_{,i32_,bool_}box_ptr` were 8.2 % + 5.9 % - /// + 5.5 % of leaf samples. - /// - /// ## Why caching only positives is sound - /// - /// Box-cell memory is **never returned to the allocator**: an address - /// minted by `js_*box_alloc*` is a box cell for the life of the thread — - /// live in the registry, or (since the #7933 follow-up) parked in the - /// release quarantine/free pool, but never recycled into a non-box - /// allocation. `js_*box_release` removes a cell from the registry AND - /// evicts it from this cache (`box_ptr_cache_evict`), so a cache hit - /// still implies "currently registered": the only writer that removes a - /// registry entry clears the matching cache slot in the same call, on - /// the same thread. A hit is therefore exactly as authoritative as the - /// probe it replaces. - /// - /// A **negative** cache would NOT be sound — an address that is not a box - /// today can be minted as one tomorrow — so a miss always falls through to - /// the hash set, and only a confirmed positive is recorded. That keeps the - /// perry#4898 rejection (a read-only `__TEXT.__cstring` address that passes - /// every structural check) exactly as strict as before. - /// - /// Thread-local like the registry it fronts: a box minted on another thread - /// is not in this thread's registry, and never enters this thread's cache. - static BOX_PTR_CACHE: [std::cell::Cell; BOX_PTR_CACHE_SLOTS] = - const { [const { std::cell::Cell::new(0) }; BOX_PTR_CACHE_SLOTS] }; - static I32_BOX_PTR_CACHE: [std::cell::Cell; BOX_PTR_CACHE_SLOTS] = - const { [const { std::cell::Cell::new(0) }; BOX_PTR_CACHE_SLOTS] }; - static BOOL_BOX_PTR_CACHE: [std::cell::Cell; BOX_PTR_CACHE_SLOTS] = - const { [const { std::cell::Cell::new(0) }; BOX_PTR_CACHE_SLOTS] }; +/// Direct-mapped **positive** cache over `BOX_REGISTRY`. +/// +/// `js_box_get`/`js_box_set` validate their operand against the registry on +/// every access (perry#4898), and that hash probe is the single largest leaf +/// in Perry's async machinery — the transform boxes every body local of an +/// `async` function, so a state machine pays one probe per local read and +/// one per write. Measured on a promise-only kernel (24 000 activations, +/// 48 000 awaits): `is_registered_{,i32_,bool_}box_ptr` were 8.2 % + 5.9 % +/// + 5.5 % of leaf samples. +/// +/// ## Why caching only positives is sound +/// +/// Box-cell memory is **never returned to the allocator**: an address +/// minted by `js_*box_alloc*` is a box cell for the life of the thread — +/// live in the registry, or (since the #7933 follow-up) parked in the +/// release quarantine/free pool, but never recycled into a non-box +/// allocation. `js_*box_release` removes a cell from the registry AND +/// evicts it from this cache (`box_ptr_cache_evict`), so a cache hit +/// still implies "currently registered": the only writer that removes a +/// registry entry clears the matching cache slot in the same call, on +/// the same thread. A hit is therefore exactly as authoritative as the +/// probe it replaces. +/// +/// A **negative** cache would NOT be sound — an address that is not a box +/// today can be minted as one tomorrow — so a miss always falls through to +/// the hash set, and only a confirmed positive is recorded. That keeps the +/// perry#4898 rejection (a read-only `__TEXT.__cstring` address that passes +/// every structural check) exactly as strict as before. +/// +/// Thread-local like the registry it fronts: a box minted on another thread +/// is not in this thread's registry, and never enters this thread's cache. +/// +/// The three caches live INLINE in this thread's [`crate::tls_hot::HotTls`]: +/// a boxed-local read probes one on every access, and a generic hot slot +/// cost one more dependent load than the value itself. +#[inline(always)] +fn box_ptr_cache() -> &'static BoxPtrCache { + &crate::tls_hot::hot().box_ptr_cache +} + +#[inline(always)] +fn i32_box_ptr_cache() -> &'static BoxPtrCache { + &crate::tls_hot::hot().i32_box_ptr_cache +} + +#[inline(always)] +fn bool_box_ptr_cache() -> &'static BoxPtrCache { + &crate::tls_hot::hot().bool_box_ptr_cache } crate::perry_thread_local! { @@ -428,7 +438,7 @@ fn publish_box_cell(addr: usize, tag: usize) { BOX_REGISTRY.with(|r| { r.borrow_mut().remove(&addr); }); - box_ptr_cache_evict(&BOX_PTR_CACHE, addr); + box_ptr_cache_evict(box_ptr_cache(), addr); unsafe { (*(addr as *mut Box)).value = crate::value::TAG_UNDEFINED }; push_free_cell(addr, &BOX_FREE_HEAD); } @@ -436,7 +446,7 @@ fn publish_box_cell(addr: usize, tag: usize) { I32_BOX_REGISTRY.with(|r| { r.borrow_mut().remove(&addr); }); - box_ptr_cache_evict(&I32_BOX_PTR_CACHE, addr); + box_ptr_cache_evict(i32_box_ptr_cache(), addr); unsafe { (*(addr as *mut I32Box)).value = -1 }; push_free_cell(addr, &I32_BOX_FREE_HEAD); } @@ -444,7 +454,7 @@ fn publish_box_cell(addr: usize, tag: usize) { BOOL_BOX_REGISTRY.with(|r| { r.borrow_mut().remove(&addr); }); - box_ptr_cache_evict(&BOOL_BOX_PTR_CACHE, addr); + box_ptr_cache_evict(bool_box_ptr_cache(), addr); unsafe { (*(addr as *mut BoolBox)).value = true }; push_free_cell(addr, &BOOL_BOX_FREE_HEAD); } @@ -628,13 +638,13 @@ fn box_ptr_cache_index(addr: usize) -> usize { } #[inline(always)] -fn box_ptr_cache_hit(cache: &'static BoxPtrCache, addr: usize) -> bool { - cache.with(|slots| slots[box_ptr_cache_index(addr)].get() == addr) +fn box_ptr_cache_hit(cache: &BoxPtrCache, addr: usize) -> bool { + cache[box_ptr_cache_index(addr)].get() == addr } #[inline(always)] -fn box_ptr_cache_record(cache: &'static BoxPtrCache, addr: usize) { - cache.with(|slots| slots[box_ptr_cache_index(addr)].set(addr)); +fn box_ptr_cache_record(cache: &BoxPtrCache, addr: usize) { + cache[box_ptr_cache_index(addr)].set(addr); } /// Evict `addr` from its direct-mapped cache slot if it currently occupies @@ -643,13 +653,11 @@ fn box_ptr_cache_record(cache: &'static BoxPtrCache, addr: usize) { /// every `js_box_get`/`js_box_set` on a parked address falling through to /// the registry probe and missing. #[inline(always)] -fn box_ptr_cache_evict(cache: &'static BoxPtrCache, addr: usize) { - cache.with(|slots| { - let slot = &slots[box_ptr_cache_index(addr)]; - if slot.get() == addr { - slot.set(0); - } - }); +fn box_ptr_cache_evict(cache: &BoxPtrCache, addr: usize) { + let slot = &cache[box_ptr_cache_index(addr)]; + if slot.get() == addr { + slot.set(0); + } } /// Allocate a new box with an initial JSValue bit pattern. @@ -671,7 +679,7 @@ pub extern "C" fn js_box_alloc_bits(initial_bits: i64) -> *mut Box { BOX_REGISTRY.with(|r| { r.borrow_mut().insert(addr); }); - box_ptr_cache_record(&BOX_PTR_CACHE, addr); + box_ptr_cache_record(box_ptr_cache(), addr); return ptr; } unsafe { @@ -690,7 +698,7 @@ pub extern "C" fn js_box_alloc_bits(initial_bits: i64) -> *mut Box { BOX_REGISTRY.with(|r| { r.borrow_mut().insert(ptr as usize); }); - box_ptr_cache_record(&BOX_PTR_CACHE, ptr as usize); + box_ptr_cache_record(box_ptr_cache(), ptr as usize); ptr } } @@ -715,7 +723,7 @@ pub extern "C" fn js_i32_box_alloc(initial_value: i32) -> *mut I32Box { I32_BOX_REGISTRY.with(|r| { r.borrow_mut().insert(addr); }); - box_ptr_cache_record(&I32_BOX_PTR_CACHE, addr); + box_ptr_cache_record(i32_box_ptr_cache(), addr); return ptr; } unsafe { @@ -731,7 +739,7 @@ pub extern "C" fn js_i32_box_alloc(initial_value: i32) -> *mut I32Box { I32_BOX_REGISTRY.with(|r| { r.borrow_mut().insert(ptr as usize); }); - box_ptr_cache_record(&I32_BOX_PTR_CACHE, ptr as usize); + box_ptr_cache_record(i32_box_ptr_cache(), ptr as usize); ptr } } @@ -750,7 +758,7 @@ pub extern "C" fn js_bool_box_alloc(initial_value: i32) -> *mut BoolBox { BOOL_BOX_REGISTRY.with(|r| { r.borrow_mut().insert(addr); }); - box_ptr_cache_record(&BOOL_BOX_PTR_CACHE, addr); + box_ptr_cache_record(bool_box_ptr_cache(), addr); return ptr; } unsafe { @@ -766,7 +774,7 @@ pub extern "C" fn js_bool_box_alloc(initial_value: i32) -> *mut BoolBox { BOOL_BOX_REGISTRY.with(|r| { r.borrow_mut().insert(ptr as usize); }); - box_ptr_cache_record(&BOOL_BOX_PTR_CACHE, ptr as usize); + box_ptr_cache_record(bool_box_ptr_cache(), ptr as usize); ptr } } @@ -808,7 +816,7 @@ pub extern "C" fn js_box_release(ptr: *mut Box) { if !was_registered { return; } - box_ptr_cache_evict(&BOX_PTR_CACHE, addr); + box_ptr_cache_evict(box_ptr_cache(), addr); unsafe { // Cleared BEFORE parking: a parked cell must read as `undefined` // through any stale path, and must retain nothing for the GC (the @@ -852,7 +860,7 @@ pub extern "C" fn js_i32_box_release(ptr: *mut I32Box) { if !was_registered { return; } - box_ptr_cache_evict(&I32_BOX_PTR_CACHE, addr); + box_ptr_cache_evict(i32_box_ptr_cache(), addr); unsafe { (*ptr).value = -1; } @@ -894,7 +902,7 @@ pub extern "C" fn js_bool_box_release(ptr: *mut BoolBox) { if !was_registered { return; } - box_ptr_cache_evict(&BOOL_BOX_PTR_CACHE, addr); + box_ptr_cache_evict(bool_box_ptr_cache(), addr); unsafe { (*ptr).value = true; } @@ -1272,12 +1280,12 @@ fn is_registered_box_ptr(ptr: *mut Box) -> bool { return false; } let addr = ptr as usize; - if box_ptr_cache_hit(&BOX_PTR_CACHE, addr) { + if box_ptr_cache_hit(box_ptr_cache(), addr) { return true; } let present = BOX_REGISTRY.with(|r| r.borrow().contains(&addr)); if present { - box_ptr_cache_record(&BOX_PTR_CACHE, addr); + box_ptr_cache_record(box_ptr_cache(), addr); } present } @@ -1319,12 +1327,12 @@ fn is_registered_i32_box_ptr(ptr: *mut I32Box) -> bool { return false; } let addr = ptr as usize; - if box_ptr_cache_hit(&I32_BOX_PTR_CACHE, addr) { + if box_ptr_cache_hit(i32_box_ptr_cache(), addr) { return true; } let present = I32_BOX_REGISTRY.with(|r| r.borrow().contains(&addr)); if present { - box_ptr_cache_record(&I32_BOX_PTR_CACHE, addr); + box_ptr_cache_record(i32_box_ptr_cache(), addr); } present } @@ -1335,12 +1343,12 @@ fn is_registered_bool_box_ptr(ptr: *mut BoolBox) -> bool { return false; } let addr = ptr as usize; - if box_ptr_cache_hit(&BOOL_BOX_PTR_CACHE, addr) { + if box_ptr_cache_hit(bool_box_ptr_cache(), addr) { return true; } let present = BOOL_BOX_REGISTRY.with(|r| r.borrow().contains(&addr)); if present { - box_ptr_cache_record(&BOOL_BOX_PTR_CACHE, addr); + box_ptr_cache_record(bool_box_ptr_cache(), addr); } present } @@ -1423,12 +1431,10 @@ pub(crate) fn test_clear_box_registry() { // only for tests — and it must drop the caches for the same reason a single // release evicts one slot: otherwise a later test would see a stale "yes" // for an address this call just un-registered. - for cache in [&BOX_PTR_CACHE, &I32_BOX_PTR_CACHE, &BOOL_BOX_PTR_CACHE] { - cache.with(|slots| { - for slot in slots { - slot.set(0); - } - }); + for cache in [box_ptr_cache(), i32_box_ptr_cache(), bool_box_ptr_cache()] { + for slot in cache { + slot.set(0); + } } } diff --git a/crates/perry-runtime/src/gc/barrier/leaf.rs b/crates/perry-runtime/src/gc/barrier/leaf.rs new file mode 100644 index 0000000000..30902a621d --- /dev/null +++ b/crates/perry-runtime/src/gc/barrier/leaf.rs @@ -0,0 +1,38 @@ +//! The write barrier's leaf-path helpers — the pieces every entry point +//! inlines ahead of the outlined body. A sibling of `barrier/mod.rs` for the +//! 2000-line file-size gate; same module tree, same visibility semantics. + +/// The authoritative arena lookup for an address-shaped raw word — the arm of +/// [`decode_heap_addr`] that a subnormal double reaches. Cold and out of line: +/// it is the only part of the decode that is more than a few compares. +#[cold] +#[inline(never)] +pub(in crate::gc) fn decode_raw_pointer_candidate(addr: usize) -> usize { + if matches!( + crate::arena::classify_heap_generation(addr), + crate::arena::HeapGeneration::Unknown + ) { + 0 + } else { + addr + } +} + +/// The barrier's cheapest exit, as an inlinable test the entry points run +/// BEFORE calling into [`write_barrier_decoded_parent`]: an inline slot whose +/// page is the one the dirty-page cache names owes the remembered set nothing +/// (the cache's invariant — see `dirty_page_cache` — is exactly what +/// `remember_old_to_young_inline_slot` would establish for this slot). Hoisted +/// so the second and third push into the same bucket return from a leaf entry, +/// paying neither the outlined function's frame nor either classification. +#[inline(always)] +pub(in crate::gc) fn inline_slot_store_on_cached_dirty_page( + parent_addr: usize, + slot_addr: usize, +) -> bool { + slot_addr != 0 + && slot_addr >= parent_addr + && super::dirty_page_cache::dirty_old_page_already_marked( + crate::arena::generation_page_for_addr(slot_addr), + ) +} diff --git a/crates/perry-runtime/src/gc/barrier/mod.rs b/crates/perry-runtime/src/gc/barrier/mod.rs index fa5ba07323..3d4e716899 100644 --- a/crates/perry-runtime/src/gc/barrier/mod.rs +++ b/crates/perry-runtime/src/gc/barrier/mod.rs @@ -990,13 +990,21 @@ fn incremental_mark_barrier_value_with_valid_ptrs( true } +#[inline(always)] pub(super) fn incremental_mark_barrier_value(value_bits: u64) -> bool { // #7469: the overwhelmingly common case is "no cycle anywhere", and // proving it must not cost a thread-local resolution — this runs on every - // heap-pointer store in compiled code. + // heap-pointer store in compiled code. Inlined into every entry point so + // that proof is one static load and a branch there, not a call. if incremental_mark_barrier_globally_idle() { return false; } + incremental_mark_barrier_value_active(value_bits) +} + +/// [`incremental_mark_barrier_value`] once a cycle is known to be active. +#[inline(never)] +fn incremental_mark_barrier_value_active(value_bits: u64) -> bool { let ptr = hot_incremental_mark_valid_ptrs().get(); if ptr.is_null() { return false; @@ -1210,7 +1218,7 @@ pub(super) fn barrier_parent_addr_is_dereferenceable(parent_addr: usize) -> bool /// here, because the Rust callers derive `parent_addr` from a live /// `*mut ArrayHeader` / `*mut ObjectHeader` / … rather than from JS value /// bits. -#[inline] +#[inline(never)] pub(super) fn write_barrier_decoded_parent( parent_addr: usize, slot_addr: usize, @@ -1230,13 +1238,7 @@ pub(super) fn write_barrier_decoded_parent( // which is the whole cost of the barrier on the second and third push into // the same bucket, or on every push into a large array whose tail sits on // one page. - if !external_slot - && slot_addr != 0 - && slot_addr >= parent_addr - && super::dirty_page_cache::dirty_old_page_already_marked( - crate::arena::generation_page_for_addr(slot_addr), - ) - { + if !external_slot && inline_slot_store_on_cached_dirty_page(parent_addr, slot_addr) { bump_write_barrier_trace_counter(BarrierTraceCounter::DirtyPageCacheHits); return; } @@ -1535,7 +1537,7 @@ pub(super) fn malloc_gc_parent_addr(parent_addr: usize) -> bool { /// Accepts POINTER_TAG / STRING_TAG / BIGINT_TAG / SHORT_STRING_TAG; /// SHORT_STRING values return 0 because they're inline data, not /// heap pointers. -#[inline] +#[inline(always)] pub(super) fn decode_heap_addr(bits: u64) -> usize { let tag = bits & TAG_MASK; if tag == POINTER_TAG || tag == STRING_TAG || tag == BIGINT_TAG { @@ -1547,19 +1549,13 @@ pub(super) fn decode_heap_addr(bits: u64) -> usize { // high bits and is rejected here without paying the page-map // classification, which dominated tight numeric store loops. Only // the (rare) subnormal doubles whose bits look address-shaped fall - // through to the authoritative arena lookup. + // through to the authoritative arena lookup — out of line, so the + // tag test above inlines into every barrier entry as a leaf. let addr = bits as usize; if (bits >> 48) != 0 || addr < 0x10000 || addr & 0x7 != 0 { return 0; } - if matches!( - crate::arena::classify_heap_generation(addr), - crate::arena::HeapGeneration::Unknown - ) { - 0 - } else { - addr - } + decode_raw_pointer_candidate(addr) } else { // SHORT_STRING_TAG (0x7FF9), INT32_TAG (0x7FFE), // primitive (0x7FFC), JS_HANDLE (0x7FFB) — none are @@ -1931,6 +1927,7 @@ pub(super) fn remembered_dirty_page_count() -> usize { }) } +mod leaf; /// Gen-GC Phase C: read the current remembered set size — used /// by tests and `PERRY_GC_DIAG=1` output to confirm barrier /// activity. Returns 0 in Phase C1 since no codegen-emitted @@ -1939,6 +1936,7 @@ pub(super) fn remembered_dirty_page_count() -> usize { // module purely for the 2000-line file-size gate; same module tree, same // visibility semantics (the statics they read are pub(super)/pub(crate)). mod maintenance; +pub(super) use leaf::{decode_raw_pointer_candidate, inline_slot_store_on_cached_dirty_page}; pub(super) use super::barrier_store::{barrier_child_prologue, barrier_remembering_active}; pub use maintenance::*; diff --git a/crates/perry-runtime/src/gc/barrier_store.rs b/crates/perry-runtime/src/gc/barrier_store.rs index 9ae974f446..f39ddc4bf6 100644 --- a/crates/perry-runtime/src/gc/barrier_store.rs +++ b/crates/perry-runtime/src/gc/barrier_store.rs @@ -207,6 +207,15 @@ pub extern "C" fn js_write_barrier_slot_validated_parent( bump_write_barrier_trace_counter(BarrierTraceCounter::NonPointerParentSkips); return; } + // Leaf exit for the common repeated store into one old page — see + // `inline_slot_store_on_cached_dirty_page`. + if super::barrier::inline_slot_store_on_cached_dirty_page( + parent_user as usize, + slot_addr as usize, + ) { + bump_write_barrier_trace_counter(BarrierTraceCounter::DirtyPageCacheHits); + return; + } write_barrier_decoded_parent(parent_user as usize, slot_addr as usize, child_addr, false); } diff --git a/crates/perry-runtime/src/gc/dirty_page_cache.rs b/crates/perry-runtime/src/gc/dirty_page_cache.rs index da33d1a6d2..0c235c79c4 100644 --- a/crates/perry-runtime/src/gc/dirty_page_cache.rs +++ b/crates/perry-runtime/src/gc/dirty_page_cache.rs @@ -85,26 +85,30 @@ use std::cell::Cell; /// `usize::MAX` would need a 76-bit address. const NO_PAGE: usize = usize::MAX; -// Hot TLS, not `std::thread_local!`: this is the HIT path of every old→young -// store the barrier remembers (an old bucket taking a young command each -// push), and the `_tlv_get_addr` resolution a plain thread-local pays per -// probe was ~1% of a 5k-entity ECS frame by itself. -crate::perry_thread_local! { - static LAST_DIRTY_OLD_PAGE: Cell = const { Cell::new(NO_PAGE) }; +/// The cache cell: an inline value in this thread's [`crate::tls_hot::HotTls`] +/// — not a `std::thread_local!` (whose `_tlv_get_addr` was ~1% of a 5k-entity +/// ECS frame by itself) and not a generic hot slot either: this is the HIT +/// path of every store the barrier consults (an old bucket taking a young +/// command each push), and the slot's extra dependent load was the measurable +/// part of what the barrier still cost after the parent/child classifications +/// were skipped on a hit. +#[inline(always)] +fn cell() -> &'static Cell { + &crate::tls_hot::hot().last_dirty_old_page } /// Is `page` known to be recorded already? See the module invariant. #[inline] pub(super) fn dirty_old_page_already_marked(page: usize) -> bool { debug_assert_ne!(page, NO_PAGE, "page number collides with the empty marker"); - LAST_DIRTY_OLD_PAGE.with(Cell::get) == page + cell().get() == page } /// Record that `page` is now in `DIRTY_OLD_PAGES` **and** stamped dirty in the /// arena page metadata. Callers must have established both immediately before. #[inline] pub(super) fn note_dirty_old_page_marked(page: usize) { - LAST_DIRTY_OLD_PAGE.with(|cell| cell.set(page)); + cell().set(page); } /// Drop the cached page. Called from every path that can remove a page from @@ -112,12 +116,12 @@ pub(super) fn note_dirty_old_page_marked(page: usize) { /// the module doc. Cheap enough (one thread-local store) that these callers do /// not check whether the page they touched is the cached one. pub(crate) fn invalidate() { - LAST_DIRTY_OLD_PAGE.with(|cell| cell.set(NO_PAGE)); + cell().set(NO_PAGE); } /// Test-only: is the cache currently empty? Lets the #7187 Phase B tests assert /// that an invalidation really happened rather than that nothing broke. #[cfg(test)] pub(super) fn is_empty_for_tests() -> bool { - LAST_DIRTY_OLD_PAGE.with(Cell::get) == NO_PAGE + cell().get() == NO_PAGE } diff --git a/crates/perry-runtime/src/gc/tests/barrier_decoded_parent.rs b/crates/perry-runtime/src/gc/tests/barrier_decoded_parent.rs index a3c98de5ba..2308395787 100644 --- a/crates/perry-runtime/src/gc/tests/barrier_decoded_parent.rs +++ b/crates/perry-runtime/src/gc/tests/barrier_decoded_parent.rs @@ -102,6 +102,49 @@ fn inline_slot_store_onto_the_cached_dirty_page_is_a_cache_hit() { reset_remembered_set(); } +/// The same cache hit through the validated-parent entry codegen calls: the +/// hoisted `inline_slot_store_on_cached_dirty_page` test answers before the +/// outlined barrier body is entered — still one remembered page, and a child +/// the classifier would reject still returns through the cache. +#[test] +fn validated_parent_entry_answers_a_cached_dirty_page_store_before_the_body() { + let _guard = GcTestIsolationGuard::new(); + reset_remembered_set(); + + let young = crate::arena::arena_alloc_gc(40, 8, GC_TYPE_OBJECT) as usize; + let (old_obj, fields) = unsafe { alloc_old_test_object(2) }; + let child_bits = ptr_bits(young); + unsafe { + *fields = child_bits; + *fields.add(1) = child_bits; + } + let page = crate::arena::generation_page_for_addr(fields as usize); + assert!(!old_page_dirty_for(page)); + + crate::gc::barrier_store::js_write_barrier_slot_validated_parent( + old_obj as u64, + fields as u64, + child_bits, + ); + assert_eq!(remembered_dirty_page_count(), 1); + assert!(old_page_dirty_for(page)); + + let old_child = crate::arena::arena_alloc_gc_old(40, 8, GC_TYPE_OBJECT) as usize; + crate::gc::barrier_store::js_write_barrier_slot_validated_parent( + old_obj as u64, + (fields as usize + 8) as u64, + ptr_bits(old_child), + ); + assert_eq!( + remembered_dirty_page_count(), + 1, + "a store onto the cached dirty page through the validated-parent entry adds no record" + ); + assert!(old_page_dirty_for(page)); + + reset_remembered_set(); +} + /// The validated-parent entry codegen takes behind its `GC_FLAG_TENURED` gate /// must remember exactly what the tag-dispatching entry remembers. #[test] diff --git a/crates/perry-runtime/src/tls_hot.rs b/crates/perry-runtime/src/tls_hot.rs index 16e279ed7c..0e42ffd6bb 100644 --- a/crates/perry-runtime/src/tls_hot.rs +++ b/crates/perry-runtime/src/tls_hot.rs @@ -153,12 +153,39 @@ pub(crate) struct HotTls { pub(crate) learned_inline_fields: *mut u8, // gc/roots/temp_roots.rs pub(crate) temp_roots: *mut u8, + // ------------------------------------------------------------------ + // Inline hot VALUES. A named pointer field and a generic slot both cost + // TSD base → `HotTls` → slot pointer → value; a value that lives here is + // one dependent load shorter (TSD base → `HotTls` → value), and on the + // hottest probes — the write barrier's dirty-page cache on every + // remembered store, the prototype rows on every indexed array write, the + // box-pointer caches on every boxed-local read — that load was the + // measurable part. Only small `Copy` values with a `const` initial state + // belong here; anything needing `Drop` stays a slot. + // ------------------------------------------------------------------ + /// `gc::dirty_page_cache` — the one-entry dirty-page cache + /// (`usize::MAX` = nothing cached). + pub(crate) last_dirty_old_page: Cell, + /// `array::prototype_addr` — this thread's memoized intrinsic prototype + /// addresses, `usize::MAX` = not yet computed. Rewritten by the + /// collector's root scan like the slot it replaced. + pub(crate) prototype_addrs: [Cell; INLINE_PROTOTYPE_ADDR_ROWS], + /// `box` — direct-mapped positive caches over the three box registries. + pub(crate) box_ptr_cache: [Cell; INLINE_BOX_PTR_CACHE_SLOTS], + pub(crate) i32_box_ptr_cache: [Cell; INLINE_BOX_PTR_CACHE_SLOTS], + pub(crate) bool_box_ptr_cache: [Cell; INLINE_BOX_PTR_CACHE_SLOTS], /// Generic slots, one per [`crate::perry_thread_local`] declaration that /// this thread has resolved at least once. Last, so the named fields above /// keep their small fixed offsets. slots: [Cell<*mut u8>; HOT_SLOT_CAPACITY], } +/// Rows of [`HotTls::prototype_addrs`]; `array::prototype_addr` sizes its +/// builtin-name table from this. +pub(crate) const INLINE_PROTOTYPE_ADDR_ROWS: usize = 2; +/// Slots of each [`HotTls`] box-pointer cache; `box` indexes with this. +pub(crate) const INLINE_BOX_PTR_CACHE_SLOTS: usize = 8; + impl HotTls { /// Read a claimed slot. `idx` must have passed the `< HOT_SLOT_CAPACITY` /// test that both sentinels fail. @@ -195,6 +222,11 @@ impl HotTls { shape_install_memo: std::ptr::null_mut(), learned_inline_fields: std::ptr::null_mut(), temp_roots: std::ptr::null_mut(), + last_dirty_old_page: Cell::new(usize::MAX), + prototype_addrs: [const { Cell::new(usize::MAX) }; INLINE_PROTOTYPE_ADDR_ROWS], + box_ptr_cache: [const { Cell::new(0) }; INLINE_BOX_PTR_CACHE_SLOTS], + i32_box_ptr_cache: [const { Cell::new(0) }; INLINE_BOX_PTR_CACHE_SLOTS], + bool_box_ptr_cache: [const { Cell::new(0) }; INLINE_BOX_PTR_CACHE_SLOTS], slots: [const { Cell::new(std::ptr::null_mut()) }; HOT_SLOT_CAPACITY], }; } diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index f009628ba9..3028ba7d3b 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -1794,10 +1794,6 @@ "file": "crates/perry-runtime/src/box.rs", "name": "BOOL_BOX_FREE_HEAD" }, - { - "file": "crates/perry-runtime/src/box.rs", - "name": "BOOL_BOX_PTR_CACHE" - }, { "file": "crates/perry-runtime/src/box.rs", "name": "BOOL_BOX_REGISTRY" @@ -1810,10 +1806,6 @@ "file": "crates/perry-runtime/src/box.rs", "name": "BOX_FREE_HEAD" }, - { - "file": "crates/perry-runtime/src/box.rs", - "name": "BOX_PTR_CACHE" - }, { "file": "crates/perry-runtime/src/box.rs", "name": "BOX_REGISTRY" @@ -1826,10 +1818,6 @@ "file": "crates/perry-runtime/src/box.rs", "name": "I32_BOX_FREE_HEAD" }, - { - "file": "crates/perry-runtime/src/box.rs", - "name": "I32_BOX_PTR_CACHE" - }, { "file": "crates/perry-runtime/src/box.rs", "name": "I32_BOX_REGISTRY"