perf: ECS round 4 — barrier early exits, inline captureless some loop, lean Map/Set lanes, empty-pop fast path, codegen-time const fold, inline hot-TLS values (−16.7%)

changelog.d/8916-barrier-dirty-page-early-exit.md

@@ -0,0 +1 @@
+- **gc:** an inline-slot store onto the page the one-entry dirty-page cache already holds returns right after the SATB prologue, skipping both the parent and the child classification — the cache's invariant (cached ⟹ recorded in `DIRTY_OLD_PAGES` and stamped dirty) already covers the store whatever the child is. Counted under `BarrierTraceCounter::DirtyPageCacheHits`. `Map` dense integer keys decide with one `as u32` round trip instead of three range tests plus the round trip. `codehz/ecs` "5k entities: 3 commands each + sync": 4.142 → 3.961 ms/op (+4.4%, 15/15 paired runs) on top of #8897.

crates/perry-runtime/src/gc/barrier/mod.rs

@@ -1220,6 +1220,26 @@ pub(super) fn write_barrier_decoded_parent(
     // Old → young check. Runtime-owned malloc GC objects are outside
     // the nursery and must be treated as old when the caller uses the
     // external-slot path for fields or side buffers.
+    // An inline slot whose page is the one the dirty-page cache names has
+    // nothing left to owe the remembered set: the cache's invariant
+    // (`dirty_page_cache`) is "cached ⟹ recorded in DIRTY_OLD_PAGES AND
+    // stamped dirty in the page metadata", and that is exactly what
+    // `remember_old_to_young_inline_slot` would establish for this slot. The
+    // SATB shading already ran in the caller's prologue. Answering here skips
+    // both page-generation classifications — the parent's and the child's —
+    // which is the whole cost of the barrier on the second and third push into
+    // the same bucket, or on every push into a large array whose tail sits on
+    // one page.
+    if !external_slot
+        && slot_addr != 0
+        && slot_addr >= parent_addr
+        && super::dirty_page_cache::dirty_old_page_already_marked(
+            crate::arena::generation_page_for_addr(slot_addr),
+        )
+    {
+        bump_write_barrier_trace_counter(BarrierTraceCounter::DirtyPageCacheHits);
+        return;
+    }
     if !barrier_parent_needs_remembering(parent_addr, external_slot) {
         bump_write_barrier_trace_counter(BarrierTraceCounter::ParentNotOldSkips);
         return;

crates/perry-runtime/src/gc/tests/barrier_decoded_parent.rs

@@ -64,6 +64,44 @@ fn runtime_write_barrier_slot_remembers_old_to_young_edge() {
     reset_remembered_set();
 }
 
+/// A second inline-slot store onto a page the dirty-page cache already names
+/// is answered by the cache alone: still exactly one remembered page, and
+/// the parent/child classifications are not consulted (the cache invariant
+/// makes the page's record sufficient).
+#[test]
+fn inline_slot_store_onto_the_cached_dirty_page_is_a_cache_hit() {
+    let _guard = GcTestIsolationGuard::new();
+    reset_remembered_set();
+
+    let young = crate::arena::arena_alloc_gc(40, 8, GC_TYPE_OBJECT) as usize;
+    let (old_obj, fields) = unsafe { alloc_old_test_object(2) };
+    let child_bits = ptr_bits(young);
+    unsafe {
+        *fields = child_bits;
+        *fields.add(1) = child_bits;
+    }
+    let page = crate::arena::generation_page_for_addr(fields as usize);
+    assert!(!old_page_dirty_for(page));
+
+    runtime_write_barrier_slot(old_obj as usize, fields as usize, child_bits);
+    assert_eq!(remembered_dirty_page_count(), 1);
+    assert!(old_page_dirty_for(page));
+
+    // Same page, next slot: the cache hit must leave the record untouched and
+    // must not require the child to be young — a value the classifier would
+    // reject still returns through the cache, because the page is covered.
+    let old_child = crate::arena::arena_alloc_gc_old(40, 8, GC_TYPE_OBJECT) as usize;
+    runtime_write_barrier_slot(old_obj as usize, fields as usize + 8, ptr_bits(old_child));
+    assert_eq!(
+        remembered_dirty_page_count(),
+        1,
+        "a store onto the cached dirty page adds no record"
+    );
+    assert!(old_page_dirty_for(page));
+
+    reset_remembered_set();
+}
+
 /// The validated-parent entry codegen takes behind its `GC_FLAG_TENURED` gate
 /// must remember exactly what the tag-dispatching entry remembers.
 #[test]

crates/perry-runtime/src/map.rs

@@ -336,9 +336,11 @@ struct NumericIndex {
 #[inline]
 fn dense_integer_key(key: NumericKey) -> Option<u32> {
     let value = f64::from_bits(key.0);
-    if !value.is_finite() || value < 0.0 || value > u32::MAX as f64 {
-        return None;
-    }
+    // `as u32` saturates (NaN → 0, negatives → 0, > u32::MAX and +inf →
+    // u32::MAX), so the round trip alone decides every case the three range
+    // tests used to pre-screen: a value that is not a finite integer in
+    // 0..=u32::MAX never converts back to itself. One conversion pair instead
+    // of three compares and a conversion pair, on every dense Map lookup.
     let integer = value as u32;
     (integer as f64 == value).then_some(integer)
 }