perf(runtime): lean Map/Set lookup lanes; empty-array pop fast path; single-pass length-0 re-arm (ECS round 4, +5.6%)

changelog.d/8934-map-set-lanes-empty-pop.md

@@ -0,0 +1 @@
+- **runtime:** `Map` lookups run an always-inlined numeric lane (small-map bit-identity scan or the dense integer range table) ahead of the outlined general `find_key_index`; small `Set` (≤ 8) number lookups scan the members before the two-hash side-table; `pop()` on an empty plain array answers from the header fast path; `length = 0` on an all-pointer array re-arms its layout in one registry pass. `codehz/ecs` "5k entities: 3 commands each + sync": +3.3%, +1.8%, +0.5% (15/15 paired runs each).

crates/perry-runtime/src/array/header_gc_slots.rs

@@ -170,6 +170,17 @@ pub(crate) unsafe fn rebuild_array_layout(arr: *mut ArrayHeader) {
     let was_all_pointer = super::header::array_object_flags_resolved(arr)
         & (crate::gc::GC_LAYOUT_STATE_MASK | crate::gc::GC_LAYOUT_ALL_POINTERS)
         == (crate::gc::GC_LAYOUT_SIDE_MASK | crate::gc::GC_LAYOUT_ALL_POINTERS);
+    if length == 0 && was_all_pointer {
+        // The branch below re-arms the all-pointer claim, and
+        // `layout_init_all_pointer_slots` already does everything the
+        // zero-slot rebuild would have done first — clears the typed-intact
+        // bit and forgets both per-object record kinds
+        // (`layout_forget_object`) before setting the state — so the rebuild
+        // was a second pass over the same registries for every
+        // `pooled.length = 0`. Skip straight to the re-arm.
+        crate::gc::layout_init_all_pointer_slots(arr as *mut u8);
+        return;
+    }
     crate::gc::layout_rebuild_from_slots(arr as *mut u8, array_elements_ptr(arr), length);
     if length == 0 {
         // `layout_rebuild_from_slots` just left the head POINTER_FREE with its

crates/perry-runtime/src/array/mod.rs

@@ -35,6 +35,8 @@ mod collection_tag_tests;
 #[cfg(test)]
 mod forwarding_tests;
 #[cfg(test)]
+mod push_pop_tests;
+#[cfg(test)]
 mod spread_dense_tests;
 #[cfg(test)]
 mod strict_store_tests;

crates/perry-runtime/src/array/push_pop.rs

@@ -1078,7 +1078,18 @@ pub extern "C" fn js_array_pop_f64(arr: *mut ArrayHeader) -> f64 {
             unsafe {
                 let length = (*arr).length;
                 let capacity = (*arr).capacity;
-                if length != 0 && length <= capacity && length <= 100_000_000 {
+                // An empty plain array: `Set(O, "length", 0)` is a no-op on a
+                // writable length (`OBJ_FLAG_ARRAY_DESCRIPTORS` is where a
+                // non-writable one is recorded, and it is excluded above), and
+                // there is no index to Get or Delete — the answer is
+                // `undefined`. Without this arm the drained pool's
+                // `pool.pop() ?? []` ran the whole generic tower (subclass and
+                // plain-object probes, a tracked classification, the flag
+                // resolution) to reach the same `length == 0` return.
+                if length == 0 {
+                    return TAG_UNDEFINED_F64;
+                }
+                if length <= capacity && length <= 100_000_000 {
                     let new_length = length - 1;
                     let elements = (arr as *mut u8)
                         .add(std::mem::size_of::<ArrayHeader>())

crates/perry-runtime/src/array/push_pop_tests.rs

@@ -0,0 +1,50 @@
+//! `pop` / `push` unit tests — split from `array/tests.rs` for the
+//! 2000-line file-size gate (extract a cohesive group into a sibling file,
+//! wire it with an explicit `mod`). No logic change.
+
+use super::*;
+
+/// `pop()` on an empty plain array is answered from the header fast path:
+/// `undefined`, length untouched — the drained pool's `pool.pop() ?? []`.
+#[test]
+fn pop_on_an_empty_plain_array_is_undefined_from_the_fast_path() {
+    let arr = js_array_alloc(4);
+    assert_eq!(
+        js_array_pop_f64(arr).to_bits(),
+        crate::value::TAG_UNDEFINED,
+        "fresh empty array"
+    );
+    assert_eq!(js_array_length(arr), 0);
+
+    let arr = js_array_push_f64(arr, 1.0);
+    assert_eq!(js_array_pop_f64(arr), 1.0);
+    assert_eq!(
+        js_array_pop_f64(arr).to_bits(),
+        crate::value::TAG_UNDEFINED,
+        "emptied by a pop"
+    );
+    assert_eq!(js_array_length(arr), 0);
+    // The slot the pop retired reads as a hole for a later length extension,
+    // exactly as before: nothing on the empty arm touches the payload.
+    js_array_set_length(arr, 1.0);
+    assert_eq!(
+        array_spec_get(arr, 0).to_bits(),
+        crate::value::TAG_UNDEFINED
+    );
+}
+
+#[test]
+fn test_array_pop_and_push() {
+    let arr = js_array_alloc(4);
+    let arr = js_array_push_f64(arr, 1.0);
+    let arr = js_array_push_f64(arr, 2.0);
+    let arr = js_array_push_f64(arr, 3.0);
+
+    let popped = js_array_pop_f64(arr);
+    assert_eq!(popped, 3.0);
+    assert_eq!(js_array_length(arr), 2);
+
+    let arr = js_array_push_f64(arr, 4.0);
+    assert_eq!(js_array_length(arr), 3);
+    assert_eq!(js_array_get_f64(arr, 2), 4.0);
+}

crates/perry-runtime/src/array/tests.rs

@@ -1397,22 +1397,6 @@ fn test_array_set_unchecked_basic() {
     assert_eq!(js_array_get_f64_unchecked(arr, 2), 3.0);
 }
 
-#[test]
-fn test_array_pop_and_push() {
-    let arr = js_array_alloc(4);
-    let arr = js_array_push_f64(arr, 1.0);
-    let arr = js_array_push_f64(arr, 2.0);
-    let arr = js_array_push_f64(arr, 3.0);
-
-    let popped = js_array_pop_f64(arr);
-    assert_eq!(popped, 3.0);
-    assert_eq!(js_array_length(arr), 2);
-
-    let arr = js_array_push_f64(arr, 4.0);
-    assert_eq!(js_array_length(arr), 3);
-    assert_eq!(js_array_get_f64(arr, 2), 4.0);
-}
-
 #[test]
 fn test_array_index_of() {
     let arr = js_array_alloc(4);

crates/perry-runtime/src/gc/tests/layout_trace/array_layout.rs

@@ -304,6 +304,13 @@ fn test_truncate_to_zero_keeps_the_layout_the_history_predicts() {
         0,
         "and must not claim a raw-f64 layout it will never use"
     );
+    // The re-arm goes straight to `layout_init_all_pointer_slots` now (no
+    // zero-slot rebuild first): the same end state — no per-object record of
+    // either kind — reached in one registry pass.
+    assert!(
+        !crate::gc::layout_tables::test_per_object_layout_present(bucket as usize),
+        "an emptied all-pointer bucket holds no per-object layout record"
+    );
     // A non-pointer store into the emptied bucket still demotes the claim.
     bucket = crate::array::js_array_push_f64(bucket, 7.0);
     let demoted = unsafe { crate::array::array_object_flags_resolved(bucket) };

crates/perry-runtime/src/map.rs

@@ -561,7 +561,7 @@ impl NumericIndex {
 /// number from every tagged value; the zero test removes the one pair of
 /// distinct bit patterns (`+0`/`-0`) that SameValueZero identifies.
 #[inline]
-fn is_plain_nonzero_number_bits(bits: u64) -> bool {
+pub(crate) fn is_plain_nonzero_number_bits(bits: u64) -> bool {
     const QNAN_PREFIX: u64 = 0x7FF8_0000_0000_0000;
     (bits & QNAN_PREFIX) != QNAN_PREFIX && (bits & !(1u64 << 63)) != 0
 }
@@ -1406,7 +1406,63 @@ pub extern "C" fn js_map_find_key_index(map_boxed: f64, key: f64) -> f64 {
 #[used]
 static KEEP_MAP_FIND_KEY_INDEX: extern "C" fn(f64, f64) -> f64 = js_map_find_key_index;
 
+/// The two lookups every hot `Map` does, with nothing else in the frame.
+///
+/// `find_key_index` grew the string-hash, pointer-index and generic-compare
+/// paths into one body, and the register pressure of those cold paths costs
+/// every lookup the full prologue/epilogue (eight callee-saved GPRs and four
+/// FP registers on arm64 — the profile put a third of the function's self
+/// time there). The lane here answers the two shapes the numeric side-table
+/// exists for — a plain (untagged, non-NaN, non-zero) number key against a
+/// small map's entries by bit identity, or against the dense integer range
+/// table — and returns `None` for everything else so [`find_key_index_cold`]
+/// decides it. A dense-range miss is definitive for its span (every insert,
+/// delete, clear and GC rewrite keeps the table exact), exactly as in the cold
+/// path; a key outside the span goes to the hashed index there.
+#[inline(always)]
+unsafe fn find_key_index_hot(map: *const MapHeader, key: f64) -> Option<i32> {
+    let size = (*map).size;
+    let key_bits = key.to_bits();
+    if !is_plain_nonzero_number_bits(key_bits) {
+        return None;
+    }
+    if size <= SIDE_TABLE_THRESHOLD {
+        let entries = entries_ptr(map);
+        for i in 0..size {
+            if ptr::read(entries.add((i as usize) * 2)).to_bits() == key_bits {
+                return Some(i as i32);
+            }
+        }
+        return Some(-1);
+    }
+    let index = (*map).numeric_index.as_ref()?;
+    let dense = index.dense.as_ref()?;
+    let integer = dense_integer_key(NumericKey(key_bits))?;
+    let offset = integer.checked_sub(dense.base)? as usize;
+    if offset >= dense.slots.len() {
+        return None;
+    }
+    let entry = *dense.slots.get_unchecked(offset);
+    if entry == DENSE_NUMERIC_EMPTY || entry >= size {
+        return Some(-1);
+    }
+    Some(entry as i32)
+}
+
+#[inline(always)]
 pub(crate) unsafe fn find_key_index(map: *const MapHeader, key: f64) -> i32 {
+    if let Some(index) = find_key_index_hot(map, key) {
+        return index;
+    }
+    find_key_index_cold(map, key)
+}
+
+/// Every lookup shape [`find_key_index_hot`] declines: tagged, zero and NaN
+/// keys, string content hashing, the pointer-identity index, the hashed
+/// numeric index, and the generic linear compare. Out of line on purpose —
+/// see the hot lane.
+#[inline(never)]
+unsafe fn find_key_index_cold(map: *const MapHeader, key: f64) -> i32 {
     let size = (*map).size;
     let key_bits = key.to_bits();
 
@@ -3079,6 +3135,64 @@ mod tests {
         }
     }
 
+    #[test]
+    fn hot_lookup_lane_agrees_with_the_cold_path_on_every_key_shape() {
+        let map = js_map_alloc(4);
+        // Small map: bit-identity scan, hit and definitive miss.
+        for key in 1..=4 {
+            js_map_set(map, key as f64, (key * 10) as f64);
+        }
+        for key in 1..=4 {
+            assert_eq!(js_map_get(map, key as f64), (key * 10) as f64);
+            assert_eq!(js_map_has(map, key as f64), 1);
+        }
+        assert_eq!(js_map_has(map, 5.0), 0);
+        assert_eq!(js_map_has(map, 2.5), 0);
+        // Zero, -0 and NaN keys are the cold path's (SameValueZero).
+        js_map_set(map, 0.0, 1.0);
+        assert_eq!(js_map_get(map, -0.0), 1.0);
+        js_map_set(map, f64::NAN, 2.0);
+        assert_eq!(js_map_get(map, f64::from_bits(0x7FF8_0000_0000_0001)), 2.0);
+
+        // A tagged key (a boolean) never takes the numeric lane.
+        let boxed_true = f64::from_bits(crate::value::TAG_TRUE);
+        js_map_set(map, boxed_true, 5.0);
+        assert_eq!(js_map_get(map, boxed_true), 5.0);
+        assert_eq!(js_map_has(map, boxed_true), 1);
+        for key in 1..=4 {
+            assert_eq!(js_map_get(map, key as f64), (key * 10) as f64);
+        }
+
+        // Dense span (a fresh map, so the run is dense enough to build the
+        // range table): hit, definitive in-span miss, out-of-span keys through
+        // the hashed index; negative / fractional / huge keys never touch the
+        // range table.
+        let dense = js_map_alloc(4);
+        for key in 1_024..1_040 {
+            js_map_set(dense, key as f64, (key * 10) as f64);
+        }
+        let (base, len) = test_map_dense_numeric_index_range(dense)
+            .expect("a dense run should activate the numeric range index");
+        js_map_set(dense, 1_000_000.0, 77.0);
+        js_map_set(dense, -3.0, 88.0);
+        js_map_set(dense, 4.5, 99.0);
+        js_map_set(dense, u32::MAX as f64 + 1.0, 66.0);
+        for key in 1_024..1_040 {
+            assert_eq!(js_map_get(dense, key as f64), (key * 10) as f64);
+            assert_eq!(js_map_has(dense, key as f64), 1);
+        }
+        assert_eq!(js_map_has(dense, 1_023.0), 0);
+        assert_eq!(js_map_has(dense, 1_040.0), 0);
+        assert_eq!(js_map_has(dense, (base as f64) + (len as f64) + 5.0), 0);
+        assert_eq!(js_map_has(dense, 1_031.5), 0);
+        assert_eq!(js_map_get(dense, 1_000_000.0), 77.0);
+        assert_eq!(js_map_get(dense, -3.0), 88.0);
+        assert_eq!(js_map_get(dense, 4.5), 99.0);
+        assert_eq!(js_map_get(dense, u32::MAX as f64 + 1.0), 66.0);
+        assert_eq!(js_map_has(dense, 0.0), 0);
+        assert_eq!(js_map_has(dense, f64::NAN), 0);
+    }
+
     #[test]
     fn numeric_index_adapts_to_dense_high_range_without_widening_for_sparse_keys() {
         let map = js_map_alloc(4);

crates/perry-runtime/src/set.rs

@@ -786,7 +786,46 @@ pub extern "C" fn js_set_find_value_index(set_boxed: f64, value: f64) -> f64 {
 #[used]
 static KEEP_SET_FIND_VALUE_INDEX: extern "C" fn(f64, f64) -> f64 = js_set_find_value_index;
 
+/// Members of a set this small are found faster by reading them than by
+/// hashing into the side-table twice (set address, then value).
+const SMALL_SET_SCAN_MAX: u32 = 8;
+
+/// The lookup a hot `Set.has` / `Set.add` does on a small set of numbers,
+/// with nothing else in the frame: a plain (untagged, non-NaN, non-zero)
+/// number against the elements by bit identity. `elements[0..size)` is exactly
+/// the membership (`delete` compacts, `add` normalises `-0`), and no tagged
+/// value equals a number, so a bit match is a hit and a full scan is a miss.
+/// Everything else — larger sets, tagged / zero / NaN values, every string —
+/// is [`find_value_index_cold`]'s, through the exact side-table.
+#[inline(always)]
+unsafe fn find_value_index_hot(set: *const SetHeader, value: f64) -> Option<i32> {
+    let bits = value.to_bits();
+    if !crate::map::is_plain_nonzero_number_bits(bits) {
+        return None;
+    }
+    let size = (*set).size;
+    if size > SMALL_SET_SCAN_MAX {
+        return None;
+    }
+    let elements = elements_ptr(set);
+    for i in 0..size {
+        if ptr::read(elements.add(i as usize)).to_bits() == bits {
+            return Some(i as i32);
+        }
+    }
+    Some(-1)
+}
+
+#[inline(always)]
 pub(crate) unsafe fn find_value_index(set: *const SetHeader, value: f64) -> i32 {
+    if let Some(index) = find_value_index_hot(set, value) {
+        return index;
+    }
+    find_value_index_cold(set, value)
+}
+
+#[inline(never)]
+unsafe fn find_value_index_cold(set: *const SetHeader, value: f64) -> i32 {
     SET_INDEX.with(|idx| {
         let idx = idx.borrow();
         if let Some(map) = idx.get(&(set as usize)) {
@@ -2067,6 +2106,49 @@ mod tests {
         }
     }
 
+    #[test]
+    fn small_set_scan_lane_agrees_with_the_side_table_on_every_value_shape() {
+        let set = js_set_alloc(4);
+        for value in 1..=4 {
+            js_set_add(set, value as f64);
+        }
+        for value in 1..=4 {
+            assert_eq!(js_set_has(set, value as f64), 1);
+        }
+        assert_eq!(js_set_has(set, 5.0), 0);
+        assert_eq!(js_set_has(set, 2.5), 0);
+        assert_eq!(js_set_has(set, -1.0), 0);
+        // Zero, -0 and NaN are the side-table's (SameValueZero).
+        js_set_add(set, -0.0);
+        assert_eq!(js_set_has(set, 0.0), 1);
+        js_set_add(set, f64::NAN);
+        assert_eq!(js_set_has(set, f64::from_bits(0x7FF8_0000_0000_0001)), 1);
+        // A tagged value never takes the scan.
+        let boxed_true = f64::from_bits(crate::value::TAG_TRUE);
+        js_set_add(set, boxed_true);
+        assert_eq!(js_set_has(set, boxed_true), 1);
+        assert_eq!(js_set_has(set, 1.0), 1);
+        // Delete compacts, so the scan keeps seeing exactly the members.
+        js_set_delete(set, 2.0);
+        assert_eq!(js_set_has(set, 2.0), 0);
+        assert_eq!(js_set_has(set, 3.0), 1);
+        assert_eq!(js_set_has(set, 4.0), 1);
+        // Re-adding a present number is a no-op for the size.
+        let size = js_set_size(set);
+        js_set_add(set, 3.0);
+        assert_eq!(js_set_size(set), size);
+        // Growing past the scan bound hands every lookup to the side-table.
+        for value in 100..120 {
+            js_set_add(set, value as f64);
+        }
+        for value in 100..120 {
+            assert_eq!(js_set_has(set, value as f64), 1);
+        }
+        assert_eq!(js_set_has(set, 1.0), 1);
+        assert_eq!(js_set_has(set, 2.0), 0);
+        assert_eq!(js_set_has(set, 120.0), 0);
+    }
+
     #[test]
     fn test_set_union() {
         let a = js_set_alloc(4);