diff --git a/changelog.d/8934-map-set-lanes-empty-pop.md b/changelog.d/8934-map-set-lanes-empty-pop.md new file mode 100644 index 0000000000..232337b1a9 --- /dev/null +++ b/changelog.d/8934-map-set-lanes-empty-pop.md @@ -0,0 +1 @@ +- **runtime:** `Map` lookups run an always-inlined numeric lane (small-map bit-identity scan or the dense integer range table) ahead of the outlined general `find_key_index`; small `Set` (≤ 8) number lookups scan the members before the two-hash side-table; `pop()` on an empty plain array answers from the header fast path; `length = 0` on an all-pointer array re-arms its layout in one registry pass. `codehz/ecs` "5k entities: 3 commands each + sync": +3.3%, +1.8%, +0.5% (15/15 paired runs each). diff --git a/crates/perry-runtime/src/array/header_gc_slots.rs b/crates/perry-runtime/src/array/header_gc_slots.rs index e6ef70887b..a56b03cf6b 100644 --- a/crates/perry-runtime/src/array/header_gc_slots.rs +++ b/crates/perry-runtime/src/array/header_gc_slots.rs @@ -170,6 +170,17 @@ pub(crate) unsafe fn rebuild_array_layout(arr: *mut ArrayHeader) { let was_all_pointer = super::header::array_object_flags_resolved(arr) & (crate::gc::GC_LAYOUT_STATE_MASK | crate::gc::GC_LAYOUT_ALL_POINTERS) == (crate::gc::GC_LAYOUT_SIDE_MASK | crate::gc::GC_LAYOUT_ALL_POINTERS); + if length == 0 && was_all_pointer { + // The branch below re-arms the all-pointer claim, and + // `layout_init_all_pointer_slots` already does everything the + // zero-slot rebuild would have done first — clears the typed-intact + // bit and forgets both per-object record kinds + // (`layout_forget_object`) before setting the state — so the rebuild + // was a second pass over the same registries for every + // `pooled.length = 0`. Skip straight to the re-arm. + crate::gc::layout_init_all_pointer_slots(arr as *mut u8); + return; + } crate::gc::layout_rebuild_from_slots(arr as *mut u8, array_elements_ptr(arr), length); if length == 0 { // `layout_rebuild_from_slots` just left the head POINTER_FREE with its diff --git a/crates/perry-runtime/src/array/mod.rs b/crates/perry-runtime/src/array/mod.rs index 8c93888319..2dcc14c09c 100644 --- a/crates/perry-runtime/src/array/mod.rs +++ b/crates/perry-runtime/src/array/mod.rs @@ -35,6 +35,8 @@ mod collection_tag_tests; #[cfg(test)] mod forwarding_tests; #[cfg(test)] +mod push_pop_tests; +#[cfg(test)] mod spread_dense_tests; #[cfg(test)] mod strict_store_tests; diff --git a/crates/perry-runtime/src/array/push_pop.rs b/crates/perry-runtime/src/array/push_pop.rs index ffa4050a81..000bcba229 100644 --- a/crates/perry-runtime/src/array/push_pop.rs +++ b/crates/perry-runtime/src/array/push_pop.rs @@ -1078,7 +1078,18 @@ pub extern "C" fn js_array_pop_f64(arr: *mut ArrayHeader) -> f64 { unsafe { let length = (*arr).length; let capacity = (*arr).capacity; - if length != 0 && length <= capacity && length <= 100_000_000 { + // An empty plain array: `Set(O, "length", 0)` is a no-op on a + // writable length (`OBJ_FLAG_ARRAY_DESCRIPTORS` is where a + // non-writable one is recorded, and it is excluded above), and + // there is no index to Get or Delete — the answer is + // `undefined`. Without this arm the drained pool's + // `pool.pop() ?? []` ran the whole generic tower (subclass and + // plain-object probes, a tracked classification, the flag + // resolution) to reach the same `length == 0` return. + if length == 0 { + return TAG_UNDEFINED_F64; + } + if length <= capacity && length <= 100_000_000 { let new_length = length - 1; let elements = (arr as *mut u8) .add(std::mem::size_of::()) diff --git a/crates/perry-runtime/src/array/push_pop_tests.rs b/crates/perry-runtime/src/array/push_pop_tests.rs new file mode 100644 index 0000000000..7cdcfd2585 --- /dev/null +++ b/crates/perry-runtime/src/array/push_pop_tests.rs @@ -0,0 +1,50 @@ +//! `pop` / `push` unit tests — split from `array/tests.rs` for the +//! 2000-line file-size gate (extract a cohesive group into a sibling file, +//! wire it with an explicit `mod`). No logic change. + +use super::*; + +/// `pop()` on an empty plain array is answered from the header fast path: +/// `undefined`, length untouched — the drained pool's `pool.pop() ?? []`. +#[test] +fn pop_on_an_empty_plain_array_is_undefined_from_the_fast_path() { + let arr = js_array_alloc(4); + assert_eq!( + js_array_pop_f64(arr).to_bits(), + crate::value::TAG_UNDEFINED, + "fresh empty array" + ); + assert_eq!(js_array_length(arr), 0); + + let arr = js_array_push_f64(arr, 1.0); + assert_eq!(js_array_pop_f64(arr), 1.0); + assert_eq!( + js_array_pop_f64(arr).to_bits(), + crate::value::TAG_UNDEFINED, + "emptied by a pop" + ); + assert_eq!(js_array_length(arr), 0); + // The slot the pop retired reads as a hole for a later length extension, + // exactly as before: nothing on the empty arm touches the payload. + js_array_set_length(arr, 1.0); + assert_eq!( + array_spec_get(arr, 0).to_bits(), + crate::value::TAG_UNDEFINED + ); +} + +#[test] +fn test_array_pop_and_push() { + let arr = js_array_alloc(4); + let arr = js_array_push_f64(arr, 1.0); + let arr = js_array_push_f64(arr, 2.0); + let arr = js_array_push_f64(arr, 3.0); + + let popped = js_array_pop_f64(arr); + assert_eq!(popped, 3.0); + assert_eq!(js_array_length(arr), 2); + + let arr = js_array_push_f64(arr, 4.0); + assert_eq!(js_array_length(arr), 3); + assert_eq!(js_array_get_f64(arr, 2), 4.0); +} diff --git a/crates/perry-runtime/src/array/tests.rs b/crates/perry-runtime/src/array/tests.rs index 1218f7ee10..9f0f775581 100644 --- a/crates/perry-runtime/src/array/tests.rs +++ b/crates/perry-runtime/src/array/tests.rs @@ -1397,22 +1397,6 @@ fn test_array_set_unchecked_basic() { assert_eq!(js_array_get_f64_unchecked(arr, 2), 3.0); } -#[test] -fn test_array_pop_and_push() { - let arr = js_array_alloc(4); - let arr = js_array_push_f64(arr, 1.0); - let arr = js_array_push_f64(arr, 2.0); - let arr = js_array_push_f64(arr, 3.0); - - let popped = js_array_pop_f64(arr); - assert_eq!(popped, 3.0); - assert_eq!(js_array_length(arr), 2); - - let arr = js_array_push_f64(arr, 4.0); - assert_eq!(js_array_length(arr), 3); - assert_eq!(js_array_get_f64(arr, 2), 4.0); -} - #[test] fn test_array_index_of() { let arr = js_array_alloc(4); diff --git a/crates/perry-runtime/src/gc/tests/layout_trace/array_layout.rs b/crates/perry-runtime/src/gc/tests/layout_trace/array_layout.rs index 996e72f564..1d6009797f 100644 --- a/crates/perry-runtime/src/gc/tests/layout_trace/array_layout.rs +++ b/crates/perry-runtime/src/gc/tests/layout_trace/array_layout.rs @@ -304,6 +304,13 @@ fn test_truncate_to_zero_keeps_the_layout_the_history_predicts() { 0, "and must not claim a raw-f64 layout it will never use" ); + // The re-arm goes straight to `layout_init_all_pointer_slots` now (no + // zero-slot rebuild first): the same end state — no per-object record of + // either kind — reached in one registry pass. + assert!( + !crate::gc::layout_tables::test_per_object_layout_present(bucket as usize), + "an emptied all-pointer bucket holds no per-object layout record" + ); // A non-pointer store into the emptied bucket still demotes the claim. bucket = crate::array::js_array_push_f64(bucket, 7.0); let demoted = unsafe { crate::array::array_object_flags_resolved(bucket) }; diff --git a/crates/perry-runtime/src/map.rs b/crates/perry-runtime/src/map.rs index 09b84ba1d8..4dae044772 100644 --- a/crates/perry-runtime/src/map.rs +++ b/crates/perry-runtime/src/map.rs @@ -561,7 +561,7 @@ impl NumericIndex { /// number from every tagged value; the zero test removes the one pair of /// distinct bit patterns (`+0`/`-0`) that SameValueZero identifies. #[inline] -fn is_plain_nonzero_number_bits(bits: u64) -> bool { +pub(crate) fn is_plain_nonzero_number_bits(bits: u64) -> bool { const QNAN_PREFIX: u64 = 0x7FF8_0000_0000_0000; (bits & QNAN_PREFIX) != QNAN_PREFIX && (bits & !(1u64 << 63)) != 0 } @@ -1406,7 +1406,63 @@ pub extern "C" fn js_map_find_key_index(map_boxed: f64, key: f64) -> f64 { #[used] static KEEP_MAP_FIND_KEY_INDEX: extern "C" fn(f64, f64) -> f64 = js_map_find_key_index; +/// The two lookups every hot `Map` does, with nothing else in the frame. +/// +/// `find_key_index` grew the string-hash, pointer-index and generic-compare +/// paths into one body, and the register pressure of those cold paths costs +/// every lookup the full prologue/epilogue (eight callee-saved GPRs and four +/// FP registers on arm64 — the profile put a third of the function's self +/// time there). The lane here answers the two shapes the numeric side-table +/// exists for — a plain (untagged, non-NaN, non-zero) number key against a +/// small map's entries by bit identity, or against the dense integer range +/// table — and returns `None` for everything else so [`find_key_index_cold`] +/// decides it. A dense-range miss is definitive for its span (every insert, +/// delete, clear and GC rewrite keeps the table exact), exactly as in the cold +/// path; a key outside the span goes to the hashed index there. +#[inline(always)] +unsafe fn find_key_index_hot(map: *const MapHeader, key: f64) -> Option { + let size = (*map).size; + let key_bits = key.to_bits(); + if !is_plain_nonzero_number_bits(key_bits) { + return None; + } + if size <= SIDE_TABLE_THRESHOLD { + let entries = entries_ptr(map); + for i in 0..size { + if ptr::read(entries.add((i as usize) * 2)).to_bits() == key_bits { + return Some(i as i32); + } + } + return Some(-1); + } + let index = (*map).numeric_index.as_ref()?; + let dense = index.dense.as_ref()?; + let integer = dense_integer_key(NumericKey(key_bits))?; + let offset = integer.checked_sub(dense.base)? as usize; + if offset >= dense.slots.len() { + return None; + } + let entry = *dense.slots.get_unchecked(offset); + if entry == DENSE_NUMERIC_EMPTY || entry >= size { + return Some(-1); + } + Some(entry as i32) +} + +#[inline(always)] pub(crate) unsafe fn find_key_index(map: *const MapHeader, key: f64) -> i32 { + if let Some(index) = find_key_index_hot(map, key) { + return index; + } + find_key_index_cold(map, key) +} + +/// Every lookup shape [`find_key_index_hot`] declines: tagged, zero and NaN +/// keys, string content hashing, the pointer-identity index, the hashed +/// numeric index, and the generic linear compare. Out of line on purpose — +/// see the hot lane. +#[inline(never)] +unsafe fn find_key_index_cold(map: *const MapHeader, key: f64) -> i32 { let size = (*map).size; let key_bits = key.to_bits(); @@ -3079,6 +3135,64 @@ mod tests { } } + #[test] + fn hot_lookup_lane_agrees_with_the_cold_path_on_every_key_shape() { + let map = js_map_alloc(4); + // Small map: bit-identity scan, hit and definitive miss. + for key in 1..=4 { + js_map_set(map, key as f64, (key * 10) as f64); + } + for key in 1..=4 { + assert_eq!(js_map_get(map, key as f64), (key * 10) as f64); + assert_eq!(js_map_has(map, key as f64), 1); + } + assert_eq!(js_map_has(map, 5.0), 0); + assert_eq!(js_map_has(map, 2.5), 0); + // Zero, -0 and NaN keys are the cold path's (SameValueZero). + js_map_set(map, 0.0, 1.0); + assert_eq!(js_map_get(map, -0.0), 1.0); + js_map_set(map, f64::NAN, 2.0); + assert_eq!(js_map_get(map, f64::from_bits(0x7FF8_0000_0000_0001)), 2.0); + + // A tagged key (a boolean) never takes the numeric lane. + let boxed_true = f64::from_bits(crate::value::TAG_TRUE); + js_map_set(map, boxed_true, 5.0); + assert_eq!(js_map_get(map, boxed_true), 5.0); + assert_eq!(js_map_has(map, boxed_true), 1); + for key in 1..=4 { + assert_eq!(js_map_get(map, key as f64), (key * 10) as f64); + } + + // Dense span (a fresh map, so the run is dense enough to build the + // range table): hit, definitive in-span miss, out-of-span keys through + // the hashed index; negative / fractional / huge keys never touch the + // range table. + let dense = js_map_alloc(4); + for key in 1_024..1_040 { + js_map_set(dense, key as f64, (key * 10) as f64); + } + let (base, len) = test_map_dense_numeric_index_range(dense) + .expect("a dense run should activate the numeric range index"); + js_map_set(dense, 1_000_000.0, 77.0); + js_map_set(dense, -3.0, 88.0); + js_map_set(dense, 4.5, 99.0); + js_map_set(dense, u32::MAX as f64 + 1.0, 66.0); + for key in 1_024..1_040 { + assert_eq!(js_map_get(dense, key as f64), (key * 10) as f64); + assert_eq!(js_map_has(dense, key as f64), 1); + } + assert_eq!(js_map_has(dense, 1_023.0), 0); + assert_eq!(js_map_has(dense, 1_040.0), 0); + assert_eq!(js_map_has(dense, (base as f64) + (len as f64) + 5.0), 0); + assert_eq!(js_map_has(dense, 1_031.5), 0); + assert_eq!(js_map_get(dense, 1_000_000.0), 77.0); + assert_eq!(js_map_get(dense, -3.0), 88.0); + assert_eq!(js_map_get(dense, 4.5), 99.0); + assert_eq!(js_map_get(dense, u32::MAX as f64 + 1.0), 66.0); + assert_eq!(js_map_has(dense, 0.0), 0); + assert_eq!(js_map_has(dense, f64::NAN), 0); + } + #[test] fn numeric_index_adapts_to_dense_high_range_without_widening_for_sparse_keys() { let map = js_map_alloc(4); diff --git a/crates/perry-runtime/src/set.rs b/crates/perry-runtime/src/set.rs index 0a7c08de60..975f8ec6fb 100644 --- a/crates/perry-runtime/src/set.rs +++ b/crates/perry-runtime/src/set.rs @@ -786,7 +786,46 @@ pub extern "C" fn js_set_find_value_index(set_boxed: f64, value: f64) -> f64 { #[used] static KEEP_SET_FIND_VALUE_INDEX: extern "C" fn(f64, f64) -> f64 = js_set_find_value_index; +/// Members of a set this small are found faster by reading them than by +/// hashing into the side-table twice (set address, then value). +const SMALL_SET_SCAN_MAX: u32 = 8; + +/// The lookup a hot `Set.has` / `Set.add` does on a small set of numbers, +/// with nothing else in the frame: a plain (untagged, non-NaN, non-zero) +/// number against the elements by bit identity. `elements[0..size)` is exactly +/// the membership (`delete` compacts, `add` normalises `-0`), and no tagged +/// value equals a number, so a bit match is a hit and a full scan is a miss. +/// Everything else — larger sets, tagged / zero / NaN values, every string — +/// is [`find_value_index_cold`]'s, through the exact side-table. +#[inline(always)] +unsafe fn find_value_index_hot(set: *const SetHeader, value: f64) -> Option { + let bits = value.to_bits(); + if !crate::map::is_plain_nonzero_number_bits(bits) { + return None; + } + let size = (*set).size; + if size > SMALL_SET_SCAN_MAX { + return None; + } + let elements = elements_ptr(set); + for i in 0..size { + if ptr::read(elements.add(i as usize)).to_bits() == bits { + return Some(i as i32); + } + } + Some(-1) +} + +#[inline(always)] pub(crate) unsafe fn find_value_index(set: *const SetHeader, value: f64) -> i32 { + if let Some(index) = find_value_index_hot(set, value) { + return index; + } + find_value_index_cold(set, value) +} + +#[inline(never)] +unsafe fn find_value_index_cold(set: *const SetHeader, value: f64) -> i32 { SET_INDEX.with(|idx| { let idx = idx.borrow(); if let Some(map) = idx.get(&(set as usize)) { @@ -2067,6 +2106,49 @@ mod tests { } } + #[test] + fn small_set_scan_lane_agrees_with_the_side_table_on_every_value_shape() { + let set = js_set_alloc(4); + for value in 1..=4 { + js_set_add(set, value as f64); + } + for value in 1..=4 { + assert_eq!(js_set_has(set, value as f64), 1); + } + assert_eq!(js_set_has(set, 5.0), 0); + assert_eq!(js_set_has(set, 2.5), 0); + assert_eq!(js_set_has(set, -1.0), 0); + // Zero, -0 and NaN are the side-table's (SameValueZero). + js_set_add(set, -0.0); + assert_eq!(js_set_has(set, 0.0), 1); + js_set_add(set, f64::NAN); + assert_eq!(js_set_has(set, f64::from_bits(0x7FF8_0000_0000_0001)), 1); + // A tagged value never takes the scan. + let boxed_true = f64::from_bits(crate::value::TAG_TRUE); + js_set_add(set, boxed_true); + assert_eq!(js_set_has(set, boxed_true), 1); + assert_eq!(js_set_has(set, 1.0), 1); + // Delete compacts, so the scan keeps seeing exactly the members. + js_set_delete(set, 2.0); + assert_eq!(js_set_has(set, 2.0), 0); + assert_eq!(js_set_has(set, 3.0), 1); + assert_eq!(js_set_has(set, 4.0), 1); + // Re-adding a present number is a no-op for the size. + let size = js_set_size(set); + js_set_add(set, 3.0); + assert_eq!(js_set_size(set), size); + // Growing past the scan bound hands every lookup to the side-table. + for value in 100..120 { + js_set_add(set, value as f64); + } + for value in 100..120 { + assert_eq!(js_set_has(set, value as f64), 1); + } + assert_eq!(js_set_has(set, 1.0), 1); + assert_eq!(js_set_has(set, 2.0), 0); + assert_eq!(js_set_has(set, 120.0), 0); + } + #[test] fn test_set_union() { let a = js_set_alloc(4);