Stride aware Fixnum Arrays

compile.c

@@ -3701,6 +3701,7 @@ iseq_peephole_optimize(rb_iseq_t *iseq, struct unreachable_unref *unref, LINK_EL
             if (vm_ci_simple(ci) && vm_ci_argc(ci) == 0 && blockiseq == NULL && vm_ci_mid(ci) == idFreeze) {
                 VALUE ary = iobj->operands[0];
                 rb_obj_reveal(ary, rb_cArray);
+                rb_ary_narrow(ary);
 
                 insn_replace_with_operands(iseq, iobj, BIN(opt_ary_freeze), 2, ary, (VALUE)ci);
                 ELEM_REMOVE(next);
@@ -5446,6 +5447,9 @@ compile_array(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *node, int pop
                 /* Create a hidden array */
                 for (; count; count--, node = RNODE_LIST(node)->nd_next)
                     rb_ary_push(ary, static_literal_value(RNODE_LIST(node)->nd_head, iseq));
+                /* An all-Fixnum template is kept narrow; duparray copies it
+                 * into a narrow array rather than sharing its buffer. */
+                rb_ary_narrow(ary);
                 RB_OBJ_SET_FROZEN_SHAREABLE(ary);
 
                 /* Emit optimized code */

gc.c

@@ -3865,6 +3865,11 @@ gc_mark_children_ctx(const struct rb_gc_mark_ctx *ctx, VALUE obj)
                 gc_mark_and_pin_internal(ctx, root);
             }
         }
+        else if (rb_array_stride(obj) != RARRAY_STRIDE_VALUE) {
+            /* narrow storage holds unboxed values, never object references, so
+             * there is nothing to trace; widening here is impossible anyway
+             * because it allocates */
+        }
         else {
             rb_len_t len = RARRAY_LEN(obj);
             const VALUE *ptr = RARRAY_CONST_PTR(obj);
@@ -4821,7 +4826,8 @@ gc_ref_update_array(void *objspace, VALUE v)
     else {
         rb_len_t len = RARRAY_LEN(v);
 
-        if (len > 0) {
+        /* narrow storage holds no object references: nothing to re-point */
+        if (len > 0 && rb_array_stride(v) == RARRAY_STRIDE_VALUE) {
             VALUE *ptr = (VALUE *)RARRAY_CONST_PTR(v);
             for (rb_len_t i = 0; i < len; i++) {
                 UPDATE_IF_MOVED(objspace, ptr[i]);

hash.c

@@ -321,6 +321,21 @@ rb_objid_hash(st_index_t index)
     return (long)st_index_hash(index);
 }
 
+/* The value NUM2LONG(rb_hash(fixnum)) would produce, without the type switch
+ * and the Fixnum round trip.  For code that hashes many Fixnums in a row. */
+long
+rb_fixnum_hash(VALUE fixnum)
+{
+    RUBY_ASSERT(FIXNUM_P(fixnum));
+    /* same clamping as the tail of any_hash() */
+    st_index_t hnum = rb_objid_hash((st_index_t)fixnum);
+    if ((SIGNED_VALUE)hnum > 0)
+        hnum &= FIXNUM_MAX;
+    else
+        hnum |= FIXNUM_MIN;
+    return (long)hnum;
+}
+
 static st_index_t
 objid_hash(VALUE obj)
 {

include/ruby/internal/core/rarray.h

@@ -111,7 +111,28 @@ enum ruby_rarray_flags {
      * store array elements.  It was a bad idea to expose this to them.
      */
     RARRAY_EMBED_LEN_MASK  = RUBY_FL_USER9 | RUBY_FL_USER8 | RUBY_FL_USER7 | RUBY_FL_USER6 |
-                                 RUBY_FL_USER5 | RUBY_FL_USER4 | RUBY_FL_USER3
+                                 RUBY_FL_USER5 | RUBY_FL_USER4 | RUBY_FL_USER3,
+
+    /**
+     * Width of one element in the backend storage.  Zero means an array of
+     * ::VALUE.  Non-zero means elements are stored unboxed and narrower than
+     * a ::VALUE; the array is widened before its buffer is handed out.  See
+     * ::rb_ary_widen.
+     *
+     * @internal
+     *
+     * 3rd parties must not be aware of this.
+     */
+    RARRAY_STRIDE_MASK     = RUBY_FL_USER11 | RUBY_FL_USER10,
+
+    /**
+     * Set when a narrowed array's unboxed elements are read as unsigned.
+     *
+     * @internal
+     *
+     * 3rd parties must not be aware of this.
+     */
+    RARRAY_STRIDE_UNSIGNED = RUBY_FL_USER2
 };
 
 /**
@@ -120,9 +141,34 @@ enum ruby_rarray_flags {
  */
 enum ruby_rarray_consts {
     /** Where ::RARRAY_EMBED_LEN_MASK resides. */
-    RARRAY_EMBED_LEN_SHIFT = RUBY_FL_USHIFT + 3
+    RARRAY_EMBED_LEN_SHIFT = RUBY_FL_USHIFT + 3,
+
+    /** Where ::RARRAY_STRIDE_MASK resides. */
+    RARRAY_STRIDE_SHIFT    = RUBY_FL_USHIFT + 10
 };
 
+/**
+ * Element widths an array can use, in bits.  ::RARRAY_STRIDE_VALUE is the
+ * default: elements are ::VALUE.
+ */
+enum ruby_rarray_stride {
+    RARRAY_STRIDE_VALUE = 0,
+    RARRAY_STRIDE_W8    = 1,
+    RARRAY_STRIDE_W16   = 2,
+    RARRAY_STRIDE_W32   = 3
+};
+
+/**
+ * @private
+ *
+ * True when a narrowed array's elements are unsigned.
+ */
+static inline int
+rb_array_stride_unsigned_p(VALUE a)
+{
+    return RB_FL_ANY_RAW(a, RARRAY_STRIDE_UNSIGNED) ? 1 : 0;
+}
+
 /** Ruby's array. */
 struct RArray {
 
@@ -163,6 +209,17 @@ struct RArray {
                 const
 #endif
                 VALUE shared_root;
+
+                /**
+                 * A frozen narrowed array never changes representation (views
+                 * may point into its buffer), so when C code asks for its
+                 * elements as ::VALUE it gets this lazily built copy instead.
+                 * Valid when the array is frozen, narrowed, and carries
+                 * ::RARRAY_SHARED_ROOT_FLAG (which also makes its capacity
+                 * read as its length, freeing this field).  NULL until first
+                 * requested.  Owned and freed with the array.
+                 */
+                const VALUE *wide_cache;
             } aux;
 
             /**
@@ -189,6 +246,26 @@ struct RArray {
 };
 
 RBIMPL_SYMBOL_EXPORT_BEGIN()
+/**
+ * Converts a narrowed array back to ::VALUE storage, in place.  A no-op for an
+ * array that already uses ::RARRAY_STRIDE_VALUE.  Allocates, so it must not be
+ * called while allocation is forbidden, for instance during GC marking.
+ *
+ * @param[out]  ary  An object of ::RArray.
+ */
+void rb_ary_widen(VALUE ary);
+
+/**
+ * The elements of a narrowed array as a ::VALUE buffer.  A mutable array is
+ * widened in place (see ::rb_ary_widen).  A frozen array is left narrow so
+ * that arrays sharing its buffer stay valid; it returns a wide copy that the
+ * frozen array owns for the rest of its life.  Allocates on first use.
+ *
+ * @param[in]  ary  A narrowed array.
+ * @return     Its elements as ::VALUE, valid as long as @p ary is.
+ */
+const VALUE *rb_ary_wide_ptr(VALUE ary);
+
 /**
  * @private
  *
@@ -282,13 +359,29 @@ RARRAY_LENINT(VALUE ary)
     return rb_len2int(RARRAY_LEN(ary));
 }
 
-RBIMPL_ATTR_PURE_UNLESS_DEBUG()
+/**
+ * @private
+ *
+ * The element width currently used by the backend storage.
+ *
+ * @param[in]  a  An object of ::RArray.
+ * @return     One of ::ruby_rarray_stride.
+ */
+static inline int
+rb_array_stride(VALUE a)
+{
+    RBIMPL_ASSERT_TYPE(a, RUBY_T_ARRAY);
+    return (int)((RBASIC(a)->flags & RARRAY_STRIDE_MASK) >> RARRAY_STRIDE_SHIFT);
+}
+
 /**
  * @private
  *
  * This is  an implementation  detail of  RARRAY_PTR().  People  do not  use it
  * directly.
  *
+ * Not pure: a narrowed array is widened in place before its buffer is returned.
+ *
  * @param[in]  a  An object of ::RArray.
  * @return     Its backend storage.
  */
@@ -297,12 +390,16 @@ rb_array_const_ptr(VALUE a)
 {
     RBIMPL_ASSERT_TYPE(a, RUBY_T_ARRAY);
 
-    if (RB_FL_ANY_RAW(a, RARRAY_EMBED_FLAG)) {
-        return FIX_CONST_VALUE_PTR(RARRAY(a)->as.ary);
-    }
-    else {
+    /* One test for the common case (heap array of ::VALUE); see the core
+     * RARRAY_AREF for why the embed and stride bits are tested together. */
+    const VALUE flags = RBASIC(a)->flags;
+    if (RB_LIKELY((flags & (RARRAY_EMBED_FLAG | RARRAY_STRIDE_MASK)) == 0)) {
         return FIX_CONST_VALUE_PTR(RARRAY(a)->as.heap.ptr);
     }
+    if (flags & RARRAY_STRIDE_MASK) {
+        return rb_ary_wide_ptr(a);
+    }
+    return FIX_CONST_VALUE_PTR(RARRAY(a)->as.ary);
 }
 
 /**

internal/array.h

@@ -23,6 +23,86 @@
 #define RARRAY_PTR_IN_USE_FLAG  FL_USER14
 #define RARRAY_FAKEARY          FL_USER19
 
+/* The element storage as it is, in the slot or on the heap, at whatever
+ * width the array uses.  For code that moves bytes around (at
+ * ary_stride_bytes per element); readers go through RARRAY_AREF or
+ * RARRAY_CONST_PTR, which decode or widen. */
+static inline const void *
+rarray_storage_ptr(VALUE ary)
+{
+    if (FL_TEST_RAW(ary, RARRAY_EMBED_FLAG)) {
+        return (const void *)RARRAY(ary)->as.ary;
+    }
+    return (const void *)RARRAY(ary)->as.heap.ptr;
+}
+
+/* The packed integers of a narrow array.  Asserts the array is narrow: on a
+ * wide array these bytes are VALUEs, not integers. */
+static inline const void *
+rarray_packed_ptr(VALUE ary)
+{
+    RUBY_ASSERT(rb_array_stride(ary) != RARRAY_STRIDE_VALUE);
+    return rarray_storage_ptr(ary);
+}
+
+/* Element i of a packed buffer as a C integer, given its width and sign.
+ * The single place that knows the encoding.  int64_t, not long: long is
+ * 32-bit on LLP64 platforms. */
+static inline int64_t
+rarray_packed_load(const void *p, int stride, int uns, rb_len_t i)
+{
+    if (uns) {
+        switch (stride) {
+          case RARRAY_STRIDE_W8:  return ((const uint8_t  *)p)[i];
+          case RARRAY_STRIDE_W16: return ((const uint16_t *)p)[i];
+          default:                return ((const uint32_t *)p)[i];
+        }
+    }
+    switch (stride) {
+      case RARRAY_STRIDE_W8:  return ((const int8_t  *)p)[i];
+      case RARRAY_STRIDE_W16: return ((const int16_t *)p)[i];
+      default:                return ((const int32_t *)p)[i];
+    }
+}
+
+/* Load element i of a narrowed array as a C integer. */
+static inline int64_t
+rarray_narrow_load_int(VALUE ary, rb_len_t i)
+{
+    return rarray_packed_load(rarray_packed_ptr(ary), rb_array_stride(ary), rb_array_stride_unsigned_p(ary), i);
+}
+
+/* Decode element i of a narrowed array into a ::VALUE. */
+static inline VALUE
+rarray_narrow_aref(VALUE ary, rb_len_t i)
+{
+    return LL2NUM(rarray_narrow_load_int(ary, i));
+}
+
+/* Core's RARRAY_AREF replaces the public macro.  It adds a bounds assertion
+ * and decodes narrowed elements in place instead of widening the array, so
+ * read-only paths leave a narrowed array narrow.  Never allocates. */
+#undef RARRAY_AREF
+RBIMPL_ATTR_ARTIFICIAL()
+static inline VALUE
+RARRAY_AREF(VALUE ary, rb_len_t i)
+{
+    RBIMPL_ASSERT_OR_ASSUME(i >= 0);
+    RBIMPL_ASSERT_OR_ASSUME(i < RARRAY_LEN(ary));
+
+    /* One test for the common case: a heap array of VALUEs.  Checking the
+     * embed and stride bits separately costs a second branch per element,
+     * which is measurable in tight read loops over wide arrays. */
+    const VALUE flags = RBASIC(ary)->flags;
+    if (RB_LIKELY((flags & (RARRAY_EMBED_FLAG | RARRAY_STRIDE_MASK)) == 0)) {
+        return RARRAY(ary)->as.heap.ptr[i];
+    }
+    if (flags & RARRAY_STRIDE_MASK) {
+        return rarray_narrow_aref(ary, i);
+    }
+    return RARRAY(ary)->as.ary[i];
+}
+
 /* array.c */
 VALUE rb_ary_hash_values(rb_len_t len, const VALUE *elements);
 VALUE rb_ary_last(int, const VALUE *, VALUE);
@@ -34,6 +114,11 @@ size_t rb_ary_memsize(VALUE);
 VALUE rb_to_array_type(VALUE obj);
 VALUE rb_to_array(VALUE obj);
 void rb_ary_cancel_sharing(VALUE ary);
+/* Settle the representation of an array about to be frozen: pack an
+ * all-Fixnum heap array to the narrowest width that holds every value, and
+ * keep an embedded array wide.  Returns the resulting stride. */
+int rb_ary_narrow(VALUE ary);
+VALUE rb_ary_wide_or_dup(VALUE ary);
 size_t rb_ary_size_as_embedded(VALUE ary);
 void rb_ary_make_embedded(VALUE ary);
 bool rb_ary_embeddable_p(VALUE ary);
@@ -59,7 +144,6 @@ static inline VALUE
 rb_ary_entry_internal(VALUE ary, rb_len_t offset)
 {
     rb_len_t len = RARRAY_LEN(ary);
-    const VALUE *ptr = RARRAY_CONST_PTR(ary);
     if (len == 0) return Qnil;
     if (offset < 0) {
         offset += len;
@@ -68,7 +152,7 @@ rb_ary_entry_internal(VALUE ary, rb_len_t offset)
     else if (len <= offset) {
         return Qnil;
     }
-    return ptr[offset];
+    return RARRAY_AREF(ary, offset); /* narrow-aware; does not allocate */
 }
 
 static inline bool
@@ -135,16 +219,5 @@ ARY_SHARED_ROOT_REFCNT(VALUE ary)
     })
 #endif
 
-#undef RARRAY_AREF
-RBIMPL_ATTR_PURE_UNLESS_DEBUG()
-RBIMPL_ATTR_ARTIFICIAL()
-static inline VALUE
-RARRAY_AREF(VALUE ary, rb_len_t i)
-{
-    RBIMPL_ASSERT_OR_ASSUME(i >= 0);
-    RBIMPL_ASSERT_OR_ASSUME(i < RARRAY_LEN(ary));
-
-    return RARRAY_CONST_PTR(ary)[i];
-}
 
 #endif /* INTERNAL_ARRAY_H */

internal/hash.h

@@ -75,6 +75,7 @@ VALUE rb_hash_default_value(VALUE hash, VALUE key);
 VALUE rb_hash_set_default(VALUE hash, VALUE ifnone);
 VALUE rb_hash_set_default_proc(VALUE hash, VALUE proc);
 long rb_dbl_long_hash(double d);
+long rb_fixnum_hash(VALUE fixnum);
 st_table *rb_init_identtable(void);
 void rb_init_existing_identtable_with_size(st_table *tbl, st_index_t size);
 st_index_t rb_any_hash(VALUE a);

prism_compile.c

@@ -843,6 +843,9 @@ pm_static_literal_value(rb_iseq_t *iseq, const pm_node_t *node, pm_scope_node_t
             rb_ary_push(value, pm_static_literal_value(iseq, elements->nodes[index], scope_node));
         }
 
+        /* An all-Fixnum template is kept narrow; duparray copies it into a
+         * narrow array rather than sharing its buffer. */
+        rb_ary_narrow(value);
         RB_OBJ_SET_FROZEN_SHAREABLE(value);
         return value;
       }
@@ -7418,6 +7421,7 @@ pm_compile_array_node(rb_iseq_t *iseq, const pm_node_t *node, const pm_node_list
                     rb_ary_push(tmp_array, pm_static_literal_value(iseq, elements->nodes[index++], scope_node));
 
                 index--; // about to be incremented by for loop
+                rb_ary_narrow(tmp_array);
                 RB_OBJ_SET_FROZEN_SHAREABLE(tmp_array);
 
                 // Emit the optimized code.

ractor.c

@@ -1756,6 +1756,11 @@ obj_traverse_i(VALUE obj, struct obj_traverse_data *data)
         {
             rb_ary_cancel_sharing(obj);
 
+            /* Narrow storage holds unboxed values, never object references:
+             * every element would hit the RB_SPECIAL_CONST_P early return in
+             * obj_traverse_i, so there is nothing to visit.  Skip the loop. */
+            if (rb_array_stride(obj) != RARRAY_STRIDE_VALUE) break;
+
             for (rb_len_t i = 0; i < RARRAY_LEN(obj); i++) {
                 VALUE e = RARRAY_AREF(obj, i);
                 if (obj_traverse_i(e, data)) return 1;