diff --git a/buddy_alloc.h b/buddy_alloc.h index f906fbc..b5214a2 100644 --- a/buddy_alloc.h +++ b/buddy_alloc.h @@ -22,6 +22,7 @@ #include #include #include +#include #ifndef BUDDY_PRINTF #include #endif @@ -44,6 +45,12 @@ size_t buddy_sizeof(size_t memory_size); */ size_t buddy_sizeof_alignment(size_t memory_size, size_t alignment); +/* + * Returns the size of a buddy required to manage a block of the specified size + * using a non-default alignment where the buddy is embedded into the block. + */ +size_t buddy_sizeof_alignment_embedded(size_t memory_size, size_t alignment); + /* Initializes a binary buddy memory allocator at the specified location */ struct buddy *buddy_init(unsigned char *at, unsigned char *main, size_t memory_size); @@ -507,6 +514,7 @@ struct buddy_embed_check { size_t buddy_size; }; +static inline size_t bits_for_order(uint8_t order); static unsigned int is_valid_alignment(size_t alignment); static size_t buddy_tree_order_for_memory(size_t memory_size, size_t alignment); static size_t depth_for_size(struct buddy *buddy, size_t requested_size); @@ -543,6 +551,37 @@ size_t buddy_sizeof_alignment(size_t memory_size, size_t alignment) { return sizeof(struct buddy) + buddy_tree_sizeof((uint8_t)buddy_tree_order); } +size_t buddy_sizeof_alignment_embedded(size_t memory_size, size_t alignment) { + size_t buddy_tree_order, buddy_size, buddy_nmembers; + + if (!is_valid_alignment(alignment)) { + return 0; /* invalid */ + } + if (memory_size < alignment) { + return 0; /* invalid */ + } + buddy_tree_order = buddy_tree_order_for_memory(memory_size, alignment); + + buddy_size = sizeof(struct buddy) + buddy_tree_sizeof((uint8_t)buddy_tree_order); + buddy_nmembers = (memory_size - buddy_size) / alignment; + + while (--buddy_tree_order) { + size_t new_buddy_size = sizeof(struct buddy) + buddy_tree_sizeof((uint8_t)buddy_tree_order); + size_t free_members = (memory_size - buddy_size) / alignment; + size_t tree_members = two_to_the_power_of(buddy_tree_order - 1); + size_t new_buddy_nmembers = free_members < tree_members ? free_members : tree_members; + + if (new_buddy_nmembers >= buddy_nmembers) { + buddy_size = new_buddy_size; + buddy_nmembers = new_buddy_nmembers; + } else { + return buddy_size; + } + } + + return buddy_size; +} + struct buddy *buddy_init(unsigned char *at, unsigned char *main, size_t memory_size) { return buddy_init_alignment(at, main, memory_size, BUDDY_ALLOC_ALIGN); } @@ -635,13 +674,13 @@ struct buddy *buddy_get_embed_at_alignment(unsigned char *main, size_t memory_si } struct buddy *buddy_resize(struct buddy *buddy, size_t new_memory_size) { - if (new_memory_size == buddy->memory_size) { - return buddy; - } - if (buddy_relative_mode(buddy)) { return buddy_resize_embedded(buddy, new_memory_size); } else { + if (new_memory_size == buddy->memory_size) { + return buddy; + } + return buddy_resize_standard(buddy, new_memory_size); } } @@ -668,6 +707,7 @@ static struct buddy *buddy_resize_standard(struct buddy *buddy, size_t new_memor /* Store the new memory size and reconstruct any virtual slots */ buddy->memory_size = new_memory_size; + buddy_toggle_virtual_slots(buddy, 1); /* Resize successful */ @@ -685,6 +725,11 @@ static struct buddy *buddy_resize_embedded(struct buddy *buddy, size_t new_memor return NULL; } + /* If the embedded arena size is the same, we can just return the current buddy */ + if (check_result.offset == buddy->memory_size) { + return buddy; + } + /* Resize the allocator in the normal way */ resized = buddy_resize_standard(buddy, check_result.offset); if (! resized) { @@ -1224,8 +1269,7 @@ static void buddy_toggle_virtual_slots(struct buddy *buddy, unsigned int state) /* toggle current pos */ if (state) { buddy_tree_mark(tree, pos); - } - else { + } else { buddy_tree_release(tree, pos); } break; @@ -1338,13 +1382,13 @@ static struct buddy_embed_check buddy_embed_offset(size_t memory_size, size_t al memset(&check_result, 0, sizeof(check_result)); check_result.can_fit = 1; - buddy_size = buddy_sizeof_alignment(memory_size, alignment); + buddy_size = buddy_sizeof_alignment_embedded(memory_size, alignment); if (buddy_size >= memory_size) { check_result.can_fit = 0; } offset = memory_size - buddy_size; - if (offset % BUDDY_ALIGNOF(struct buddy) != 0) { + if (offset % BUDDY_ALIGNOF(struct buddy)) { buddy_size += offset % BUDDY_ALIGNOF(struct buddy); if (buddy_size >= memory_size) { check_result.can_fit = 0; @@ -1394,12 +1438,14 @@ enum buddy_tree_flags { }; struct internal_position { - size_t local_offset; size_t bitset_location; + uint8_t local_offset; + uint8_t local_fillval; }; static inline size_t size_for_order(uint8_t order, uint8_t to); static inline size_t buddy_tree_index_internal(struct buddy_tree_pos pos); +static inline struct internal_position buddy_index_internal_sibling(struct internal_position pos); static struct buddy_tree_pos buddy_tree_leftmost_child_internal(size_t tree_order); static struct internal_position buddy_tree_internal_position_order( size_t tree_order, struct buddy_tree_pos pos); @@ -1415,16 +1461,36 @@ static inline size_t buddy_tree_size_for_order(struct buddy_tree *t, uint8_t to) static void write_to_internal_position(struct buddy_tree* t, struct internal_position pos, size_t value); static inline size_t read_from_internal_position(unsigned char *bitset, struct internal_position pos); static inline unsigned char compare_with_internal_position(unsigned char *bitset, struct internal_position pos, size_t value); +static inline size_t buddy_1_2_4_8_mul(size_t a, uint8_t b); #ifdef BUDDY_EXPERIMENTAL_CHANGE_TRACKING static inline void buddy_tree_track_change(struct buddy_tree* t, unsigned char* addr, size_t length); #endif /* BUDDY_EXPERIMENTAL_CHANGE_TRACKING */ +static inline size_t bits_for_order(uint8_t order) { + if (order >= 16) { + return 8; + } else if (order >= 4) { + return 4; + } else if (order >= 2) { + return 2; + } else { + return order; + } + // return order >= 8 ? 8 : order; +} + static inline size_t size_for_order(uint8_t order, uint8_t to) { size_t result = 0; size_t multi = 1u; while (order != to) { - result += order * multi; + size_t nbits = bits_for_order(order) * multi; + if (nbits % 16) { // 16 bit alignment of tree levels allows us to xor the offset to get the sibling + nbits &= ~(15); + nbits += 16; + } + + result += nbits; order--; multi *= 2; } @@ -1436,22 +1502,39 @@ static inline struct internal_position buddy_tree_internal_position_order( struct internal_position p; size_t total_offset, local_index; - p.local_offset = tree_order - buddy_tree_depth(pos) + 1; - total_offset = size_for_order((uint8_t) tree_order, (uint8_t) p.local_offset); + p.local_fillval = tree_order - buddy_tree_depth(pos) + 1; + p.local_offset = bits_for_order(p.local_fillval); + total_offset = size_for_order((uint8_t) tree_order, (uint8_t) p.local_fillval); local_index = buddy_tree_index_internal(pos); p.bitset_location = total_offset + (p.local_offset * local_index); return p; } +static inline size_t buddy_1_2_4_8_mul(size_t a, uint8_t b) { + // If we know that b can only be 1, 2, 4, 8 in the case of multiplying by bit offsets, + // we can attempt to bully the compiler into using shifts to implement the multiplication + switch (b) { + case 1: return a; + case 2: return a << 1; + case 4: return a << 2; + case 8: return a << 3; + default: { + assert(false); + return 0; + } + } +} + static inline struct internal_position buddy_tree_internal_position_tree( struct buddy_tree *t, struct buddy_tree_pos pos) { struct internal_position p; size_t total_offset, local_index; - p.local_offset = t->order - buddy_tree_depth(pos) + 1; - total_offset = buddy_tree_size_for_order(t, (uint8_t) p.local_offset); + p.local_fillval = t->order - buddy_tree_depth(pos) + 1; + p.local_offset = bits_for_order(p.local_fillval); + total_offset = buddy_tree_size_for_order(t, p.local_fillval); local_index = buddy_tree_index_internal(pos); - p.bitset_location = total_offset + (p.local_offset * local_index); + p.bitset_location = total_offset + buddy_1_2_4_8_mul(local_index, p.local_offset); return p; } @@ -1462,7 +1545,7 @@ static size_t buddy_tree_sizeof(uint8_t order) { /* Account for the bitset */ bitset_size = bitset_sizeof(size_for_order(order, 0)); if (bitset_size % sizeof(size_t)) { - bitset_size += (bitset_size % sizeof(size_t)); + bitset_size += sizeof(size_t) - (bitset_size % sizeof(size_t)); } /* Account for the size_for_order memoization */ size_for_order_size = ((order+2) * sizeof(size_t)); @@ -1649,7 +1732,7 @@ static inline size_t buddy_tree_index_internal(struct buddy_tree_pos pos) { /* Clear out the highest bit, this gives us the index * in a row of sibling nodes */ size_t mask = two_to_the_power_of(pos.depth - 1u); - size_t result = pos.index & ~mask; + size_t result = pos.index ^ mask; return result; } @@ -1676,11 +1759,40 @@ static inline size_t buddy_tree_size_for_order(struct buddy_tree *t, static void write_to_internal_position(struct buddy_tree* t, struct internal_position pos, size_t value) { unsigned char *bitset = buddy_tree_bits(t); - struct bitset_range clear_range = to_bitset_range(pos.bitset_location, pos.bitset_location + pos.local_offset - 1); + struct bitset_range clear_range = to_bitset_range(pos.bitset_location, pos.bitset_location + pos.local_offset - 1); + if (pos.local_offset == 8) { + assert(pos.local_offset == 8); + assert(!(pos.bitset_location % 8)); + bitset[pos.bitset_location / 8] = value; + } else if (pos.local_offset == 4) { + assert((value & 0b1111) == value); + assert(!(pos.bitset_location % 4)); + if (pos.bitset_location % 8) { + bitset[pos.bitset_location / 8] &= 0b00001111; + bitset[pos.bitset_location / 8] += (uint8_t)(value << 4); + } else { + bitset[pos.bitset_location / 8] &= 0b11110000; + bitset[pos.bitset_location / 8] += value; + } + } else if (pos.local_offset == 2) { + const uint8_t inv_masks[8] = {0b11111100, 0, 0b11110011, 0, 0b11001111, 0, 0b00111111, 0}; + + uint8_t invmask = inv_masks[pos.bitset_location % 8]; + bitset[pos.bitset_location / 8] &= invmask; + bitset[pos.bitset_location / 8] += value << (pos.bitset_location % 8); + + } else { + const uint8_t inv_masks[8] = {0b11111110, 0b11111101, 0b11111011, 0b11110111, 0b11101111, 0b11011111, 0b10111111, 0b01111111}; + bitset[pos.bitset_location / 8] &= inv_masks[pos.bitset_location % 8]; + + if (value) { + bitset[pos.bitset_location / 8] |= 1 << (pos.bitset_location % 8); + } - bitset_clear_range(bitset, clear_range); - if (value) { - bitset_set_range(bitset, to_bitset_range(pos.bitset_location, pos.bitset_location+value-1)); + // bitset_clear_range(bitset, clear_range); + // if (value) { + // bitset_set_range(bitset, to_bitset_range(pos.bitset_location, pos.bitset_location+value-1)); + // } } #ifdef BUDDY_EXPERIMENTAL_CHANGE_TRACKING @@ -1689,15 +1801,104 @@ static void write_to_internal_position(struct buddy_tree* t, struct internal_pos #endif } +static const uint8_t lut_extract1[8][256] = { + {0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1, +0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,}, + {0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1, +0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,}, + {0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1, +0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,}, + {0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1, +0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,}, + {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,}, + {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,}, + {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,}, + {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,}, +}; +static const uint8_t lut_extract2[8][256] = { + {0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3, +0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,}, + {0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3, +0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,0,0,1,1,2,2,3,3,}, + {0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3, +0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,}, + {0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3, +0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,}, + {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,}, + {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,}, + {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, +2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,}, + {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,}, +}; +static const uint8_t lut_extract4[8][256] = { + {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10,11,12 +,13,14,15,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10, +11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,}, + {0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,0,0,1,1,2,2,3,3,4,4,5,5,6,6, +7,7,8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13, +13,14,14,15,15,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,}, + {0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,6,6,6,6,7,7,7,7,8,8,8,8,9,9,9,9,10,10,10,10,11,11,11,11,12,12,12,12,13,13,13,13,14,14,14,14,15,15,15,15,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,6,6,6,6,7,7,7,7,8,8,8,8,9,9,9,9,10,10,10,10,11,11,11,11, +12,12,12,12,13,13,13,13,14,14,14,14,15,15,15,15,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,6,6,6,6,7,7,7,7,8,8,8,8,9,9,9,9,10,10,10,10,11,11,11,11,12,12,12,12,13,13,13,13,14,14,14,14,15,15,15,15,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,6,6,6,6,7,7,7 +,7,8,8,8,8,9,9,9,9,10,10,10,10,11,11,11,11,12,12,12,12,13,13,13,13,14,14,14,14,15,15,15,15,}, + {0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,11,11,11,11,11,11,11,11,12,12,12,12,12,12,12,12,13,13,13,13,13,13,13,13, +14,14,14,14,14,14,14,14,15,15,15,15,15,15,15,15,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,11,11,11,11,11,11,11,11,12,12 +,12,12,12,12,12,12,13,13,13,13,13,13,13,13,14,14,14,14,14,14,14,14,15,15,15,15,15,15,15,15,}, + {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, +8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,14,14 +,14,14,14,14,14,14,14,14,14,14,14,14,14,14,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,}, + {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, +4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,}, + {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, +2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,}, + {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,}, +}; + static inline size_t read_from_internal_position(unsigned char *bitset, struct internal_position pos) { - if (! bitset_test(bitset, pos.bitset_location)) { - return 0; /* Fast test without complete extraction */ + if (pos.local_offset == 8) { + assert(pos.local_offset == 8); + assert(!(pos.bitset_location % 8)); + + return bitset[pos.bitset_location / 8]; + } else if (pos.local_offset == 4) { + uint8_t val = bitset[pos.bitset_location / 8]; + + return lut_extract4[pos.bitset_location % 8][val]; + } else if (pos.local_offset == 2) { + uint8_t val = (bitset[pos.bitset_location / 8]); + return lut_extract2[pos.bitset_location % 8][val]; + } else { + uint8_t val = (bitset[pos.bitset_location / 8]); + return lut_extract1[pos.bitset_location % 8][val]; } - return bitset_count_range(bitset, to_bitset_range(pos.bitset_location, pos.bitset_location+pos.local_offset-1)); } static inline unsigned char compare_with_internal_position(unsigned char *bitset, struct internal_position pos, size_t value) { - return bitset_test(bitset, pos.bitset_location+value-1); + if (pos.local_offset == 8) { + assert(pos.local_offset == 8); + assert(!(pos.bitset_location % 8)); + + return bitset[pos.bitset_location / 8] >= value; + } else if (pos.local_offset == 4) { + uint8_t val = bitset[pos.bitset_location / 8]; + + if (pos.bitset_location % 8) { + return (val >> 4) >= (uint8_t)value; + } else { + return (val & 0b00001111) >= (uint8_t)value; + } + } else if (pos.local_offset == 2) { + return ((bitset[pos.bitset_location / 8] >> (pos.bitset_location % 8)) & 0b11) >= (uint8_t)value; + } else { + return bitset_test(bitset, pos.bitset_location+value-1); + } } static struct buddy_tree_interval to_buddy_tree_interval(struct buddy_tree *t, struct buddy_tree_pos pos) { @@ -1763,17 +1964,17 @@ static void buddy_tree_mark(struct buddy_tree *t, struct buddy_tree_pos pos) { struct internal_position internal = buddy_tree_internal_position_tree(t, pos); /* Mark the node as used */ - write_to_internal_position(t, internal, internal.local_offset); + write_to_internal_position(t, internal, internal.local_fillval); /* Update the tree upwards */ - update_parent_chain(t, pos, internal, internal.local_offset); + update_parent_chain(t, pos, internal, internal.local_fillval); } static enum buddy_tree_release_status buddy_tree_release(struct buddy_tree *t, struct buddy_tree_pos pos) { /* Calling release on an unused or a partially-used position a bug in caller */ struct internal_position internal = buddy_tree_internal_position_tree(t, pos); - if (read_from_internal_position(buddy_tree_bits(t), internal) != internal.local_offset) { + if (read_from_internal_position(buddy_tree_bits(t), internal) != internal.local_fillval) { return BUDDY_TREE_RELEASE_FAIL_PARTIALLY_USED; } @@ -1786,22 +1987,26 @@ static enum buddy_tree_release_status buddy_tree_release(struct buddy_tree *t, s return BUDDY_TREE_RELEASE_SUCCESS; } +static inline struct internal_position buddy_index_internal_sibling(struct internal_position pos) { + pos.bitset_location ^= pos.local_offset; + return pos; +} + static void update_parent_chain(struct buddy_tree *t, struct buddy_tree_pos pos, struct internal_position pos_internal, size_t size_current) { size_t size_sibling, size_parent, target_parent; unsigned char *bits = buddy_tree_bits(t); while (pos.index != 1) { - pos_internal.bitset_location += pos_internal.local_offset - - (2 * pos_internal.local_offset * (pos.index & 1u)); + pos_internal = buddy_index_internal_sibling(pos_internal); size_sibling = read_from_internal_position(bits, pos_internal); pos = buddy_tree_parent(pos); pos_internal = buddy_tree_internal_position_tree(t, pos); size_parent = read_from_internal_position(bits, pos_internal); - target_parent = (size_current || size_sibling) - * ((size_current <= size_sibling ? size_current : size_sibling) + 1); + target_parent = (size_current <= size_sibling ? size_current : size_sibling) + (size_current || size_sibling); + if (target_parent == size_parent) { return; } @@ -1814,7 +2019,7 @@ static void update_parent_chain(struct buddy_tree *t, struct buddy_tree_pos pos, static struct buddy_tree_pos buddy_tree_find_free(struct buddy_tree *t, uint8_t target_depth) { struct buddy_tree_pos current_pos, left_pos, right_pos; uint8_t target_status; - size_t current_depth, right_status; + size_t current_depth, left_status, right_status; struct internal_position left_internal, right_internal; unsigned char *tree_bits; @@ -1838,21 +2043,16 @@ static struct buddy_tree_pos buddy_tree_find_free(struct buddy_tree *t, uint8_t right_internal = left_internal; right_internal.bitset_location += right_internal.local_offset; /* advance to the right */ - if (compare_with_internal_position(tree_bits, left_internal, target_status+1)) { /* left branch is busy, pick right */ + left_status = read_from_internal_position(tree_bits, left_internal); + if (left_status >= (target_status + 1)) { /* left branch is busy, pick right */ current_pos = right_pos; - } else if (compare_with_internal_position(tree_bits, right_internal, target_status+1)) { /* right branch is busy, pick left */ + } else if ((right_status = read_from_internal_position(tree_bits, right_internal), right_status >= (target_status + 1))) { /* right branch is busy, pick left */ current_pos = left_pos; } else { - /* One of the child nodes must be read in order to compare it to its sibling. */ - right_status = read_from_internal_position(tree_bits, right_internal); - if (right_status) { - if (compare_with_internal_position(tree_bits, left_internal, right_status)) { - current_pos = left_pos; /* Left is equal or more busy than right, prefer left */ - } else { - current_pos = right_pos; - } - } else { /* Right is empty, prefer left */ - current_pos = left_pos; + if (left_status >= right_status) { + current_pos = left_pos; /* Left is equal or more busy than right, prefer left */ + } else { + current_pos = right_pos; } } } @@ -1868,7 +2068,7 @@ static bool buddy_tree_is_free(struct buddy_tree *t, struct buddy_tree_pos pos) struct internal_position internal = buddy_tree_internal_position_tree(t, pos); size_t value = read_from_internal_position(buddy_tree_bits(t), internal); if (value) { - return value != internal.local_offset; + return value != internal.local_fillval; } pos = buddy_tree_parent(pos); } @@ -1910,7 +2110,7 @@ static void buddy_tree_debug(struct buddy_tree *t, struct buddy_tree_pos pos, BUDDY_PRINTF("pos index: %zu pos depth: %zu status: %zu bitset-len: %zu bitset-at: %zu", state.current_pos.index, state.current_pos.depth, pos_status, pos_internal.local_offset, pos_internal.bitset_location); - if (pos_status == pos_internal.local_offset) { + if (pos_status == pos_internal.local_fillval) { BUDDY_PRINTF(" size: %zu", pos_size); } BUDDY_PRINTF("\n"); @@ -1935,7 +2135,7 @@ unsigned int buddy_tree_check_invariant(struct buddy_tree *t, struct buddy_tree_ violated = 1; } } else { - if ((current_status > 0) && (current_status < current_internal.local_offset)) { + if ((current_status > 0) && (current_status < current_internal.local_fillval)) { violated = 1; } } @@ -2100,9 +2300,9 @@ static size_t bitset_count_range(unsigned char *bitset, struct bitset_range rang result = popcount_byte(bitset[range.from_bucket] & bitset_char_mask[range.from_index][7]) + popcount_byte(bitset[range.to_bucket] & bitset_char_mask[0][range.to_index]); - while(++range.from_bucket != range.to_bucket) { - result += popcount_byte(bitset[range.from_bucket]); - } + // while(++range.from_bucket != range.to_bucket) { + // result += popcount_byte(bitset[range.from_bucket]); + // } return result; }