diff --git a/vortex-buffer/src/bit/buf.rs b/vortex-buffer/src/bit/buf.rs index cf69ce35449..aac8ead42fe 100644 --- a/vortex-buffer/src/bit/buf.rs +++ b/vortex-buffer/src/bit/buf.rs @@ -95,6 +95,7 @@ impl BitBuffer { /// Create a new `BoolBuffer` backed by a [`ByteBuffer`] with `len` bits in view. /// /// Panics if the buffer is not large enough to hold `len` bits. + #[inline] pub fn new(buffer: ByteBuffer, len: usize) -> Self { assert!( buffer.len() * 8 >= len, @@ -115,6 +116,7 @@ impl BitBuffer { /// the given `offset` (in bits). /// /// Panics if the buffer is not large enough to hold `len` bits after the offset. + #[inline] pub fn new_with_offset(buffer: ByteBuffer, len: usize, offset: usize) -> Self { assert!( len.saturating_add(offset) <= buffer.len().saturating_mul(8), @@ -142,6 +144,7 @@ impl BitBuffer { } /// Create a new `BoolBuffer` of length `len` where all bits are set (true). + #[inline] pub fn new_set(len: usize) -> Self { let words = len.div_ceil(8); let buffer = buffer![0xFF; words]; @@ -154,6 +157,7 @@ impl BitBuffer { } /// Create a new `BoolBuffer` of length `len` where all bits are unset (false). + #[inline] pub fn new_unset(len: usize) -> Self { let words = len.div_ceil(8); let buffer = Buffer::zeroed(words); @@ -171,11 +175,13 @@ impl BitBuffer { } /// Create a new empty `BitBuffer`. + #[inline] pub fn empty() -> Self { Self::new_set(0) } /// Create a new `BitBuffer` of length `len` where all bits are set to `value`. + #[inline] pub fn full(value: bool, len: usize) -> Self { if value { Self::new_set(len) @@ -271,6 +277,7 @@ impl BitBuffer { } /// Clear all bits in the buffer, preserving existing capacity. + #[inline] pub fn clear(&mut self) { self.buffer.clear(); self.len = 0; @@ -345,6 +352,7 @@ impl BitBuffer { /// for `len` bits. /// /// Panics if the slice would extend beyond the end of the buffer. + #[inline] pub fn slice(&self, range: impl RangeBounds) -> Self { let (byte_offset, meta) = BitBufferMeta::new(self.offset, self.len).slice(range); @@ -376,6 +384,7 @@ impl BitBuffer { } /// Access chunks of the buffer aligned to 8 byte boundary as [prefix, \, suffix] + #[inline] pub fn unaligned_chunks(&self) -> UnalignedBitChunk<'_> { UnalignedBitChunk::new(self.buffer.as_slice(), self.offset, self.len) } @@ -383,6 +392,7 @@ impl BitBuffer { /// Access chunks of the underlying buffer as 8 byte chunks with a final trailer /// /// If you're performing operations on a single buffer, prefer [BitBuffer::unaligned_chunks] + #[inline] pub fn chunks(&self) -> BitChunks<'_> { BitChunks::new(self.buffer.as_slice(), self.offset, self.len) } @@ -413,6 +423,7 @@ impl BitBuffer { /// which logical bit position holds that rank. /// /// Returns `None` if `nth` is greater than or equal to the number of set bits. + #[inline] pub fn select(&self, nth: usize) -> Option { bit_select(self.buffer.as_slice(), self.offset, self.len, nth) } @@ -424,16 +435,19 @@ impl BitBuffer { } /// Iterator over bits in the buffer + #[inline] pub fn iter(&self) -> BitIterator<'_> { BitIterator::new(self.buffer.as_slice(), self.offset, self.len) } /// Iterator over set indices of the underlying buffer + #[inline] pub fn set_indices(&self) -> BitIndexIterator<'_> { BitIndexIterator::new(self.buffer.as_slice(), self.offset, self.len) } /// Iterator over set slices of the underlying buffer + #[inline] pub fn set_slices(&self) -> BitSliceIterator<'_> { BitSliceIterator::new(self.buffer.as_slice(), self.offset, self.len) } @@ -484,11 +498,13 @@ impl BitBuffer { impl BitBuffer { /// Returns the offset, len and underlying buffer. + #[inline] pub fn into_inner(self) -> (usize, usize, ByteBuffer) { (self.offset, self.len, self.buffer) } /// Attempt to convert this `BitBuffer` into a mutable version. + #[inline] pub fn try_into_mut(self) -> Result { match self.buffer.try_into_mut() { Ok(buffer) => Ok(BitBufferMut::from_buffer(buffer, self.offset, self.len)), @@ -510,6 +526,7 @@ impl From> for BitBuffer { } impl FromIterator for BitBuffer { + #[inline] fn from_iter>(iter: T) -> Self { BitBufferMut::from_iter(iter).freeze() } diff --git a/vortex-buffer/src/bit/buf_mut.rs b/vortex-buffer/src/bit/buf_mut.rs index b3e7207a640..a1bb0c82c04 100644 --- a/vortex-buffer/src/bit/buf_mut.rs +++ b/vortex-buffer/src/bit/buf_mut.rs @@ -102,6 +102,7 @@ pub struct BitBufferMut { impl BitBufferMut { /// Create new bit buffer from given byte buffer and logical bit length + #[inline] pub fn from_buffer(buffer: ByteBufferMut, offset: usize, len: usize) -> Self { assert!( len <= buffer.len() * 8, @@ -125,6 +126,7 @@ impl BitBufferMut { } /// Create a new empty mutable bit buffer with requested capacity (in bits). + #[inline] pub fn with_capacity(capacity: usize) -> Self { Self { buffer: BufferMut::with_capacity(capacity.div_ceil(8)), @@ -134,6 +136,7 @@ impl BitBufferMut { } /// Create a new mutable buffer with requested `len` and all bits set to `true`. + #[inline] pub fn new_set(len: usize) -> Self { Self { buffer: buffer_mut![0xFF; len.div_ceil(8)], @@ -143,6 +146,7 @@ impl BitBufferMut { } /// Create a new mutable buffer with requested `len` and all bits set to `false`. + #[inline] pub fn new_unset(len: usize) -> Self { Self { buffer: BufferMut::zeroed(len.div_ceil(8)), @@ -158,6 +162,7 @@ impl BitBufferMut { } /// Create a new mutable buffer with requested `len` and all bits set to `value`. + #[inline] pub fn full(value: bool, len: usize) -> Self { if value { Self::new_set(len) @@ -247,11 +252,13 @@ impl BitBufferMut { } /// Return the underlying byte buffer. + #[inline] pub fn inner(&self) -> &ByteBufferMut { &self.buffer } /// Consumes the buffer and return the underlying byte buffer. + #[inline] pub fn into_inner(self) -> ByteBufferMut { self.buffer } @@ -299,6 +306,7 @@ impl BitBufferMut { } /// Reserve additional bit capacity for the buffer. + #[inline] pub fn reserve(&mut self, additional: usize) { let required_bits = self.offset + self.len + additional; let required_bytes = required_bits.div_ceil(8); // Rounds up. @@ -308,6 +316,7 @@ impl BitBufferMut { } /// Clears the bit buffer (but keeps any allocated memory). + #[inline] pub fn clear(&mut self) { // Also clear the byte buffer (not just `len`) so the "bits beyond len are zero" // invariant holds; `append_false` and `append_buffer` rely on it. @@ -415,6 +424,7 @@ impl BitBufferMut { /// Truncate the buffer to the given length. /// /// If the given length is greater than the current length, this is a no-op. + #[inline] pub fn truncate(&mut self, len: usize) { if len > self.len { return; @@ -609,16 +619,19 @@ impl BitBufferMut { } /// Freeze the buffer in its current state into an immutable `BoolBuffer`. + #[inline] pub fn freeze(self) -> BitBuffer { BitBuffer::new_with_offset(self.buffer.freeze(), self.len, self.offset) } /// Get the underlying bytes as a slice + #[inline] pub fn as_slice(&self) -> &[u8] { self.buffer.as_slice() } /// Get the underlying bytes as a mutable slice + #[inline] pub fn as_mut_slice(&mut self) -> &mut [u8] { self.buffer.as_mut_slice() } diff --git a/vortex-buffer/src/bit/meta.rs b/vortex-buffer/src/bit/meta.rs index 75659f3397a..621783b8e4a 100644 --- a/vortex-buffer/src/bit/meta.rs +++ b/vortex-buffer/src/bit/meta.rs @@ -19,6 +19,7 @@ impl BitBufferMeta { /// /// Panics if `offset >= 8`. Use [`from_raw_offset`](Self::from_raw_offset) to normalize a /// larger offset. + #[inline] pub fn new(offset: usize, len: usize) -> Self { assert!(offset < 8, "BitBufferMeta offset must be < 8, got {offset}"); Self { offset, len } @@ -29,6 +30,7 @@ impl BitBufferMeta { /// /// Returns `(byte_offset, meta)` so the caller can slice its backing buffer by `byte_offset` /// and store the remaining sub-byte offset in `meta`. + #[inline] pub fn from_raw_offset(offset: usize, len: usize) -> (usize, Self) { ( offset / 8, @@ -44,6 +46,7 @@ impl BitBufferMeta { /// # Panics /// /// Panics if the range is out of bounds or its end precedes its start. + #[inline] pub fn slice(&self, range: impl RangeBounds) -> (usize, Self) { let start = match range.start_bound() { Bound::Included(&start) => start, diff --git a/vortex-buffer/src/bit/view.rs b/vortex-buffer/src/bit/view.rs index 722119ee4ab..f4d46c93da7 100644 --- a/vortex-buffer/src/bit/view.rs +++ b/vortex-buffer/src/bit/view.rs @@ -63,6 +63,7 @@ impl<'a> BitBufferView<'a> { /// Create a new view over `buffer` with `len` bits, starting at bit zero. /// /// Panics if the buffer is not large enough to hold `len` bits. + #[inline] pub fn new(buffer: &'a [u8], len: usize) -> Self { Self::new_with_offset(buffer, len, 0) } @@ -70,6 +71,7 @@ impl<'a> BitBufferView<'a> { /// Create a new view over `buffer` with `len` bits, starting at the given bit `offset`. /// /// Panics if the buffer is not large enough to hold `len` bits after the offset. + #[inline] pub fn new_with_offset(buffer: &'a [u8], len: usize, offset: usize) -> Self { assert!( len.saturating_add(offset) <= buffer.len().saturating_mul(8), @@ -86,11 +88,13 @@ impl<'a> BitBufferView<'a> { } /// Create a new view over `buffer` described by `meta`. + #[inline] pub fn from_meta(buffer: &'a [u8], meta: BitBufferMeta) -> Self { Self::new_with_offset(buffer, meta.len(), meta.offset()) } /// Returns the [`BitBufferMeta`] (offset and length) describing this view. + #[inline] pub fn meta(&self) -> BitBufferMeta { BitBufferMeta::new(self.offset, self.len) } @@ -142,6 +146,7 @@ impl<'a> BitBufferView<'a> { /// Create a new view over the range `[start, end)` of this view. /// /// Panics if the slice would extend beyond the end of the view. + #[inline] pub fn slice(&self, range: impl RangeBounds) -> BitBufferView<'a> { let (start, end) = resolve_range(range, self.len); BitBufferView::new_with_offset(self.buffer, end - start, self.offset + start) @@ -149,41 +154,49 @@ impl<'a> BitBufferView<'a> { /// Access chunks of the buffer aligned to an 8 byte boundary as /// `[prefix, , suffix]`. + #[inline] pub fn unaligned_chunks(&self) -> UnalignedBitChunk<'a> { UnalignedBitChunk::new(self.buffer, self.offset, self.len) } /// Access chunks of the underlying buffer as 8 byte chunks with a final trailer. + #[inline] pub fn chunks(&self) -> BitChunks<'a> { BitChunks::new(self.buffer, self.offset, self.len) } /// Get the number of set bits in the view. + #[inline] pub fn true_count(&self) -> usize { count_ones(self.buffer, self.offset, self.len) } /// Get the number of unset bits in the view. + #[inline] pub fn false_count(&self) -> usize { self.len - self.true_count() } /// Returns the position of the `nth` set bit (0-indexed), or `None` if out of range. + #[inline] pub fn select(&self, nth: usize) -> Option { bit_select(self.buffer, self.offset, self.len, nth) } /// Iterator over bits in the view. + #[inline] pub fn iter(&self) -> BitIterator<'a> { BitIterator::new(self.buffer, self.offset, self.len) } /// Iterator over set indices of the underlying buffer. + #[inline] pub fn set_indices(&self) -> BitIndexIterator<'a> { BitIndexIterator::new(self.buffer, self.offset, self.len) } /// Iterator over set slices of the underlying buffer. + #[inline] pub fn set_slices(&self) -> BitSliceIterator<'a> { BitSliceIterator::new(self.buffer, self.offset, self.len) } @@ -240,6 +253,7 @@ impl<'a> BitBufferMutView<'a> { /// Create a new mutable view over `buffer` with `len` bits, starting at bit zero. /// /// Panics if the buffer is not large enough to hold `len` bits. + #[inline] pub fn new(buffer: &'a mut [u8], len: usize) -> Self { Self::new_with_offset(buffer, len, 0) } @@ -247,6 +261,7 @@ impl<'a> BitBufferMutView<'a> { /// Create a new mutable view over `buffer` with `len` bits, starting at bit `offset`. /// /// Panics if the buffer is not large enough to hold `len` bits after the offset. + #[inline] pub fn new_with_offset(buffer: &'a mut [u8], len: usize, offset: usize) -> Self { assert!( len.saturating_add(offset) <= buffer.len().saturating_mul(8), @@ -324,16 +339,19 @@ impl<'a> BitBufferMutView<'a> { } /// Get the number of set bits in the view. + #[inline] pub fn true_count(&self) -> usize { self.as_view().true_count() } /// Get the number of unset bits in the view. + #[inline] pub fn false_count(&self) -> usize { self.as_view().false_count() } /// Iterator over bits in the view. + #[inline] pub fn iter(&self) -> BitIterator<'_> { self.as_view().iter() } @@ -341,6 +359,7 @@ impl<'a> BitBufferMutView<'a> { /// Set the bit at `index` to the given boolean value. /// /// Panics if `index` exceeds the view length. + #[inline] pub fn set_to(&mut self, index: usize, value: bool) { if value { self.set(index); @@ -354,6 +373,7 @@ impl<'a> BitBufferMutView<'a> { /// # Safety /// /// Caller must ensure that `index` is within the range of the view. + #[inline] pub unsafe fn set_to_unchecked(&mut self, index: usize, value: bool) { if value { // SAFETY: checked by caller @@ -367,6 +387,7 @@ impl<'a> BitBufferMutView<'a> { /// Set the bit at `index` to `true`. /// /// Panics if `index` exceeds the view length. + #[inline] pub fn set(&mut self, index: usize) { assert!(index < self.len, "index {index} exceeds len {}", self.len); // SAFETY: checked by assertion @@ -376,6 +397,7 @@ impl<'a> BitBufferMutView<'a> { /// Set the bit at `index` to `false`. /// /// Panics if `index` exceeds the view length. + #[inline] pub fn unset(&mut self, index: usize) { assert!(index < self.len, "index {index} exceeds len {}", self.len); // SAFETY: checked by assertion