From 2d0a705ec420a85a4172b8de7caff1cef3fe2448 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Tue, 8 Sep 2026 00:11:57 +0000 Subject: [PATCH] Delta validity is a top level child For performant delta encoding we have to read all the values in deltas buffer that means that undefined values in deltas buffer are significant. Instead of performing careful transformation dance to ensure correctness we remove null values from deltas and keep them as a top level child. Cascading encodings don't need any special handling and will preserve all values since they're all valid. Signed-off-by: Robert Kruszewski --- encodings/fastlanes/src/bit_transpose.rs | 223 ----------- .../src/delta/array/delta_compress.rs | 139 ++----- .../src/delta/array/delta_decompress.rs | 4 +- encodings/fastlanes/src/delta/array/mod.rs | 10 +- encodings/fastlanes/src/delta/compute/cast.rs | 28 +- encodings/fastlanes/src/delta/mod.rs | 3 + encodings/fastlanes/src/delta/tests.rs | 259 ++++++++++++ encodings/fastlanes/src/delta/vtable/mod.rs | 81 +++- encodings/fastlanes/src/delta/vtable/rules.rs | 4 +- encodings/fastlanes/src/delta/vtable/slice.rs | 9 +- .../fastlanes/src/delta/vtable/validity.rs | 72 ++-- encodings/fastlanes/src/lib.rs | 102 ++--- .../fastlanes/src/rle/array/rle_compress.rs | 8 +- encodings/fastlanes/src/transposed_bool.rs | 368 ------------------ vortex-btrblocks/src/schemes/integer/delta.rs | 9 +- vortex-btrblocks/tests/golden.rs | 25 ++ ...compact__int_monotone_jitter_nullable.snap | 9 + ...regular__int_monotone_jitter_nullable.snap | 11 + vortex/benches/single_encoding_throughput.rs | 12 +- 19 files changed, 528 insertions(+), 848 deletions(-) delete mode 100644 encodings/fastlanes/src/bit_transpose.rs create mode 100644 encodings/fastlanes/src/delta/tests.rs delete mode 100644 encodings/fastlanes/src/transposed_bool.rs create mode 100644 vortex-btrblocks/tests/snapshots/golden__compact__int_monotone_jitter_nullable.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__regular__int_monotone_jitter_nullable.snap diff --git a/encodings/fastlanes/src/bit_transpose.rs b/encodings/fastlanes/src/bit_transpose.rs deleted file mode 100644 index b9761ee20cf..00000000000 --- a/encodings/fastlanes/src/bit_transpose.rs +++ /dev/null @@ -1,223 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::mem; -use std::mem::MaybeUninit; - -use vortex_buffer::Alignment; -use vortex_buffer::BitBuffer; -use vortex_buffer::BufferMut; -use vortex_buffer::ByteBuffer; -use vortex_error::VortexExpect; - -use crate::FL_CHUNK_SIZE; - -/// Transposes `bits` into FastLanes order, padding the result to whole 1,024-bit chunks. -/// -/// Bit `i` of the result holds logical bit [`fastlanes::transpose(i)`](fastlanes::transpose), the -/// same element order [`fastlanes::Transpose::transpose`] gives a value vector. To address the -/// result by logical index instead, invert that with `crate::untranspose_idx`. -pub fn transpose_bitbuffer(bits: BitBuffer) -> BitBuffer { - bits_op(bits, fastlanes::transpose_bits::) -} - -/// Untransposes whole 1,024-bit FastLanes chunks back into sequential bit order. -pub fn untranspose_bitbuffer(bits: BitBuffer) -> BitBuffer { - assert!( - bits.len().is_multiple_of(FL_CHUNK_SIZE), - "Transposed BitBuffer length must be a multiple of {FL_CHUNK_SIZE}" - ); - bits_op(bits, fastlanes::untranspose_bits) -} - -fn bits_op(bits: BitBuffer, op: F) -> BitBuffer { - // Normalize to offset 0 with no bytes outside the logical range, so that byte-buffer chunk - // boundaries line up with logical 1,024-bit chunks regardless of how the buffer was sliced. - let bits = bits.sliced(); - let (_offset, len, bytes) = bits.into_inner(); - - if len.is_multiple_of(FL_CHUNK_SIZE) && bytes.is_aligned(Alignment::of::()) { - match bytes.try_into_mut() { - Ok(mut bytes_mut) => { - let (chunks, _) = bytes_mut.as_chunks_mut::<128>(); - let mut tmp = [0u64; 16]; - for chunk in chunks { - // SAFETY: the buffer is u64-aligned and chunks start at 128-byte multiples. - let chunk_u64 = - unsafe { mem::transmute::<&mut [u8; 128], &mut [u64; 16]>(chunk) }; - op(chunk_u64, &mut tmp); - chunk_u64.copy_from_slice(&tmp); - } - BitBuffer::new(bytes_mut.freeze().into_byte_buffer(), len) - } - Err(bytes) => bits_op_with_copy(bytes, len, op), - } - } else { - bits_op_with_copy(bytes, len, op) - } -} - -fn bits_op_with_copy( - bytes: ByteBuffer, - len: usize, - op: F, -) -> BitBuffer { - let output_len = bytes.len().div_ceil(8).next_multiple_of(16); - let mut output = BufferMut::::with_capacity(output_len); - let (input_chunks, input_trailer) = bytes.as_chunks::<128>(); - // Bound to the requested `output_len`: `spare_capacity_mut` may expose extra over-aligned - // capacity, which would otherwise split into spurious trailing chunks and make `last_mut` - // below target a chunk past the data we actually initialize. - let (output_chunks, _) = unsafe { - mem::transmute::<&mut [MaybeUninit], &mut [u64]>( - &mut output.spare_capacity_mut()[..output_len], - ) - } - .as_chunks_mut::<16>(); - - for (input, output) in input_chunks.iter().zip(output_chunks.iter_mut()) { - op(&load_chunk_unaligned(input), output); - } - - if !input_trailer.is_empty() { - let mut padded_input = [0u8; 128]; - padded_input[0..input_trailer.len()].clone_from_slice(input_trailer); - op( - &load_chunk_unaligned(&padded_input), - output_chunks - .last_mut() - .vortex_expect("Output wasn't a multiple of 128 bytes"), - ); - } - - unsafe { output.set_len(output_len) }; - BitBuffer::new( - output.freeze().into_byte_buffer(), - len.next_multiple_of(FL_CHUNK_SIZE), - ) -} - -/// Loads a 128-byte chunk into a `[u64; 16]` without requiring the input to be 8-byte aligned. -/// -/// Native endianness is required: this must produce the same words as the aligned in-place path, -/// which reinterprets the buffer as host-endian `u64`s. -#[allow(clippy::host_endian_bytes)] -fn load_chunk_unaligned(chunk: &[u8; 128]) -> [u64; 16] { - let mut words = [0u64; 16]; - let (bytes, _) = chunk.as_chunks::<8>(); - for (word, bytes) in words.iter_mut().zip(bytes) { - *word = u64::from_ne_bytes(*bytes); - } - words -} - -#[cfg(test)] -mod tests { - use vortex_buffer::BitBuffer; - use vortex_buffer::BitBufferMut; - use vortex_buffer::ByteBuffer; - use vortex_buffer::ByteBufferMut; - - use super::*; - - fn make_validity_bits(num_bits: usize) -> BitBuffer { - let mut builder = BitBufferMut::with_capacity(num_bits); - for i in 0..num_bits { - builder.append(i % 3 != 0); - } - builder.freeze() - } - - fn force_copy_path(bits: BitBuffer) -> (BitBuffer, ByteBuffer) { - let (offset, len, bytes) = bits.into_inner(); - let extra_ref = bytes.clone(); - (BitBuffer::new_with_offset(bytes, len, offset), extra_ref) - } - - #[test] - fn transpose_padding_copy_produces_same_bits() { - let bits = make_validity_bits(500); - let transposed = transpose_bitbuffer(bits.clone()); - assert_eq!(transposed.len(), 1024); - let untransposed = untranspose_bitbuffer(transposed); - assert_eq!(untransposed.slice(0..500), bits) - } - - #[test] - fn transpose_inplace_and_copy_produce_same_bits() { - let bits = make_validity_bits(2048); - - let inplace_result = transpose_bitbuffer(bits.clone()); - - let (bits_shared, _hold) = force_copy_path(bits); - let copy_result = transpose_bitbuffer(bits_shared); - - assert_eq!(inplace_result.len(), copy_result.len()); - assert_eq!(inplace_result, copy_result); - } - - #[test] - fn transpose_bitbuffer_roundtrip_non_aligned() { - let original_len = 1500; - let bits = make_validity_bits(original_len); - - let transposed = transpose_bitbuffer(bits.clone()); - let roundtripped = untranspose_bitbuffer(transposed); - assert_eq!(bits, roundtripped.slice(0..original_len)); - } - - /// Regression: the copy path split the over-aligned spare capacity into extra 128-byte - /// chunks and wrote the padded remainder via `last_mut()`, which landed past the requested - /// length and left the real trailing chunk uninitialized. Whether the surplus capacity - /// produced an extra chunk depended on the allocation address, so we repeat across sizes to - /// defeat that luck; the fix bounds the spare slice so the result no longer depends on it. - #[test] - fn transpose_copy_path_survives_overallocation() { - for original_len in [129, 500, 1500, 9999] { - let bits = make_validity_bits(original_len); - for _ in 0..64 { - let transposed = transpose_bitbuffer(bits.clone()); - let roundtripped = untranspose_bitbuffer(transposed); - assert_eq!( - roundtripped.slice(0..original_len), - bits, - "len={original_len}" - ); - } - } - } - - /// Regression: buffers that are not 8-byte aligned (e.g. views into file segments) must not - /// panic and must produce the same result as the aligned path. - #[test] - fn untranspose_unaligned_buffer() { - let bits = make_validity_bits(2048); - let transposed = transpose_bitbuffer(bits.clone()); - let expected = untranspose_bitbuffer(transposed.clone()); - - // Rebuild the transposed bytes at an odd offset within a larger allocation so the - // resulting buffer is misaligned for u64. - let (_, _, transposed_bytes) = transposed.sliced().into_inner(); - let mut shifted = ByteBufferMut::with_capacity(transposed_bytes.len() + 1); - shifted.push(0xFF); - shifted.extend_from_slice(&transposed_bytes); - let misaligned = shifted.freeze().slice(1..transposed_bytes.len() + 1); - assert!(!misaligned.is_aligned(Alignment::of::())); - - let untransposed = untranspose_bitbuffer(BitBuffer::new(misaligned, 2048)); - assert_eq!(untransposed, expected); - assert_eq!(untransposed.slice(0..2048), bits); - } - - /// Regression: a bit-offset view of transposed chunks (a sliced validity) must untranspose - /// only the viewed chunks rather than assuming the buffer starts at a chunk boundary. - #[test] - fn untranspose_chunk_aligned_bit_offset() { - let bits = make_validity_bits(3 * FL_CHUNK_SIZE); - let transposed = transpose_bitbuffer(bits.clone()); - - let view = transposed.slice(FL_CHUNK_SIZE..3 * FL_CHUNK_SIZE); - let untransposed = untranspose_bitbuffer(view); - assert_eq!(untransposed, bits.slice(FL_CHUNK_SIZE..3 * FL_CHUNK_SIZE)); - } -} diff --git a/encodings/fastlanes/src/delta/array/delta_compress.rs b/encodings/fastlanes/src/delta/array/delta_compress.rs index c2ef38ceb79..4a6f4b184f8 100644 --- a/encodings/fastlanes/src/delta/array/delta_compress.rs +++ b/encodings/fastlanes/src/delta/array/delta_compress.rs @@ -8,22 +8,20 @@ use fastlanes::Delta; use fastlanes::FastLanes; use fastlanes::Transpose; use vortex_array::ExecutionCtx; -use vortex_array::IntoArray; -use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::dtype::NativePType; use vortex_array::match_each_unsigned_integer_ptype; use vortex_array::validity::Validity; -use vortex_buffer::BitBufferMut; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; use vortex_error::VortexResult; +use crate::ChunkBoundary; use crate::FL_CHUNK_SIZE; -use crate::bit_transpose::transpose_bitbuffer; use crate::fill_forward_nulls; +/// Encode nonnullable bases and deltas. The caller retains the source validity. pub fn delta_compress( array: &PrimitiveArray, ctx: &mut ExecutionCtx, @@ -32,45 +30,15 @@ pub fn delta_compress( let original_ptype = array.ptype(); let array = array.reinterpret_cast(original_ptype.to_unsigned()); - let (bases, deltas) = match_each_unsigned_integer_ptype!(array.ptype(), |T| { - // Fill-forward null values so that transposed deltas at null positions remain - // small. Without this, bitpacking may skip patches for null positions, and the - // corrupted delta values propagate through the cumulative sum during decompression. - let filled = fill_forward_nulls(array.to_buffer::(), &validity, ctx)?; + Ok(match_each_unsigned_integer_ptype!(array.ptype(), |T| { + let filled = + fill_forward_nulls(array.to_buffer::(), &validity, ChunkBoundary::Carry, ctx)?; let (bases, deltas) = compress_primitive::(&filled); - let validity = match validity { - Validity::Array(mask) => { - let bits = mask.execute::(ctx)?.into_bit_buffer(); - let pad = bits.len().next_multiple_of(FL_CHUNK_SIZE) - bits.len(); - // Pad remainder bits as valid to match last-value remainder padding. - // `transpose_bitbuffer` uses the same element order as the value transpose, so - // the pad bits land on exactly the padded value slots; zero-filling them would - // mark those slots null and bitpacking would then skip their patches. - let bits = if pad == 0 { - bits - } else { - // `sliced` first so the copy covers only the logical range, not whatever - // wider buffer the mask was sliced out of. - let mut padded = BitBufferMut::copy_from(&bits.sliced()); - padded.append_n(true, pad); - padded.freeze() - }; - Validity::Array( - BoolArray::new(transpose_bitbuffer(bits), Validity::NonNullable).into_array(), - ) - } - validity => validity, - }; ( - PrimitiveArray::new(bases, array.dtype().nullability().into()), - PrimitiveArray::new(deltas, validity), + PrimitiveArray::new(bases, Validity::NonNullable).reinterpret_cast(original_ptype), + PrimitiveArray::new(deltas, Validity::NonNullable).reinterpret_cast(original_ptype), ) - }); - - Ok(( - bases.reinterpret_cast(original_ptype), - deltas.reinterpret_cast(original_ptype), - )) + })) } fn compress_primitive(array: &[T]) -> (Buffer, Buffer) @@ -86,7 +54,8 @@ where // Allocate result arrays. let mut bases = BufferMut::with_capacity(bases_len); let mut deltas = BufferMut::with_capacity(padded_len); - let (output_deltas, _) = deltas.spare_capacity_mut().as_chunks_mut::(); + let (output_deltas, _) = + deltas.spare_capacity_mut()[..padded_len].as_chunks_mut::(); // Loop over all full 1024-element chunks. let mut transposed: [T; FL_CHUNK_SIZE] = [T::default(); FL_CHUNK_SIZE]; @@ -133,19 +102,15 @@ mod tests { use rstest::rstest; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; - use vortex_array::arrays::Bool; - use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::validity::Validity; - use vortex_error::VortexExpect; use vortex_error::VortexResult; - use vortex_error::vortex_bail; use vortex_session::VortexSession; use crate::Delta; + use crate::DeltaArraySlotsExt; use crate::FL_CHUNK_SIZE; - use crate::bit_transpose::untranspose_bitbuffer; use crate::bitpack_compress::bitpack_encode; use crate::delta::array::delta_decompress::delta_decompress; use crate::delta_compress; @@ -185,6 +150,7 @@ mod tests { fn test_compress(#[case] array: PrimitiveArray) -> VortexResult<()> { let delta = Delta::try_from_primitive_array(&array, &mut SESSION.create_execution_ctx())?; assert_eq!(delta.len(), array.len()); + assert!(!delta.deltas().dtype().is_nullable()); let decompressed = delta_decompress(&delta, &mut SESSION.create_execution_ctx())?; assert_arrays_eq!(decompressed, array, &mut SESSION.create_execution_ctx()); Ok(()) @@ -213,21 +179,14 @@ mod tests { Ok(()) } - /// Padding remainder validity with `true` must not change logical nulls, including leading - /// and trailing nulls in the unaligned tail. After untranspose, pad bits are valid and are - /// sliced off by `logical_len`. - /// - /// Which transposed slots the pad bits land on varies with the remainder length, so cover - /// several, plus an aligned length that pads nothing at all. + // Validity stays logical even when the numeric children are padded. #[rstest] #[case::one_row_remainder(1025)] #[case::mid_chunk_remainder(1500)] #[case::two_chunks_plus_one(2049)] #[case::one_row_short_of_aligned(3071)] #[case::already_aligned(2048)] - fn remainder_validity_pad_does_not_clobber_logical_nulls( - #[case] len: usize, - ) -> VortexResult<()> { + fn remainder_preserves_logical_validity(#[case] len: usize) -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); // Nulls at both ends, so a leading null and a null inside the padded tail are covered. let array = PrimitiveArray::from_option_iter( @@ -241,24 +200,14 @@ mod tests { let padded_len = len.next_multiple_of(FL_CHUNK_SIZE); assert_eq!(deltas.len(), padded_len); - let Validity::Array(storage) = deltas.validity()? else { - vortex_bail!("expected array-backed storage validity") - }; - let sequential = - untranspose_bitbuffer(storage.execute::(&mut ctx)?.into_bit_buffer()); - assert_eq!(sequential.len(), padded_len); - for i in 0..len { - assert_eq!( - sequential.value(i), - array.is_valid(i, &mut ctx)?, - "logical validity changed at {i}" - ); - } - for i in len..padded_len { - assert!(sequential.value(i), "pad bit {i} should be valid"); - } - - let delta = Delta::try_new(bases.into_array(), deltas.into_array(), 0, len)?; + assert!(matches!(deltas.validity()?, Validity::NonNullable)); + let delta = Delta::try_new( + bases.into_array(), + deltas.into_array(), + array.validity()?, + 0, + len, + )?; assert_eq!(delta.len(), len); assert!(!delta.is_valid(0, &mut ctx)?); assert!(!delta.is_valid(len - 1, &mut ctx)?); @@ -266,41 +215,6 @@ mod tests { Ok(()) } - /// The transposed validity must line up slot for slot with the transposed delta values: - /// storage slot `j` describes logical row `transpose(j)`, the position `Transpose::transpose` - /// put that row's delta in. `fastlanes` unified the element order of the bit and value - /// transposes; before that these two permutations disagreed. - #[test] - fn storage_validity_aligns_with_transposed_value_slots() -> VortexResult<()> { - let mut ctx = SESSION.create_execution_ctx(); - // Chunk-aligned, so no remainder padding takes part. - const LEN: usize = 2 * FL_CHUNK_SIZE; - let valid = |i: usize| !i.is_multiple_of(3); - let array = - PrimitiveArray::from_option_iter((0..LEN).map(|i| valid(i).then_some(i as u32))); - - let (_bases, deltas) = delta_compress(&array, &mut ctx)?; - let Validity::Array(storage) = deltas.validity()? else { - vortex_bail!("expected array-backed storage validity") - }; - let bits = storage.execute::(&mut ctx)?.into_bit_buffer(); - - for chunk in 0..LEN / FL_CHUNK_SIZE { - let base = chunk * FL_CHUNK_SIZE; - for slot in 0..FL_CHUNK_SIZE { - assert_eq!( - bits.value(base + slot), - valid(base + fastlanes::transpose(slot)), - "chunk={chunk} slot={slot}" - ); - } - } - Ok(()) - } - - /// Regression test: delta + bitpacked encoding must correctly round-trip nullable arrays - /// where null positions contain arbitrary values. Without fill-forward, the delta cumulative - /// sum propagates corrupted values from null positions. #[test] fn delta_bitpacked_trailing_nulls() -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); @@ -308,19 +222,14 @@ mod tests { (0u8..200).map(|i| (!(50..100).contains(&i)).then_some(i)), ); let (bases, deltas) = delta_compress(&array, &mut ctx)?; - let Validity::Array(storage_validity) = deltas.validity()? else { - vortex_bail!("test input should have array-backed validity") - }; - assert!(storage_validity.is::()); - let bitpacked_deltas = bitpack_encode(&deltas, 1, None, &mut ctx)?; let packed_delta = Delta::try_new( bases.into_array(), bitpacked_deltas.into_array(), + array.validity()?, 0, array.len(), - ) - .vortex_expect("Delta array construction should succeed"); + )?; let packed_delta_prim = packed_delta .as_array() .clone() diff --git a/encodings/fastlanes/src/delta/array/delta_decompress.rs b/encodings/fastlanes/src/delta/array/delta_decompress.rs index 3ccfede27c1..4bddbff66d1 100644 --- a/encodings/fastlanes/src/delta/array/delta_decompress.rs +++ b/encodings/fastlanes/src/delta/array/delta_decompress.rs @@ -63,9 +63,7 @@ where remainder.is_empty(), "deltas must be padded to a multiple of 1024" ); - // Use >= because cross-type casts (e.g. u32→u64) may produce more bases than the - // target LANES requires. Only the first chunks.len() * LANES bases are used. - assert!(bases.len() >= chunks.len() * LANES); + assert_eq!(bases.len(), chunks.len() * LANES); // Allocate a result array. let mut output = BufferMut::with_capacity(deltas.len()); diff --git a/encodings/fastlanes/src/delta/array/mod.rs b/encodings/fastlanes/src/delta/array/mod.rs index 444c66e11c5..6b1079d0e4f 100644 --- a/encodings/fastlanes/src/delta/array/mod.rs +++ b/encodings/fastlanes/src/delta/array/mod.rs @@ -23,9 +23,12 @@ pub struct DeltaSlots { /// The base values for each block of deltas. #[slot(0)] pub bases: ArrayRef, - /// The delta-encoded values relative to the base values. + /// Nonnullable delta-encoded values relative to the base values. #[slot(1)] pub deltas: ArrayRef, + /// Logical validity, without transposition or chunk padding. + #[slot(2)] + pub validity_child: Option, } /// A FastLanes-style delta-encoded array of primitive values. @@ -81,7 +84,10 @@ pub struct DeltaSlots { /// [FastLanes](https://www.vldb.org/pvldb/vol16/p2132-afroozeh.pdf) order which splits the 1,024 /// values into one contiguous sub-sequence per-lane, thus permitting delta encoding. /// -/// Note the validity is stored in the deltas array. +/// Validity is stored at the top level; bases and deltas are nonnullable. Before encoding, +/// null source values repeat the preceding valid value, also across chunk boundaries, or zero +/// before the first valid value. Their deltas are retained, including in child min/max +/// statistics, because later values depend on them. Logical statistics exclude nulls. #[derive(Clone, Debug)] pub struct DeltaData { pub(super) offset: usize, diff --git a/encodings/fastlanes/src/delta/compute/cast.rs b/encodings/fastlanes/src/delta/compute/cast.rs index 5fe84f62b62..a6321c96560 100644 --- a/encodings/fastlanes/src/delta/compute/cast.rs +++ b/encodings/fastlanes/src/delta/compute/cast.rs @@ -4,7 +4,6 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; -use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability::NonNullable; use vortex_array::scalar_fn::fns::cast::CastReduce; @@ -29,11 +28,22 @@ impl CastReduce for Delta { return Ok(None); } - let casted_bases = array.bases().cast(dtype.with_nullability(NonNullable))?; - let casted_deltas = array.deltas().cast(dtype.clone())?; + let validity = array.validity()?; + let validity = if dtype.is_nullable() { + validity.into_nullable() + } else { + validity + }; Ok(Some( - Delta::try_new(casted_bases, casted_deltas, array.offset(), array.len())?.into_array(), + Delta::try_new( + array.bases().clone(), + array.deltas().clone(), + validity, + array.offset(), + array.len(), + )? + .into_array(), )) } } @@ -54,9 +64,11 @@ mod tests { use vortex_array::dtype::PType; use vortex_buffer::buffer; use vortex_error::VortexResult; + use vortex_error::vortex_err; use vortex_session::VortexSession; use crate::Delta; + use crate::DeltaArraySlotsExt; static SESSION: LazyLock = LazyLock::new(|| { let session = vortex_array::array_session(); crate::initialize(&session); @@ -111,7 +123,6 @@ mod tests { #[test] fn test_cast_delta_add_nullability() -> VortexResult<()> { - // Same ptype, only adding nullability — handled by the kernel without decompressing. let values = PrimitiveArray::from_iter([10u32, 20, 5, 30, 15]); let array = Delta::try_from_primitive_array(&values, &mut SESSION.create_execution_ctx())?; @@ -123,6 +134,11 @@ mod tests { casted.dtype(), &DType::Primitive(PType::U32, Nullability::Nullable) ); + let reduced = casted + .as_opt::() + .ok_or_else(|| vortex_err!("expected nullability cast to preserve Delta"))?; + assert!(!reduced.deltas().dtype().is_nullable()); + assert!(!reduced.bases().dtype().is_nullable()); assert_arrays_eq!( casted, PrimitiveArray::from_option_iter([Some(10u32), Some(20), Some(5), Some(30), Some(15)]), @@ -133,8 +149,6 @@ mod tests { #[test] fn test_cast_delta_nullability_preserves_nulls() -> VortexResult<()> { - // A nullable Delta array carries its validity in the deltas child; a same-ptype - // nullability cast must round-trip the null positions. let values = PrimitiveArray::from_option_iter([Some(10u32), None, Some(30), Some(15), None]); let array = Delta::try_from_primitive_array(&values, &mut SESSION.create_execution_ctx())?; diff --git a/encodings/fastlanes/src/delta/mod.rs b/encodings/fastlanes/src/delta/mod.rs index 3cf2d4a6815..9c24a78c960 100644 --- a/encodings/fastlanes/src/delta/mod.rs +++ b/encodings/fastlanes/src/delta/mod.rs @@ -10,6 +10,9 @@ pub use array::delta_compress::delta_compress; mod compute; +#[cfg(test)] +mod tests; + mod vtable; pub use vtable::Delta; pub use vtable::DeltaArray; diff --git a/encodings/fastlanes/src/delta/tests.rs b/encodings/fastlanes/src/delta/tests.rs new file mode 100644 index 00000000000..c785ca5bdcb --- /dev/null +++ b/encodings/fastlanes/src/delta/tests.rs @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Delta roundtrips and compute operations against primitive arrays. + +use rand::RngExt; +use rand::SeedableRng; +use rand::rngs::StdRng; +use rstest::rstest; +use vortex_array::ArrayContext; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::aggregate_fn::fns::min_max::min_max; +use vortex_array::arrays::BoolArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::assert_arrays_eq; +use vortex_array::dtype::NativePType; +use vortex_array::dtype::PType; +use vortex_array::match_each_integer_ptype; +use vortex_array::serde::SerializeOptions; +use vortex_array::serde::SerializedArray; +use vortex_array::validity::Validity; +use vortex_buffer::Buffer; +use vortex_buffer::ByteBufferMut; +use vortex_error::VortexResult; +use vortex_error::vortex_err; +use vortex_mask::Mask; +use vortex_session::VortexSession; +use vortex_session::registry::ReadContext; + +use crate::Delta; +use crate::DeltaArray; +use crate::DeltaArraySlotsExt; +use crate::FL_CHUNK_SIZE; +use crate::bitpack_compress::bitpack_encode; + +fn session() -> VortexSession { + let session = vortex_array::array_session(); + crate::initialize(&session); + session +} + +fn check_roundtrip(source: &PrimitiveArray, ctx: &mut ExecutionCtx) -> VortexResult { + let delta = Delta::try_from_primitive_array(source, ctx)?; + assert!(matches!(delta.bases().validity()?, Validity::NonNullable)); + assert!(matches!(delta.deltas().validity()?, Validity::NonNullable)); + assert!(!delta.bases().dtype().is_nullable()); + assert!(!delta.deltas().dtype().is_nullable()); + if let Some(validity) = delta.validity_child() { + assert_eq!(validity.len(), source.len()); + assert_eq!(validity.dtype(), &Validity::DTYPE); + } + let decoded = delta.as_array().clone().execute::(ctx)?; + assert_arrays_eq!(decoded, source, ctx); + assert_eq!( + min_max(delta.as_array(), ctx, Default::default())?, + min_max(source.as_array(), ctx, Default::default())?, + ); + + let valid = source.validity()?.execute_mask(source.len(), ctx)?; + match_each_integer_ptype!(source.ptype(), |T| { + check_filled_values( + source.as_slice::(), + decoded.to_buffer::(), + &valid, + ctx, + ); + }); + Ok(delta) +} + +fn check_filled_values( + values: &[T], + decoded: Buffer, + valid: &Mask, + ctx: &mut ExecutionCtx, +) { + let mut filled = Vec::with_capacity(values.len()); + let mut previous = T::default(); + for (&value, is_valid) in values.iter().zip(valid.iter()) { + if is_valid { + previous = value; + } + filled.push(previous); + } + assert_arrays_eq!( + PrimitiveArray::new(decoded, Validity::NonNullable), + PrimitiveArray::from_iter(filled), + ctx + ); +} + +#[rstest] +#[case::all_null(0)] +#[case::alternating(1)] +#[case::leading_nulls(2)] +fn delta_null_patterns(#[case] pattern: usize) -> VortexResult<()> { + let session = session(); + let mut ctx = session.create_execution_ctx(); + for len in [0, 1, 63, 1023, 1024, 1025, 2049, 3072] { + let validity = Validity::from_iter((0..len).map(|index| match pattern { + 0 => false, + 1 => index % 2 == 1, + _ => index % FL_CHUNK_SIZE >= 63, + })); + let source = PrimitiveArray::new( + (0..len) + .map(|index| 1000 + index as u32) + .collect::>(), + validity, + ); + let delta = check_roundtrip(&source, &mut ctx)?; + let deltas = delta.deltas().clone().execute::(&mut ctx)?; + let packed = Delta::try_new( + delta.bases().clone(), + bitpack_encode(&deltas, 1, None, &mut ctx)?.into_array(), + delta.validity()?, + 0, + len, + )?; + assert_arrays_eq!(packed, source, &mut ctx); + + for array in [delta.into_array(), packed.into_array()] { + let write_ctx = ArrayContext::empty(); + let mut bytes = ByteBufferMut::empty(); + for buffer in array.serialize(&write_ctx, &session, &SerializeOptions::default())? { + bytes.extend_from_slice(&buffer); + } + let restored = SerializedArray::try_from(bytes.freeze())?.decode( + array.dtype(), + array.len(), + &ReadContext::new(write_ctx.to_ids()), + &session, + )?; + let restored_delta = restored + .as_opt::() + .ok_or_else(|| vortex_err!("expected deserialized Delta"))?; + assert!(!restored_delta.deltas().dtype().is_nullable()); + assert_arrays_eq!(restored, source, &mut ctx); + } + } + Ok(()) +} + +#[rstest] +#[case::u8(PType::U8)] +#[case::u16(PType::U16)] +#[case::u32(PType::U32)] +#[case::u64(PType::U64)] +#[case::i8(PType::I8)] +#[case::i16(PType::I16)] +#[case::i32(PType::I32)] +#[case::i64(PType::I64)] +fn delta_randomized_model(#[case] ptype: PType) -> VortexResult<()> { + let session = session(); + let mut ctx = session.create_execution_ctx(); + let mut rng = StdRng::seed_from_u64(0xde17a); + for _ in 0..64 { + let len = rng.random_range(1..=3073); + let valid_probability = rng.random_range(0.0..=1.0); + let validity = Validity::from_iter((0..len).map(|_| rng.random_bool(valid_probability))); + let source = match_each_integer_ptype!(ptype, |T| { + PrimitiveArray::new( + (0..len).map(|_| rng.random::()).collect::>(), + validity, + ) + }); + let delta = check_roundtrip(&source, &mut ctx)?.into_array(); + let start = rng.random_range(0..=len); + let end = rng.random_range(start..=len); + let sliced = delta.slice(start..end)?; + assert_arrays_eq!(sliced, source.slice(start..end)?, &mut ctx); + let nested_start = rng.random_range(0..=end - start); + assert_arrays_eq!( + sliced.slice(nested_start..end - start)?, + source.slice(start + nested_start..end)?, + &mut ctx + ); + + let indices = PrimitiveArray::from_option_iter((0..32).map(|_| { + rng.random_bool(0.8) + .then(|| rng.random_range(0..len) as u64) + })) + .into_array(); + assert_arrays_eq!( + delta.take(indices.clone())?, + source.take(indices)?, + &mut ctx + ); + let mask = Mask::from_iter((0..len).map(|_| rng.random_bool(0.5))); + assert_arrays_eq!(delta.filter(mask.clone())?, source.filter(mask)?, &mut ctx); + } + Ok(()) +} + +#[test] +fn delta_stats_include_null_slot_residuals() -> VortexResult<()> { + let session = session(); + let mut ctx = session.create_execution_ctx(); + let source = PrimitiveArray::from_option_iter( + (0u32..1024).map(|index| (index % 2 == 1).then_some(1000 + index)), + ); + let delta = check_roundtrip(&source, &mut ctx)?; + let deltas = delta.deltas().clone().execute::(&mut ctx)?; + assert_eq!(deltas.statistics().compute_min::(&mut ctx), Some(0)); + assert_eq!(deltas.statistics().compute_max::(&mut ctx), Some(1001)); + assert_eq!(delta.statistics().compute_min::(&mut ctx), Some(1001)); + assert_eq!(delta.statistics().compute_max::(&mut ctx), Some(2023)); + Ok(()) +} + +#[rstest] +#[case::all_valid(Validity::AllValid)] +#[case::all_invalid(Validity::AllInvalid)] +#[case::bitmap(Validity::from_iter((0..FL_CHUNK_SIZE).map(|index| index % 2 == 0)))] +fn delta_rejects_nullable_deltas(#[case] validity: Validity) -> VortexResult<()> { + let session = session(); + let mut ctx = session.create_execution_ctx(); + let source = PrimitiveArray::from_iter(0u32..1024); + let delta = Delta::try_from_primitive_array(&source, &mut ctx)?; + let deltas = delta.deltas().clone().execute::(&mut ctx)?; + let nullable = PrimitiveArray::new(deltas.to_buffer::(), validity); + assert!( + Delta::try_new( + delta.bases().clone(), + nullable.into_array(), + Validity::AllValid, + 0, + source.len() + ) + .is_err() + ); + Ok(()) +} + +#[test] +fn delta_rejects_invalid_top_validity() -> VortexResult<()> { + let session = session(); + let mut ctx = session.create_execution_ctx(); + let source = PrimitiveArray::from_iter(0u32..100); + let delta = Delta::try_from_primitive_array(&source, &mut ctx)?; + for validity in [ + BoolArray::from_iter([true, false]).into_array(), + PrimitiveArray::from_iter(0u32..100).into_array(), + ] { + assert!( + Delta::try_new( + delta.bases().clone(), + delta.deltas().clone(), + Validity::Array(validity), + 0, + source.len() + ) + .is_err() + ); + } + Ok(()) +} diff --git a/encodings/fastlanes/src/delta/vtable/mod.rs b/encodings/fastlanes/src/delta/vtable/mod.rs index 1f2e3823689..8111c73eaf4 100644 --- a/encodings/fastlanes/src/delta/vtable/mod.rs +++ b/encodings/fastlanes/src/delta/vtable/mod.rs @@ -21,7 +21,9 @@ use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::PType; use vortex_array::serde::ArrayChildren; +use vortex_array::validity::Validity; use vortex_array::vtable::VTable; +use vortex_array::vtable::validity_to_child; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_err; @@ -30,6 +32,7 @@ use vortex_session::VortexSession; use vortex_session::registry::CachedId; use crate::DeltaData; +use crate::FL_CHUNK_SIZE; use crate::delta::array::DeltaArrayExt; use crate::delta::array::DeltaArraySlotsExt; use crate::delta::array::DeltaSlots; @@ -89,6 +92,7 @@ impl VTable for Delta { validate_parts( delta_slots.bases, delta_slots.deltas, + delta_slots.validity_child, data.offset, dtype, len, @@ -155,26 +159,37 @@ impl VTable for Delta { buffers.len() ); vortex_ensure!( - children.len() == 2, - "DeltaArray expects 2 children, got {}", + children.len() == 2 || children.len() == 3, + "DeltaArray expects 2 or 3 children, got {}", children.len() ); let metadata = DeltaMetadata::decode(metadata)?; let ptype = PType::try_from(dtype)?; let lanes = lane_count(ptype); - // Compute the length of the bases array let deltas_len = usize::try_from(metadata.deltas_len) .map_err(|_| vortex_err!("deltas_len {} overflowed usize", metadata.deltas_len))?; - let num_chunks = deltas_len / 1024; - let remainder_base_size = if deltas_len % 1024 > 0 { 1 } else { 0 }; - let bases_len = num_chunks * lanes + remainder_base_size; + vortex_ensure!( + deltas_len.is_multiple_of(FL_CHUNK_SIZE), + "deltas length must be a multiple of {FL_CHUNK_SIZE}" + ); + let bases_len = deltas_len / FL_CHUNK_SIZE * lanes; - let bases = children.get(0, dtype, bases_len)?; - let deltas = children.get(1, dtype, deltas_len)?; + let bases = children.get(0, &dtype.as_nonnullable(), bases_len)?; + let deltas = children.get(1, &dtype.as_nonnullable(), deltas_len)?; + let validity_child = if children.len() == 3 { + Some(children.get(2, &Validity::DTYPE, len)?) + } else { + None + }; let data = DeltaData::try_new(metadata.offset as usize)?; - let slots = DeltaSlots { bases, deltas }.into_slots(); + let slots = DeltaSlots { + bases, + deltas, + validity_child, + } + .into_slots(); Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } @@ -189,15 +204,22 @@ impl VTable for Delta { pub struct Delta; impl Delta { + /// Construct Delta from nonnullable numeric children and logical validity. pub fn try_new( bases: ArrayRef, deltas: ArrayRef, + validity: Validity, offset: usize, len: usize, ) -> VortexResult { - let dtype = bases.dtype().with_nullability(deltas.dtype().nullability()); + let dtype = bases.dtype().with_nullability(validity.nullability()); let data = DeltaData::try_new(offset)?; - let slots = DeltaSlots { bases, deltas }.into_slots(); + let slots = DeltaSlots { + bases, + deltas, + validity_child: validity_to_child(&validity, len), + } + .into_slots(); Array::try_from_parts(ArrayParts::new(Delta, dtype, len, data).with_slots(slots)) } @@ -208,24 +230,35 @@ impl Delta { ) -> VortexResult { let logical_len = array.len(); let (bases, deltas) = delta_compress(array, ctx)?; - Self::try_new(bases.into_array(), deltas.into_array(), 0, logical_len) + Self::try_new( + bases.into_array(), + deltas.into_array(), + array.validity()?, + 0, + logical_len, + ) } } fn validate_parts( bases: &ArrayRef, deltas: &ArrayRef, + validity_child: Option<&ArrayRef>, offset: usize, dtype: &DType, len: usize, ) -> VortexResult<()> { vortex_ensure!( - offset + len <= deltas.len(), + offset <= deltas.len() && len <= deltas.len() - offset, "offset + len, {offset} + {len}, must be less than or equal to the size of deltas: {}", deltas.len() ); vortex_ensure!( - bases.dtype().eq_ignore_nullability(deltas.dtype()), + !bases.dtype().is_nullable() && !deltas.dtype().is_nullable(), + "DeltaArray: bases and deltas must be nonnullable" + ); + vortex_ensure!( + bases.dtype() == deltas.dtype(), "DeltaArray: bases and deltas must have the same dtype, got {} and {}", bases.dtype(), deltas.dtype() @@ -237,11 +270,25 @@ fn validate_parts( bases.dtype() ); - let expected_dtype = bases.dtype().with_nullability(deltas.dtype().nullability()); + let expected_dtype = bases.dtype().with_nullability(dtype.nullability()); vortex_ensure!( dtype == &expected_dtype, "DeltaArray dtype mismatch: expected {expected_dtype}, got {dtype}" ); + if let Some(validity) = validity_child { + vortex_ensure!( + dtype.is_nullable(), + "DeltaArray: validity requires a nullable dtype" + ); + vortex_ensure!( + validity.dtype() == &Validity::DTYPE, + "DeltaArray: validity must be nonnullable bool" + ); + vortex_ensure!( + validity.len() == len, + "DeltaArray: validity length must equal logical length {len}" + ); + } let lanes = lane_count(bases.dtype().as_ptype()); @@ -251,8 +298,8 @@ fn validate_parts( deltas.len(), ); vortex_ensure!( - bases.len().is_multiple_of(lanes), - "bases length ({}) must be a multiple of LANES ({lanes})", + bases.len() == deltas.len() / FL_CHUNK_SIZE * lanes, + "bases length ({}) must equal the number of chunks times LANES ({lanes})", bases.len(), ); Ok(()) diff --git a/encodings/fastlanes/src/delta/vtable/rules.rs b/encodings/fastlanes/src/delta/vtable/rules.rs index 898dc3cfeb6..d6892897ab5 100644 --- a/encodings/fastlanes/src/delta/vtable/rules.rs +++ b/encodings/fastlanes/src/delta/vtable/rules.rs @@ -3,11 +3,11 @@ use vortex_array::arrays::slice::SliceReduceAdaptor; use vortex_array::optimizer::rules::ParentRuleSet; +use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor; use crate::delta::vtable::Delta; pub(crate) static RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&SliceReduceAdaptor(Delta)), - // TODO(joe): fixme, this is incorrect.. - // ParentRuleSet::lift(&CastReduceAdaptor(Delta)), + ParentRuleSet::lift(&CastReduceAdaptor(Delta)), ]); diff --git a/encodings/fastlanes/src/delta/vtable/slice.rs b/encodings/fastlanes/src/delta/vtable/slice.rs index be028b2f2ca..02bac650f35 100644 --- a/encodings/fastlanes/src/delta/vtable/slice.rs +++ b/encodings/fastlanes/src/delta/vtable/slice.rs @@ -34,7 +34,14 @@ impl SliceReduce for Delta { .slice(min(start_chunk * 1024, deltas.len())..min(stop_chunk * 1024, deltas.len()))?; Ok(Some( - Delta::try_new(new_bases, new_deltas, physical_start % 1024, range.len())?.into_array(), + Delta::try_new( + new_bases, + new_deltas, + array.validity()?.slice(range.clone())?, + physical_start % 1024, + range.len(), + )? + .into_array(), )) } } diff --git a/encodings/fastlanes/src/delta/vtable/validity.rs b/encodings/fastlanes/src/delta/vtable/validity.rs index 3e56e696d45..5ba4ca2bcd2 100644 --- a/encodings/fastlanes/src/delta/vtable/validity.rs +++ b/encodings/fastlanes/src/delta/vtable/validity.rs @@ -2,25 +2,20 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_array::ArrayView; -use vortex_array::IntoArray; use vortex_array::validity::Validity; use vortex_array::vtable::ValidityVTable; +use vortex_array::vtable::child_to_validity; use vortex_error::VortexResult; use crate::Delta; -use crate::TransposedBool; -use crate::delta::array::DeltaArrayExt; use crate::delta::array::DeltaArraySlotsExt; impl ValidityVTable for Delta { fn validity(array: ArrayView<'_, Delta>) -> VortexResult { - let start = array.offset(); - let stop = start + array.len(); - let validity = match array.deltas().validity()? { - Validity::Array(mask) => Validity::Array(TransposedBool::try_new(mask)?.into_array()), - validity => validity, - }; - validity.slice(start..stop) + Ok(child_to_validity( + array.validity_child(), + array.dtype().nullability(), + )) } } @@ -30,6 +25,7 @@ mod tests { use std::sync::LazyLock; use rstest::rstest; + use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; @@ -45,7 +41,6 @@ mod tests { use vortex_session::VortexSession; use super::*; - use crate::TransposedBool; use crate::delta::array::delta_compress::delta_compress; static SESSION: LazyLock = LazyLock::new(|| { @@ -89,13 +84,10 @@ mod tests { let Validity::Array(validity) = sliced.validity()? else { vortex_bail!("expected array-backed validity") }; - assert!(validity.is::()); assert_arrays_eq!(validity, expected_validity(1000..1050), &mut ctx); Ok(()) } - /// Slicing a DeltaArray must slice its lazily-untransposed validity to the same logical - /// range, wherever the slice falls relative to 1,024-element chunk boundaries. #[rstest] #[case::within_first_chunk(10..1000)] #[case::cross_chunk_boundary(1000..1050)] @@ -111,13 +103,11 @@ mod tests { let Validity::Array(validity) = sliced.validity()? else { vortex_bail!("expected array-backed validity") }; - assert!(validity.is::()); assert_arrays_eq!(validity, expected_validity(range.clone()), &mut ctx); assert_arrays_eq!(sliced, primitive.slice(range)?, &mut ctx); Ok(()) } - /// A slice of a slice must compose the physical offsets before untransposing the validity. #[test] fn validity_of_nested_slice_composes_offsets() -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); @@ -128,31 +118,30 @@ mod tests { let Validity::Array(validity) = sliced.validity()? else { vortex_bail!("expected array-backed validity") }; - assert!(validity.is::()); assert_arrays_eq!(validity, expected_validity(1000..1600), &mut ctx); assert_arrays_eq!(sliced, primitive.slice(1000..1600)?, &mut ctx); Ok(()) } - /// Regression: the deltas' storage validity is not always a raw `Bool` array — slicing or a - /// file round-trip can leave it wrapped in a lazy encoding such as `vortex.slice`. The Delta - /// validity must accept it rather than bail. #[test] - fn validity_handles_slice_encoded_storage_validity() -> VortexResult<()> { + fn validity_handles_slice_encoding() -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); let primitive = PrimitiveArray::from_option_iter( (0u32..2048).map(|value| (value % 3 != 0).then_some(value)), ); let (bases, deltas) = delta_compress(&primitive, &mut ctx)?; - // Rebuild the deltas with a lazily slice-encoded validity, as produced when the deltas - // child is sliced and the validity encoding has no static slice reduction. - let Validity::Array(storage_validity) = deltas.validity()? else { - vortex_bail!("expected array-backed storage validity") + let Validity::Array(validity) = primitive.validity()? else { + vortex_bail!("expected array-backed validity") }; - let lazy_validity = SliceArray::try_new(storage_validity, 0..deltas.len())?.into_array(); - let deltas = PrimitiveArray::new(deltas.to_buffer::(), Validity::Array(lazy_validity)); - let delta = Delta::try_new(bases.into_array(), deltas.into_array(), 0, primitive.len())?; + let lazy_validity = SliceArray::try_new(validity, 0..primitive.len())?.into_array(); + let delta = Delta::try_new( + bases.into_array(), + deltas.into_array(), + Validity::Array(lazy_validity), + 0, + primitive.len(), + )?; let Validity::Array(validity) = delta.validity()? else { vortex_bail!("expected array-backed validity") @@ -166,25 +155,24 @@ mod tests { Ok(()) } - /// Regression: the transposed validity bits may sit in a buffer that is not u64-aligned - /// (e.g. a view into a file segment). Reading the delta validity — whole or sliced — must - /// take the copying untranspose path instead of panicking on alignment. #[test] - fn validity_from_unaligned_storage_buffer() -> VortexResult<()> { + fn validity_from_unaligned_buffer() -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); let primitive = nullable_primitive(); let (bases, deltas) = delta_compress(&primitive, &mut ctx)?; - // Rebuild the deltas with the same transposed validity bits in a misaligned buffer. - let Validity::Array(storage_validity) = deltas.validity()? else { - vortex_bail!("expected array-backed storage validity") + let Validity::Array(validity) = primitive.validity()? else { + vortex_bail!("expected array-backed validity") }; - let bits = storage_validity - .execute::(&mut ctx)? - .into_bit_buffer(); + let bits = validity.execute::(&mut ctx)?.into_bit_buffer(); let unaligned = BoolArray::new(misalign_bits(bits), Validity::NonNullable).into_array(); - let deltas = PrimitiveArray::new(deltas.to_buffer::(), Validity::Array(unaligned)); - let delta = Delta::try_new(bases.into_array(), deltas.into_array(), 0, primitive.len())?; + let delta = Delta::try_new( + bases.into_array(), + deltas.into_array(), + Validity::Array(unaligned), + 0, + primitive.len(), + )?; let Validity::Array(validity) = delta.validity()? else { vortex_bail!("expected array-backed validity") @@ -200,9 +188,6 @@ mod tests { Ok(()) } - /// Creating a DeltaArray from a primitive whose validity mask is backed by an unaligned bit - /// buffer must take the copying transpose path and round-trip losslessly. The length is a - /// whole number of chunks so that only the misalignment forces the copy. #[test] fn compress_primitive_with_unaligned_validity_buffer() -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); @@ -216,7 +201,6 @@ mod tests { let Validity::Array(validity) = delta.validity()? else { vortex_bail!("expected array-backed validity") }; - assert!(validity.is::()); assert_arrays_eq!(validity, expected_validity(0..len as usize), &mut ctx); assert_arrays_eq!(delta, primitive, &mut ctx); Ok(()) diff --git a/encodings/fastlanes/src/lib.rs b/encodings/fastlanes/src/lib.rs index 43d83c6fc7f..dcf0f43a54e 100644 --- a/encodings/fastlanes/src/lib.rs +++ b/encodings/fastlanes/src/lib.rs @@ -30,7 +30,6 @@ pub use bitpacking::*; pub use delta::*; pub use r#for::*; pub use rle::*; -pub use transposed_bool::*; use vortex_array::ExecutionCtx; use vortex_array::arrays::BoolArray; use vortex_array::arrays::bool::BoolArrayExt; @@ -39,32 +38,13 @@ use vortex_buffer::Buffer; use vortex_buffer::BufferMut; use vortex_error::VortexResult; -pub mod bit_transpose; mod bitpacking; mod delta; mod r#for; mod rle; -mod transposed_bool; pub const FL_CHUNK_SIZE: usize = 1024; -/// Returns the position in a `FastLanes`-transposed chunk that holds logical element `idx`. -/// -/// The inverse of [`fastlanes::transpose`], which reads the other way round: it returns the -/// logical element held by a given transposed position. `fastlanes` exports only that direction, -/// so accessors that address a transposed chunk by logical index need this one. -/// -/// `fastlanes::transpose` composes `lane`, `order` and `row` as -/// `lane * 64 + FL_ORDER[order] * 8 + row`, so recovering them from the transposed position only -/// needs `FL_ORDER` inverted, and `FL_ORDER` is its own inverse. -pub(crate) const fn untranspose_idx(idx: usize) -> usize { - let lane = idx / 64; - let order = fastlanes::FL_ORDER[(idx % 64) / 8]; - let row = idx % 8; - - row * 128 + order * 16 + lane -} - use bitpacking::compute::is_constant::BitPackedIsConstantKernel; use r#for::compute::is_constant::FoRIsConstantKernel; use r#for::compute::is_sorted::FoRIsSortedKernel; @@ -89,7 +69,6 @@ pub fn initialize(session: &VortexSession) { session.arrays().register(Delta); session.arrays().register(FoR); session.arrays().register(RLE); - session.arrays().register(TransposedBool); bitpacking::initialize(session); r#for::initialize(session); rle::initialize(session); @@ -112,19 +91,27 @@ pub fn initialize(session: &VortexSession) { ); } +/// What the fill-forward carry does when it enters a new [`FL_CHUNK_SIZE`] chunk. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum ChunkBoundary { + /// Restart from `T::default()`. Encodings whose values are chunk-local, such as RLE + /// indices, must not see a value from the previous chunk. + Reset, + /// Keep the last valid value. Delta encodes each chunk against its own bases, so a + /// carried value only keeps the first residual small. + Carry, +} + /// Fill-forward null values in a buffer, replacing each null with the last valid value seen. /// -/// The fill-forward state resets to `T::default()` at every [`FL_CHUNK_SIZE`] boundary -/// so that values from one chunk never leak into the next. This is important because -/// both RLE and Delta encodings treat each chunk independently: a fill-forwarded value -/// that crosses a chunk boundary can become an invalid chunk-local index (for RLE) or -/// an incorrect delta base (for Delta). +/// `boundary` decides whether the carried value survives a [`FL_CHUNK_SIZE`] boundary. /// /// Returns the original buffer if there are no nulls (i.e. the validity is /// `NonNullable` or `AllValid`), avoiding any allocation or copy. pub(crate) fn fill_forward_nulls( values: Buffer, validity: &Validity, + boundary: ChunkBoundary, ctx: &mut ExecutionCtx, ) -> VortexResult> { match validity { @@ -136,6 +123,7 @@ pub(crate) fn fill_forward_nulls( .execute::(ctx)? .to_bit_buffer(); let mut last_valid = T::default(); + let resets = boundary == ChunkBoundary::Reset; match values.try_into_mut() { Ok(mut to_fill_mut) => { for (i, (v, is_valid)) in @@ -143,9 +131,10 @@ pub(crate) fn fill_forward_nulls( { if is_valid { last_valid = *v; - } else if i.is_multiple_of(FL_CHUNK_SIZE) { - last_valid = T::default(); } else { + if resets && i.is_multiple_of(FL_CHUNK_SIZE) { + last_valid = T::default(); + } *v = last_valid; } } @@ -165,7 +154,7 @@ pub(crate) fn fill_forward_nulls( { if is_valid { last_valid = *v; - } else if i.is_multiple_of(FL_CHUNK_SIZE) { + } else if resets && i.is_multiple_of(FL_CHUNK_SIZE) { last_valid = T::default(); } out.write(last_valid); @@ -182,6 +171,7 @@ pub(crate) fn fill_forward_nulls( mod test { use std::sync::LazyLock; + use rstest::rstest; use vortex_array::VortexSessionExecute; use vortex_buffer::BitBufferMut; use vortex_session::VortexSession; @@ -194,44 +184,34 @@ mod test { session }); - /// `untranspose_idx` derives the inverse of `fastlanes::transpose` from `FL_ORDER` being - /// its own inverse, so prove the round trip over a whole chunk in both directions. - #[test] - fn untranspose_idx_inverts_transpose() { - for idx in 0..FL_CHUNK_SIZE { - assert_eq!(untranspose_idx(fastlanes::transpose(idx)), idx, "idx={idx}"); - assert_eq!(fastlanes::transpose(untranspose_idx(idx)), idx, "idx={idx}"); - } - } - - #[test] - fn fill_forward_nulls_resets_at_chunk_boundary() -> VortexResult<()> { + /// Only one value is valid, the last of chunk 0. Chunk 1 is all null and must either + /// restart from zero or repeat that value, and the shared-buffer copy path must agree. + #[rstest] + #[case::reset_owned(ChunkBoundary::Reset, false, 0)] + #[case::reset_shared(ChunkBoundary::Reset, true, 0)] + #[case::carry_owned(ChunkBoundary::Carry, false, 42)] + #[case::carry_shared(ChunkBoundary::Carry, true, 42)] + fn fill_forward_nulls_at_chunk_boundary( + #[case] boundary: ChunkBoundary, + #[case] shared: bool, + #[case] next_chunk_fill: u32, + ) -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); - // Build a buffer spanning two chunks where the last valid value in chunk 0 - // is non-zero. Null positions at the start of chunk 1 must get T::default() - // (0), not the carry-over from chunk 0. - let mut values = BufferMut::zeroed(2 * FL_CHUNK_SIZE); - // Place a non-zero valid value near the end of chunk 0. + let mut values = BufferMut::from_iter(std::iter::repeat_n(99u32, 2 * FL_CHUNK_SIZE)); values[FL_CHUNK_SIZE - 1] = 42; let mut validity_bits = BitBufferMut::new_unset(2 * FL_CHUNK_SIZE); - validity_bits.set(FL_CHUNK_SIZE - 1); // only this position is valid + validity_bits.set(FL_CHUNK_SIZE - 1); let validity = Validity::from(validity_bits.freeze()); - let result = fill_forward_nulls(values.freeze(), &validity, &mut ctx)?; - - // Within chunk 0, nulls before the valid element get 0 (default), and the - // valid element itself is 42. - assert_eq!(result[FL_CHUNK_SIZE - 1], 42); - - // Chunk 1 has no valid elements. Every position must be T::default() (0), - // NOT 42 carried over from chunk 0. - for i in FL_CHUNK_SIZE..2 * FL_CHUNK_SIZE { - assert_eq!( - result[i], 0, - "position {i} should be 0, not carried from chunk 0" - ); - } + let values = values.freeze(); + let _shared = shared.then(|| values.clone()); + let result = fill_forward_nulls(values, &validity, boundary, &mut ctx)?; + + let mut expected = BufferMut::zeroed(2 * FL_CHUNK_SIZE); + expected[FL_CHUNK_SIZE - 1] = 42; + expected[FL_CHUNK_SIZE..].fill(next_chunk_fill); + assert_eq!(result, expected.freeze()); Ok(()) } } diff --git a/encodings/fastlanes/src/rle/array/rle_compress.rs b/encodings/fastlanes/src/rle/array/rle_compress.rs index f623b84d778..ea288d95571 100644 --- a/encodings/fastlanes/src/rle/array/rle_compress.rs +++ b/encodings/fastlanes/src/rle/array/rle_compress.rs @@ -20,6 +20,7 @@ use vortex_buffer::BufferMut; use vortex_error::VortexExpect; use vortex_error::VortexResult; +use crate::ChunkBoundary; use crate::FL_CHUNK_SIZE; use crate::RLE; use crate::RLEArray; @@ -47,7 +48,12 @@ where { // Fill-forward null values so the RLE encoder doesn't see garbage at null positions, // which would create spurious run boundaries and inflate the dictionary. - let values = fill_forward_nulls(array.to_buffer::(), &array.validity()?, ctx)?; + let values = fill_forward_nulls( + array.to_buffer::(), + &array.validity()?, + ChunkBoundary::Reset, + ctx, + )?; let len = values.len(); let padded_len = len.next_multiple_of(FL_CHUNK_SIZE); diff --git a/encodings/fastlanes/src/transposed_bool.rs b/encodings/fastlanes/src/transposed_bool.rs deleted file mode 100644 index 92da3b3b79c..00000000000 --- a/encodings/fastlanes/src/transposed_bool.rs +++ /dev/null @@ -1,368 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::fmt::Display; -use std::fmt::Formatter; -use std::hash::Hash; -use std::hash::Hasher; -use std::ops::Range; - -use vortex_array::Array; -use vortex_array::ArrayEq; -use vortex_array::ArrayHash; -use vortex_array::ArrayId; -use vortex_array::ArrayParts; -use vortex_array::ArrayRef; -use vortex_array::ArrayView; -use vortex_array::EqMode; -use vortex_array::ExecutionCtx; -use vortex_array::ExecutionResult; -use vortex_array::IntoArray; -use vortex_array::TypedArrayRef; -use vortex_array::arrays::BoolArray; -use vortex_array::arrays::slice::SliceReduce; -use vortex_array::arrays::slice::SliceReduceAdaptor; -use vortex_array::buffer::BufferHandle; -use vortex_array::dtype::DType; -use vortex_array::dtype::Nullability; -use vortex_array::optimizer::rules::ParentRuleSet; -use vortex_array::scalar::Scalar; -use vortex_array::serde::ArrayChildren; -use vortex_array::smallvec::smallvec; -use vortex_array::validity::Validity; -use vortex_array::vtable::OperationsVTable; -use vortex_array::vtable::VTable; -use vortex_array::vtable::ValidityVTable; -use vortex_error::VortexExpect; -use vortex_error::VortexResult; -use vortex_error::vortex_bail; -use vortex_error::vortex_ensure; -use vortex_error::vortex_panic; -use vortex_session::VortexSession; -use vortex_session::registry::CachedId; - -use crate::FL_CHUNK_SIZE; -use crate::bit_transpose::untranspose_bitbuffer; -use crate::untranspose_idx; - -/// A non-nullable boolean array stored in FastLanes-transposed order. -pub type TransposedBoolArray = Array; - -/// The array encoding for a boolean bitmap stored in FastLanes-transposed order. -#[derive(Clone, Debug)] -pub struct TransposedBool; - -/// The transposed bitmap, as a non-nullable boolean array covering whole 1,024-bit chunks. -const TRANSPOSED_SLOT: usize = 0; - -/// Per-array data for a [`TransposedBoolArray`]. -#[derive(Clone, Debug)] -pub struct TransposedBoolData { - offset: usize, -} - -impl Display for TransposedBoolData { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "offset: {}", self.offset) - } -} - -impl ArrayHash for TransposedBoolData { - fn array_hash(&self, state: &mut H, _accuracy: EqMode) { - self.offset.hash(state); - } -} - -impl ArrayEq for TransposedBoolData { - fn array_eq(&self, other: &Self, _accuracy: EqMode) -> bool { - self.offset == other.offset - } -} - -/// Accessors for a [`TransposedBoolArray`]. -pub trait TransposedBoolArrayExt: TypedArrayRef { - /// Returns the logical offset into the first transposed chunk. - fn offset(&self) -> usize { - self.deref().offset - } - - /// Returns the backing bitmap in FastLanes-transposed order. - fn transposed(&self) -> &ArrayRef { - self.as_ref().slots()[TRANSPOSED_SLOT] - .as_ref() - .vortex_expect("TransposedBoolArray transposed slot") - } -} - -impl> TransposedBoolArrayExt for T {} - -impl TransposedBool { - /// Creates an array from a boolean array already stored in FastLanes-transposed order. - /// - /// The `transposed` array may use any encoding (e.g. a lazy slice); it is canonicalized when - /// this array is executed. - /// - /// # Errors - /// - /// Returns an error if `transposed` is not a non-nullable boolean array containing complete - /// 1,024-bit chunks. - pub fn try_new(transposed: ArrayRef) -> VortexResult { - let len = transposed.len(); - Self::try_new_view(transposed, 0, len) - } - - fn try_new_view( - transposed: ArrayRef, - offset: usize, - len: usize, - ) -> VortexResult { - Array::try_from_parts( - ArrayParts::new( - TransposedBool, - DType::Bool(Nullability::NonNullable), - len, - TransposedBoolData { offset }, - ) - .with_slots(smallvec![Some(transposed)]), - ) - } -} - -impl VTable for TransposedBool { - type TypedArrayData = TransposedBoolData; - type OperationsVTable = Self; - type ValidityVTable = Self; - - fn id(&self) -> ArrayId { - static ID: CachedId = CachedId::new("fastlanes.transposed_bool"); - *ID - } - - fn validate( - &self, - data: &Self::TypedArrayData, - dtype: &DType, - len: usize, - slots: &[Option], - ) -> VortexResult<()> { - vortex_ensure!( - dtype == &DType::Bool(Nullability::NonNullable), - "TransposedBoolArray must have non-nullable boolean dtype, got {dtype}" - ); - vortex_ensure!( - slots.len() == 1, - "TransposedBoolArray expects one slot, got {}", - slots.len() - ); - let transposed = slots[TRANSPOSED_SLOT] - .as_ref() - .vortex_expect("TransposedBoolArray transposed slot"); - vortex_ensure!( - transposed.dtype() == &DType::Bool(Nullability::NonNullable), - "TransposedBoolArray transposed child must be a non-nullable boolean array, got {}", - transposed.dtype() - ); - vortex_ensure!( - transposed.len().is_multiple_of(FL_CHUNK_SIZE), - "TransposedBoolArray transposed child length {} must be a multiple of {FL_CHUNK_SIZE}", - transposed.len() - ); - vortex_ensure!( - data.offset < FL_CHUNK_SIZE, - "TransposedBoolArray offset {} must be less than {FL_CHUNK_SIZE}", - data.offset - ); - let end = data - .offset - .checked_add(len) - .ok_or_else(|| vortex_error::vortex_err!("TransposedBoolArray range end overflow"))?; - vortex_ensure!( - end <= transposed.len(), - "TransposedBoolArray range {}..{} exceeds transposed child length {}", - data.offset, - end, - transposed.len() - ); - Ok(()) - } - - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { - 0 - } - - fn buffer(_array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { - vortex_panic!("TransposedBoolArray buffer index {idx} out of bounds") - } - - fn buffer_name(_array: ArrayView<'_, Self>, _idx: usize) -> Option { - None - } - - fn with_buffers( - &self, - array: ArrayView<'_, Self>, - buffers: &[BufferHandle], - ) -> VortexResult> { - vortex_array::vtable::with_empty_buffers(self, array, buffers) - } - - fn serialize( - _array: ArrayView<'_, Self>, - _session: &VortexSession, - ) -> VortexResult>> { - vortex_bail!("Cannot serialise TransposedBoolArray"); - } - - fn deserialize( - &self, - _dtype: &DType, - _len: usize, - _metadata: &[u8], - _buffers: &[BufferHandle], - _children: &dyn ArrayChildren, - _session: &VortexSession, - ) -> VortexResult> { - vortex_bail!("Cannot deserialise TransposedBoolArray"); - } - - fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { - match idx { - TRANSPOSED_SLOT => "transposed".to_string(), - _ => vortex_panic!("TransposedBoolArray slot index {idx} out of bounds"), - } - } - - fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { - let len = array.len(); - let offset = array.offset(); - let bits = array - .transposed() - .clone() - .execute::(ctx)? - .into_bit_buffer(); - let untransposed = BoolArray::new(untranspose_bitbuffer(bits), Validity::NonNullable); - Ok(ExecutionResult::done( - untransposed.slice(offset..offset + len)?, - )) - } - - fn reduce_parent( - array: ArrayView<'_, Self>, - parent: &ArrayRef, - child_idx: usize, - ) -> VortexResult> { - RULES.evaluate(array, parent, child_idx) - } -} - -impl OperationsVTable for TransposedBool { - type ProbeState = (); - - fn scalar_at( - array: ArrayView<'_, TransposedBool>, - index: usize, - ctx: &mut ExecutionCtx, - ) -> VortexResult { - let logical_index = array.offset() + index; - let chunk_start = logical_index / FL_CHUNK_SIZE * FL_CHUNK_SIZE; - let transposed_index = chunk_start + untranspose_idx(logical_index % FL_CHUNK_SIZE); - array.transposed().execute_scalar(transposed_index, ctx) - } -} - -impl ValidityVTable for TransposedBool { - fn validity(_array: ArrayView<'_, TransposedBool>) -> VortexResult { - Ok(Validity::NonNullable) - } -} - -impl SliceReduce for TransposedBool { - fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { - let physical_start = array.offset() + range.start; - let physical_stop = array.offset() + range.end; - let start_chunk = physical_start / FL_CHUNK_SIZE; - let stop_chunk = physical_stop.div_ceil(FL_CHUNK_SIZE); - let transposed = array - .transposed() - .slice(start_chunk * FL_CHUNK_SIZE..stop_chunk * FL_CHUNK_SIZE)?; - - Ok(Some( - TransposedBool::try_new_view(transposed, physical_start % FL_CHUNK_SIZE, range.len())? - .into_array(), - )) - } -} - -static RULES: ParentRuleSet = - ParentRuleSet::new(&[ParentRuleSet::lift(&SliceReduceAdaptor(TransposedBool))]); - -#[cfg(test)] -mod tests { - use vortex_array::VortexSessionExecute; - use vortex_array::array_session; - use vortex_array::arrays::SliceArray; - use vortex_array::assert_arrays_eq; - use vortex_buffer::BitBuffer; - use vortex_error::VortexResult; - - use super::*; - use crate::bit_transpose::transpose_bitbuffer; - - fn test_bits() -> BitBuffer { - BitBuffer::from_iter((0..2 * FL_CHUNK_SIZE).map(|i| i % 3 != 0 && i % 11 != 0)) - } - - fn transposed_bool_array(bits: BitBuffer) -> ArrayRef { - BoolArray::new(transpose_bitbuffer(bits), Validity::NonNullable).into_array() - } - - #[test] - fn execute_full_array() -> VortexResult<()> { - let expected = test_bits(); - let array = TransposedBool::try_new(transposed_bool_array(expected.clone()))?; - let mut ctx = array_session().create_execution_ctx(); - - assert_arrays_eq!(array, BoolArray::from(expected), &mut ctx); - Ok(()) - } - - #[test] - fn slice_stays_lazy_and_translates_scalars() -> VortexResult<()> { - let expected = test_bits(); - let array = TransposedBool::try_new(transposed_bool_array(expected.clone()))?; - let sliced = array.slice(1000..1050)?; - assert!(sliced.is::()); - - let mut ctx = array_session().create_execution_ctx(); - for index in [0, 23, 49] { - assert_eq!( - sliced.execute_scalar(index, &mut ctx)?.as_bool().value(), - Some(expected.value(1000 + index)) - ); - } - assert_arrays_eq!( - sliced, - BoolArray::from(expected.slice(1000..1050)), - &mut ctx - ); - Ok(()) - } - - /// Regression: the transposed child may be lazily encoded (e.g. a `vortex.slice` wrapper), - /// which must be canonicalized at execution rather than rejected. - #[test] - fn execute_slice_encoded_child() -> VortexResult<()> { - let expected = test_bits(); - let child = transposed_bool_array(expected.clone()); - let lazy_slice = SliceArray::try_new(child, FL_CHUNK_SIZE..2 * FL_CHUNK_SIZE)?.into_array(); - let array = TransposedBool::try_new(lazy_slice)?; - - let mut ctx = array_session().create_execution_ctx(); - assert_arrays_eq!( - array, - BoolArray::from(expected.slice(FL_CHUNK_SIZE..2 * FL_CHUNK_SIZE)), - &mut ctx - ); - Ok(()) - } -} diff --git a/vortex-btrblocks/src/schemes/integer/delta.rs b/vortex-btrblocks/src/schemes/integer/delta.rs index 46b2f1e302e..17e430e1967 100644 --- a/vortex-btrblocks/src/schemes/integer/delta.rs +++ b/vortex-btrblocks/src/schemes/integer/delta.rs @@ -218,6 +218,13 @@ impl Scheme for DeltaScheme { exec_ctx, )?; - Delta::try_new(compressed_bases, compressed_deltas, 0, len).map(IntoArray::into_array) + Delta::try_new( + compressed_bases, + compressed_deltas, + primitive.validity()?, + 0, + len, + ) + .map(IntoArray::into_array) } } diff --git a/vortex-btrblocks/tests/golden.rs b/vortex-btrblocks/tests/golden.rs index fc636252c27..e35f33e1d29 100644 --- a/vortex-btrblocks/tests/golden.rs +++ b/vortex-btrblocks/tests/golden.rs @@ -135,6 +135,10 @@ fn golden_snapshots( fn corpus() -> VortexResult> { Ok(vec![ ("int_monotone_jitter", int_monotone_jitter()), + ( + "int_monotone_jitter_nullable", + int_monotone_jitter_nullable(), + ), ("int_arithmetic_sequence", int_arithmetic_sequence()), ("int_low_cardinality", int_low_cardinality()), ("int_runs", int_runs()), @@ -170,6 +174,27 @@ fn int_monotone_jitter() -> ArrayRef { PrimitiveArray::new(values, Validity::NonNullable).into_array() } +/// [`int_monotone_jitter`] with a tenth of the slots null: covers the nullable Delta tree, whose +/// validity is a top-level child rather than part of the deltas, once Delta rejoins +/// [`ALL_SCHEMES`](vortex_btrblocks::ALL_SCHEMES). +fn int_monotone_jitter_nullable() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(101); + let mut value = 1_700_000_000_000u64; + let mut validity: Vec = Vec::with_capacity(N); + let values: Buffer = (0..N) + .map(|_| { + value += 900 + rng.random_range(0..200); + validity.push(rng.random_range(0..10) != 0); + value + }) + .collect(); + PrimitiveArray::new( + values, + Validity::Array(BoolArray::from_iter(validity).into_array()), + ) + .into_array() +} + /// Exact arithmetic sequence: Sequence habitat (distinct == len, no nulls). fn int_arithmetic_sequence() -> ArrayRef { let values: Buffer = (0..N as i64).map(|i| 10_000 + 7 * i).collect(); diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__int_monotone_jitter_nullable.snap b/vortex-btrblocks/tests/snapshots/golden__compact__int_monotone_jitter_nullable.snap new file mode 100644 index 00000000000..e179db60728 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__int_monotone_jitter_nullable.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: u64?, len=16384, nbytes=133120 +root: vortex.pco(u64?, len=16384) nbytes=17456 + metadata: ptype: u64, nrows: 16384, slice: 0..16384 + validity: vortex.bool(bool, len=16384) nbytes=2048 + metadata: offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__regular__int_monotone_jitter_nullable.snap b/vortex-btrblocks/tests/snapshots/golden__regular__int_monotone_jitter_nullable.snap new file mode 100644 index 00000000000..de29c6229ee --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__regular__int_monotone_jitter_nullable.snap @@ -0,0 +1,11 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: u64?, len=16384, nbytes=133120 +root: fastlanes.for(u64?, len=16384) nbytes=51200 + metadata: reference: 1700000001036u64 + encoded: fastlanes.bitpacked(u64?, len=16384) nbytes=51200 + metadata: bit_width: 24, offset: 0 + validity_child: vortex.bool(bool, len=16384) nbytes=2048 + metadata: offset: 0 diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index 2cc7bba0481..de2ace23491 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -189,9 +189,15 @@ fn bench_delta_compress_u32(bencher: Bencher) { fn bench_delta_decompress_u32(bencher: Bencher) { let (uint_array, ..) = setup_primitive_arrays(NUM_VALUES); let (bases, deltas) = delta_compress(&uint_array, &mut SESSION.create_execution_ctx()).unwrap(); - let compressed = Delta::try_new(bases.into_array(), deltas.into_array(), 0, uint_array.len()) - .unwrap() - .into_array(); + let compressed = Delta::try_new( + bases.into_array(), + deltas.into_array(), + uint_array.validity().unwrap(), + 0, + uint_array.len(), + ) + .unwrap() + .into_array(); with_byte_counter(bencher, NUM_VALUES * 4) .with_inputs(|| (&compressed, SESSION.create_execution_ctx()))