Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 2 additions & 3 deletions benchmarks/compress-bench/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,8 @@ cargo run -p compress-bench --profile release_debug
`--gpu-decompress` is opt-in, requires the `cuda` feature, and restricts the suite to the
GPU dataset list in `src/main.rs`. It measures decompression only, for two backends:

- **Vortex** — the file is written with CUDA-compatible BtrBlocks encodings only
(`only_cuda_compatible`) and a CUDA flat layout, then decoded on the device all the way to
canonical arrays.
- **Vortex**: the file is written with the CUDA output edition and a CUDA flat layout,
then decoded on the device all the way to canonical arrays.
- **Parquet** — the file is rewritten with GPU-friendly writer settings (see below) and read
back with [cuDF](https://github.com/rapidsai/cudf)'s `read_parquet`, which performs the
whole read on the device: page header decode, codec decompression, dictionary/RLE/plain
Expand Down
22 changes: 14 additions & 8 deletions benchmarks/compress-bench/src/gpu/vortex.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,14 @@ use vortex::array::IntoArray;
use vortex::array::VortexSessionExecute;
use vortex::array::arrays::StructArray;
use vortex::array::arrays::struct_::StructArrayExt;
use vortex::compressor::BtrBlocksCompressorBuilder;
use vortex::compressor::BtrBlocksCompressor;
use vortex::compressor::CompressionSessionExt;
use vortex::editions::EditionSessionExt;
use vortex::editions::EnabledEditions;
use vortex::editions::cuda::CUDA_2026_09_0;
use vortex::error::VortexResult;
use vortex::file::OpenOptionsSessionExt;
use vortex::file::VortexFile;
use vortex::file::WriteOptionsSessionExt;
use vortex::layout::layouts::chunked::writer::ChunkedLayoutStrategy;
use vortex::layout::layouts::compressed::CompressingStrategy;
Expand All @@ -37,7 +42,6 @@ use vortex_bench::compress::CompressedData;
use vortex_bench::compress::Compressor;
use vortex_bench::compress::Uncompressed;
use vortex_bench::conversions::parquet_to_vortex_chunks_with_batch_size;
use vortex_bench::retain_edition_encodings;
use vortex_cuda::CanonicalCudaExt;
use vortex_cuda::CudaExecutionCtx;
use vortex_cuda::CudaOpenOptionsExt;
Expand Down Expand Up @@ -100,11 +104,13 @@ impl Compressor for GpuVortexCompressor {
// partition rather than whatever the default strategy would regroup them into.
let strategy = Arc::new(ChunkedLayoutStrategy::new(CompressingStrategy::new(
CudaFlatLayoutStrategy::default(),
retain_edition_encodings(
&SESSION,
BtrBlocksCompressorBuilder::default().only_cuda_compatible(),
)
.build(),
{
let compression_session = CompressionSessionExt::fork_compression(&*SESSION);
compression_session.register(EnabledEditions::default());
EditionSessionExt::set_enabled_editions(&compression_session, [CUDA_2026_09_0])
.expect("CUDA edition must be registered");
BtrBlocksCompressor::from_session(&compression_session)
},
)));
let start = Instant::now();
SESSION
Expand Down Expand Up @@ -169,7 +175,7 @@ impl Compressor for GpuVortexCompressor {
/// Windows too, and the whole crate still has to compile on a developer's macOS machine. Asking
/// for `--gpu-direct-io` where it cannot be honoured is an error rather than a silent no-op,
/// because the flag changes what the resulting number means.
async fn open_gpu(path: &Path, direct_io: bool) -> Result<vortex::file::VortexFile> {
async fn open_gpu(path: &Path, direct_io: bool) -> Result<VortexFile> {
let open_options = SESSION.open_options().with_cuda();

#[cfg(target_os = "linux")]
Expand Down
37 changes: 25 additions & 12 deletions benchmarks/string-bench/src/serialized.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ use vortex::array::IntoArray;
use vortex::array::VortexSessionExecute;
use vortex::array::arrays::ChunkedArray;
use vortex::array::arrays::VarBinViewArray;
use vortex::compressor::BtrBlocksCompressorBuilder;
use vortex::compressor::BtrBlocksCompressor;
use vortex::file::OpenOptionsSessionExt;
use vortex::file::WriteOptionsSessionExt;
use vortex::file::WriteStrategyBuilder;
Expand All @@ -57,7 +57,7 @@ use crate::prepare_column;
use crate::throughput;
use crate::verify_canonicalized;

/// The btrblocks string schemes that `BtrBlocksCompressorBuilder::default()` can
/// The btrblocks string schemes that `BtrBlocksCompressor::from_session` can
/// choose between. Forcing one encoder excludes every entry except its own
/// scheme, so this list must track the default scheme set: add a row whenever a
/// new string encoder becomes selectable by default (e.g. Zstd).
Expand Down Expand Up @@ -170,16 +170,26 @@ impl SerializedResult {

/// Build the file writer strategy that forces one selected string scheme while
/// leaving editioned non-string child compression enabled.
fn serialized_write_strategy(encoder: StringEncoder) -> Arc<dyn LayoutStrategy> {
fn serialized_write_strategy(
session: &VortexSession,
encoder: StringEncoder,
) -> Arc<dyn LayoutStrategy> {
let forced = encoder.scheme_id();
let compressor = BtrBlocksCompressorBuilder::default().exclude_schemes(
default_string_scheme_ids()
let compressor = {
let compression_session =
vortex_btrblocks::CompressionSessionExt::fork_compression(session);
for id in default_string_scheme_ids()
.into_iter()
.filter(|&id| id != forced)
.chain([DeltaScheme::default().id()]),
);
WriteStrategyBuilder::default()
.with_btrblocks_builder(compressor)
.chain([DeltaScheme::default().id()])
{
vortex_btrblocks::CompressionSessionExt::compression(&compression_session)
.unregister(id);
}
BtrBlocksCompressor::from_session(&compression_session)
};
WriteStrategyBuilder::from_session(session)
.with_btrblocks_compressor(compressor)
.build()
}

Expand Down Expand Up @@ -251,7 +261,7 @@ async fn prepare_serialized_file(
verify: bool,
ctx: &mut ExecutionCtx,
) -> Result<SerializedFile> {
let strategy = serialized_write_strategy(encoder);
let strategy = serialized_write_strategy(session, encoder);
let data = write_serialized_file(session, input, &strategy).await?;
let file_bytes = data.len() as u64;

Expand Down Expand Up @@ -355,7 +365,7 @@ mod tests {
use vortex::io::runtime::BlockingRuntime;
use vortex::io::runtime::current::CurrentThreadRuntime;
use vortex::io::session::RuntimeSessionExt;
use vortex_btrblocks::ALL_SCHEMES;
use vortex_btrblocks::CompressionSessionExt;
use vortex_btrblocks::SchemeExt;

use super::*;
Expand All @@ -365,8 +375,11 @@ mod tests {
// Every default scheme whose dtype gate accepts canonical Utf8 must be
// excluded when another root string encoding is forced.
let canonical = Canonical::VarBinView(VarBinViewArray::from_iter_str(["value"]));
let mut actual = ALL_SCHEMES
let session = VortexSession::default();
let mut actual = session
.registered_schemes()
.iter()
.filter(|scheme| !scheme.id().to_string().starts_with("vortex.compressor."))
.filter(|scheme| scheme.matches(&canonical))
.map(|scheme| scheme.id())
.collect::<Vec<_>>();
Expand Down
1 change: 1 addition & 0 deletions encodings/alp/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ num-traits = { workspace = true }
prost = { workspace = true }
vortex-array = { workspace = true }
vortex-buffer = { workspace = true }
vortex-compressor = { workspace = true }
vortex-error = { workspace = true }
vortex-fastlanes = { workspace = true }
vortex-mask = { workspace = true }
Expand Down
3 changes: 3 additions & 0 deletions encodings/alp/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -50,3 +50,6 @@ pub fn initialize(session: &VortexSession) {
&compute::nan_count::ALPNanCountKernel,
);
}

/// Compression schemes and their session registration.
pub mod schemes;
Original file line number Diff line number Diff line change
Expand Up @@ -3,37 +3,42 @@

//! ALP (Adaptive Lossless floating-Point) encoding.

use vortex_alp::ALP;
use vortex_alp::ALPArrayExt;
use vortex_alp::ALPArraySlotsExt;
use vortex_alp::alp_encode;
use vortex_array::ArrayId;
use vortex_array::ArrayRef;
use vortex_array::Canonical;
use vortex_array::ExecutionCtx;
use vortex_array::IntoArray;
use vortex_array::VTable;
use vortex_array::arrays::Constant;
use vortex_array::arrays::Patched;
use vortex_array::arrays::patched::use_experimental_patches;
use vortex_array::arrays::primitive::PrimitiveArrayExt;
use vortex_array::dtype::PType;
use vortex_compressor::CascadingCompressor;
use vortex_compressor::compress_patches;
use vortex_compressor::scheme::CompressionEstimate;
use vortex_compressor::scheme::CompressorContext;
use vortex_compressor::scheme::DeferredEstimate;
use vortex_compressor::scheme::EstimateVerdict;
use vortex_compressor::scheme::Scheme;
use vortex_compressor::scheme::SchemeExt;
use vortex_compressor::stats::ArrayAndStats;
use vortex_error::VortexResult;

use crate::ArrayAndStats;
use crate::CascadingCompressor;
use crate::CompressorContext;
use crate::Scheme;
use crate::SchemeExt;
use crate::compress_patches;
use crate::ALP;
use crate::ALPArrayExt;
use crate::ALPArraySlotsExt;
use crate::alp_encode;

/// ALP (Adaptive Lossless floating-Point) encoding.
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub struct ALPScheme;

impl Scheme for ALPScheme {
fn selection_priority(&self) -> u16 {
10
}

fn scheme_name(&self) -> &'static str {
"vortex.float.alp"
}
Expand All @@ -43,7 +48,7 @@ impl Scheme for ALPScheme {
}

fn produced_encodings(&self) -> Vec<ArrayId> {
let mut encodings = vec![ALP.id()];
let mut encodings = vec![ALP.id(), Constant.id()];
if use_experimental_patches() {
encodings.push(Patched.id());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,35 +3,40 @@

//! ALPRD (ALP with Real Double) encoding variant.

use vortex_alp::ALPRDArrayExt;
use vortex_alp::ALPRDArrayOwnedExt;
use vortex_alp::RDEncoder;
use vortex_alp::RDEncoderExt;
use vortex_array::ArrayId;
use vortex_array::ArrayRef;
use vortex_array::Canonical;
use vortex_array::ExecutionCtx;
use vortex_array::IntoArray;
use vortex_array::VTable;
use vortex_array::arrays::Constant;
use vortex_array::arrays::primitive::PrimitiveArrayExt;
use vortex_array::dtype::PType;
use vortex_compressor::CascadingCompressor;
use vortex_compressor::compress_patches;
use vortex_compressor::scheme::CompressionEstimate;
use vortex_compressor::scheme::CompressorContext;
use vortex_compressor::scheme::DeferredEstimate;
use vortex_compressor::scheme::EstimateVerdict;
use vortex_compressor::scheme::Scheme;
use vortex_compressor::stats::ArrayAndStats;
use vortex_error::VortexResult;
use vortex_error::vortex_panic;

use crate::ArrayAndStats;
use crate::CascadingCompressor;
use crate::CompressorContext;
use crate::Scheme;
use crate::compress_patches;
use crate::ALPRDArrayExt;
use crate::ALPRDArrayOwnedExt;
use crate::RDEncoder;
use crate::RDEncoderExt;

/// ALPRD (ALP with Real Double) encoding variant.
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub struct ALPRDScheme;

impl Scheme for ALPRDScheme {
fn selection_priority(&self) -> u16 {
20
}

fn scheme_name(&self) -> &'static str {
"vortex.float.alprd"
}
Expand All @@ -41,7 +46,7 @@ impl Scheme for ALPRDScheme {
}

fn produced_encodings(&self) -> Vec<ArrayId> {
vec![vortex_alp::ALPRD.id()]
vec![crate::ALPRD.id(), Constant.id()]
}

fn expected_compression_ratio(
Expand Down Expand Up @@ -82,7 +87,7 @@ impl Scheme for ALPRDScheme {
.map(|p| compress_patches(p, exec_ctx))
.transpose()?;

Ok(vortex_alp::ALPRD::try_new(
Ok(crate::ALPRD::try_new(
dtype,
parts.left_parts,
parts.left_parts_dictionary,
Expand Down
Loading
Loading