experiment: pinned CPU thread pool

benchmarks/datafusion-bench/src/main.rs

@@ -230,18 +230,14 @@ async fn main() -> anyhow::Result<()> {
     Ok(())
 }
 
-fn use_scan_api() -> bool {
-    std::env::var("VORTEX_USE_SCAN_API").is_ok_and(|v| v == "1")
-}
-
 async fn register_benchmark_tables<B: Benchmark + ?Sized>(
     session: &SessionContext,
     benchmark: &B,
     format: Format,
 ) -> anyhow::Result<()> {
     match format {
         Format::Arrow => register_arrow_tables(session, benchmark).await,
-        _ if use_scan_api() && matches!(format, Format::OnDiskVortex | Format::VortexCompact) => {
+        Format::OnDiskVortex | Format::VortexCompact => {
             register_v2_tables(session, benchmark, format).await
         }
         _ => {

vortex-bench/src/lib.rs

@@ -68,7 +68,7 @@ use vortex::session::VortexSession;
 static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
 
 pub static SESSION: LazyLock<VortexSession> =
-    LazyLock::new(|| VortexSession::default().with_tokio());
+    LazyLock::new(|| VortexSession::default().with_pinned_cpu_pool());
 
 #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)]
 pub struct Target {

vortex-io/Cargo.toml

@@ -44,6 +44,7 @@ vortex-session = { workspace = true }
 
 [target.'cfg(unix)'.dependencies]
 custom-labels = { workspace = true }
+libc = "0.2"
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 # Smol is our default impl, so we don't want it to be optional, but it cannot be part of wasm

vortex-io/src/runtime/mod.rs

@@ -22,6 +22,8 @@ pub use handle::*;
 #[cfg(not(target_arch = "wasm32"))]
 pub mod current;
 #[cfg(not(target_arch = "wasm32"))]
+pub mod pinned_pool;
+#[cfg(not(target_arch = "wasm32"))]
 mod pool;
 #[cfg(not(target_arch = "wasm32"))]
 pub mod single;