diff --git a/.github/workflows/sql-bench-matrix.yml b/.github/workflows/sql-bench-matrix.yml index 0db85313641..d81bec1c933 100644 --- a/.github/workflows/sql-bench-matrix.yml +++ b/.github/workflows/sql-bench-matrix.yml @@ -136,6 +136,12 @@ jobs: chmod +x duckdb echo "$PWD" >> "$GITHUB_PATH" + # SSB has no Rust generator; `vx-bench prepare-data ssb` builds the reference C `dbgen` + # from source. See vortex-bench/sql/ssb/README.md. + - name: Install cmake + if: matrix.subcommand == 'ssb' + run: sudo apt-get update && sudo apt-get install -y cmake + - uses: ./.github/actions/system-info - name: Download binaries diff --git a/bench-orchestrator/README.md b/bench-orchestrator/README.md index ecd90e140e4..283bc366729 100644 --- a/bench-orchestrator/README.md +++ b/bench-orchestrator/README.md @@ -41,7 +41,7 @@ vx-bench run [options] **Arguments:** -- `benchmark`: Benchmark suite to run (`appian`, `tpch`, `tpcds`, `clickbench`, `fineweb`, `gh-archive`, `polarsignals`, `public-bi`, `statpopgen`) +- `benchmark`: Benchmark suite to run (`appian`, `tpch`, `tpcds`, `clickbench`, `fineweb`, `gh-archive`, `polarsignals`, `public-bi`, `ssb`, `statpopgen`) **Options:** diff --git a/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py b/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py index 9f8f0fee915..398e6c12e31 100644 --- a/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py +++ b/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py @@ -263,6 +263,19 @@ "develop": STANDARD_WITH_DUCKDB, }, ), + BenchmarkCase( + id="ssb-nvme", + benchmark=Benchmark.SSB, + name="SSB SF=10 on NVME", + scale_factor=10.0, + iterations=10, + runs={ + "pr-compact": COMPACT, + "pr-all": COMPACT, + "pr-full": DEFAULT_WITH_DUCKDB_PR_FULL, + "develop": STANDARD_WITH_DUCKDB, + }, + ), BenchmarkCase( id="vortex-queries", benchmark=Benchmark.VORTEX_QUERIES, diff --git a/bench-orchestrator/bench_orchestrator/config.py b/bench-orchestrator/bench_orchestrator/config.py index dfb1aaa920e..bd3f9977289 100644 --- a/bench-orchestrator/bench_orchestrator/config.py +++ b/bench-orchestrator/bench_orchestrator/config.py @@ -62,6 +62,7 @@ class Benchmark(Enum): PUBLIC_BI = "public-bi" STATPOPGEN = "statpopgen" SPATIALBENCH = "spatialbench" + SSB = "ssb" VORTEX_QUERIES = "vortex" diff --git a/bench-orchestrator/tests/test_matrix.py b/bench-orchestrator/tests/test_matrix.py index d0f765be246..37b42f913fe 100644 --- a/bench-orchestrator/tests/test_matrix.py +++ b/bench-orchestrator/tests/test_matrix.py @@ -31,6 +31,7 @@ "fineweb-s3", "polarsignals", "appian-nvme", + "ssb-nvme", "vortex-queries", ) COMPACT_IDS = tuple( @@ -42,7 +43,7 @@ "pr": tuple( benchmark_id for benchmark_id in REGULAR_IDS - if benchmark_id not in {"tpch-s3-10", "appian-nvme", "vortex-queries"} + if benchmark_id not in {"tpch-s3-10", "appian-nvme", "ssb-nvme", "vortex-queries"} ), "pr-compact": COMPACT_IDS, "pr-all": PR_ALL_IDS, diff --git a/vortex-bench/sql/ssb/README.md b/vortex-bench/sql/ssb/README.md new file mode 100644 index 00000000000..6c77d325ca3 --- /dev/null +++ b/vortex-bench/sql/ssb/README.md @@ -0,0 +1,67 @@ +# Star Schema Benchmark + +The [Star Schema Benchmark](https://www.cs.umb.edu/~poneil/StarSchemaB.PDF) (O'Neil, O'Neil & +Chen) is TPC-H redesigned as a textbook star schema: TPC-H's `lineitem` and `orders` are +denormalized into one wide `lineorder` fact table, joined against four dimensions — `customer`, +`supplier`, `part`, and `dwdate`. + +That shape is what makes it worth running alongside TPC-H. Every query is a scan of the fact +table under dimension-derived filters of a known, deliberately varied selectivity, so the suite +isolates filter pushdown, zone-map pruning, and dimension-join throughput rather than mixing them +with TPC-H's subqueries and correlated predicates. + +## Queries + +The 13 queries are organized into four "flights", each holding the query shape fixed while +tightening the filters. The harness keys queries on a plain index, so the paper's numbering maps +onto `q1.sql` ... `q13.sql` in order: + +| File | SSB | Flight | +| --- | --- | --- | +| `q1.sql` | Q1.1 | Flight 1 — single-dimension discount/quantity filter on `lineorder` x `dwdate` | +| `q2.sql` | Q1.2 | | +| `q3.sql` | Q1.3 | | +| `q4.sql` | Q2.1 | Flight 2 — `part` and `supplier` restriction, grouped by year and brand | +| `q5.sql` | Q2.2 | | +| `q6.sql` | Q2.3 | | +| `q7.sql` | Q3.1 | Flight 3 — `customer` x `supplier` geography join, narrowing region to nation to city | +| `q8.sql` | Q3.2 | | +| `q9.sql` | Q3.3 | | +| `q10.sql` | Q3.4 | | +| `q11.sql` | Q4.1 | Flight 4 — all four dimensions, profit aggregation | +| `q12.sql` | Q4.2 | | +| `q13.sql` | Q4.3 | | + +Each file names its SSB query in a leading comment. + +## Schema notes + +The date dimension is registered as **`dwdate`**, not `date`: `date` is a reserved word in both +DataFusion's and DuckDB's parsers. This is the same rename the reference SSB load scripts apply, +for the same reason. Column names (`d_datekey`, `d_year`, ...) are unchanged. + +## Data + +There is no Rust SSB generator, and SSB is not derivable from TPC-H output — the dimension +cardinalities differ (`customer` is SF x 30k rather than SF x 150k, `supplier` SF x 2k rather than +SF x 10k, `part` is `200000 * floor(1 + log2(SF))`) and `dwdate` has no TPC-H analogue. So +[`src/ssb/datagen.rs`](../../src/ssb/datagen.rs) clones and builds the pinned reference C `dbgen`, +runs it, and converts its `.tbl` output to Parquet with the `duckdb` CLI. Generating data +therefore needs a C compiler, `cmake`, `git`, and the `duckdb` CLI on `PATH`; everything is +cached and idempotent after the first run. + +SSB has no official upstream, only a tree of unsynchronized `dbgen` forks, and they are not +interchangeable — the module docs record which fork is pinned and why. At SF 10 the source +Parquet is ~1.7 GB, dominated by `lineorder`'s 59,986,217 rows. + +## CI variant + +CI runs this suite at scale factor 10 from local NVMe as the `SSB SF=10 on NVME` PR comment, +comparing DataFusion and DuckDB over Parquet, Vortex, and vortex-compact files (plus a native +DuckDB baseline). Like Appian, it is part of the full SQL matrix rather than the quick PR one. + +## Running locally + +```bash +vx-bench run ssb --engine datafusion,duckdb --format parquet,vortex --opt scale-factor=10.0 +``` diff --git a/vortex-bench/sql/ssb/q1.sql b/vortex-bench/sql/ssb/q1.sql new file mode 100644 index 00000000000..225223947ae --- /dev/null +++ b/vortex-bench/sql/ssb/q1.sql @@ -0,0 +1,11 @@ +-- SSB Q1.1 +select + sum(lo_extendedprice * lo_discount) as revenue +from + lineorder, + dwdate +where + lo_orderdate = d_datekey + and d_year = 1993 + and lo_discount between 1 and 3 + and lo_quantity < 25 diff --git a/vortex-bench/sql/ssb/q10.sql b/vortex-bench/sql/ssb/q10.sql new file mode 100644 index 00000000000..070fb87f77f --- /dev/null +++ b/vortex-bench/sql/ssb/q10.sql @@ -0,0 +1,25 @@ +-- SSB Q3.4 +select + c_city, + s_city, + d_year, + sum(lo_revenue) as lo_revenue +from + customer, + lineorder, + supplier, + dwdate +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_orderdate = d_datekey + and (c_city = 'UNITED KI1' or c_city = 'UNITED KI5') + and (s_city = 'UNITED KI1' or s_city = 'UNITED KI5') + and d_yearmonth = 'Dec1997' +group by + c_city, + s_city, + d_year +order by + d_year asc, + lo_revenue desc diff --git a/vortex-bench/sql/ssb/q11.sql b/vortex-bench/sql/ssb/q11.sql new file mode 100644 index 00000000000..867c5e52a7f --- /dev/null +++ b/vortex-bench/sql/ssb/q11.sql @@ -0,0 +1,25 @@ +-- SSB Q4.1 +select + d_year, + c_nation, + sum(lo_revenue - lo_supplycost) as profit +from + dwdate, + customer, + supplier, + part, + lineorder +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_partkey = p_partkey + and lo_orderdate = d_datekey + and c_region = 'AMERICA' + and s_region = 'AMERICA' + and (p_mfgr = 'MFGR#1' or p_mfgr = 'MFGR#2') +group by + d_year, + c_nation +order by + d_year, + c_nation diff --git a/vortex-bench/sql/ssb/q12.sql b/vortex-bench/sql/ssb/q12.sql new file mode 100644 index 00000000000..7583c22f4d0 --- /dev/null +++ b/vortex-bench/sql/ssb/q12.sql @@ -0,0 +1,29 @@ +-- SSB Q4.2 +select + d_year, + s_nation, + p_category, + sum(lo_revenue - lo_supplycost) as profit +from + dwdate, + customer, + supplier, + part, + lineorder +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_partkey = p_partkey + and lo_orderdate = d_datekey + and c_region = 'AMERICA' + and s_region = 'AMERICA' + and (d_year = 1997 or d_year = 1998) + and (p_mfgr = 'MFGR#1' or p_mfgr = 'MFGR#2') +group by + d_year, + s_nation, + p_category +order by + d_year, + s_nation, + p_category diff --git a/vortex-bench/sql/ssb/q13.sql b/vortex-bench/sql/ssb/q13.sql new file mode 100644 index 00000000000..3a18e460cc6 --- /dev/null +++ b/vortex-bench/sql/ssb/q13.sql @@ -0,0 +1,28 @@ +-- SSB Q4.3 +select + d_year, + s_city, + p_brand1, + sum(lo_revenue - lo_supplycost) as profit +from + dwdate, + customer, + supplier, + part, + lineorder +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_partkey = p_partkey + and lo_orderdate = d_datekey + and s_nation = 'UNITED STATES' + and (d_year = 1997 or d_year = 1998) + and p_category = 'MFGR#14' +group by + d_year, + s_city, + p_brand1 +order by + d_year, + s_city, + p_brand1 diff --git a/vortex-bench/sql/ssb/q2.sql b/vortex-bench/sql/ssb/q2.sql new file mode 100644 index 00000000000..2603b75504a --- /dev/null +++ b/vortex-bench/sql/ssb/q2.sql @@ -0,0 +1,11 @@ +-- SSB Q1.2 +select + sum(lo_extendedprice * lo_discount) as revenue +from + lineorder, + dwdate +where + lo_orderdate = d_datekey + and d_yearmonthnum = 199401 + and lo_discount between 4 and 6 + and lo_quantity between 26 and 35 diff --git a/vortex-bench/sql/ssb/q3.sql b/vortex-bench/sql/ssb/q3.sql new file mode 100644 index 00000000000..da69afde545 --- /dev/null +++ b/vortex-bench/sql/ssb/q3.sql @@ -0,0 +1,12 @@ +-- SSB Q1.3 +select + sum(lo_extendedprice * lo_discount) as revenue +from + lineorder, + dwdate +where + lo_orderdate = d_datekey + and d_weeknuminyear = 6 + and d_year = 1994 + and lo_discount between 5 and 7 + and lo_quantity between 26 and 35 diff --git a/vortex-bench/sql/ssb/q4.sql b/vortex-bench/sql/ssb/q4.sql new file mode 100644 index 00000000000..f2b127b5642 --- /dev/null +++ b/vortex-bench/sql/ssb/q4.sql @@ -0,0 +1,22 @@ +-- SSB Q2.1 +select + sum(lo_revenue) as lo_revenue, + d_year, + p_brand1 +from + lineorder, + dwdate, + part, + supplier +where + lo_orderdate = d_datekey + and lo_partkey = p_partkey + and lo_suppkey = s_suppkey + and p_category = 'MFGR#12' + and s_region = 'AMERICA' +group by + d_year, + p_brand1 +order by + d_year, + p_brand1 diff --git a/vortex-bench/sql/ssb/q5.sql b/vortex-bench/sql/ssb/q5.sql new file mode 100644 index 00000000000..2b55898855f --- /dev/null +++ b/vortex-bench/sql/ssb/q5.sql @@ -0,0 +1,22 @@ +-- SSB Q2.2 +select + sum(lo_revenue) as lo_revenue, + d_year, + p_brand1 +from + lineorder, + dwdate, + part, + supplier +where + lo_orderdate = d_datekey + and lo_partkey = p_partkey + and lo_suppkey = s_suppkey + and p_brand1 between 'MFGR#2221' and 'MFGR#2228' + and s_region = 'ASIA' +group by + d_year, + p_brand1 +order by + d_year, + p_brand1 diff --git a/vortex-bench/sql/ssb/q6.sql b/vortex-bench/sql/ssb/q6.sql new file mode 100644 index 00000000000..39001dbb714 --- /dev/null +++ b/vortex-bench/sql/ssb/q6.sql @@ -0,0 +1,22 @@ +-- SSB Q2.3 +select + sum(lo_revenue) as lo_revenue, + d_year, + p_brand1 +from + lineorder, + dwdate, + part, + supplier +where + lo_orderdate = d_datekey + and lo_partkey = p_partkey + and lo_suppkey = s_suppkey + and p_brand1 = 'MFGR#2239' + and s_region = 'EUROPE' +group by + d_year, + p_brand1 +order by + d_year, + p_brand1 diff --git a/vortex-bench/sql/ssb/q7.sql b/vortex-bench/sql/ssb/q7.sql new file mode 100644 index 00000000000..62e9586eb0f --- /dev/null +++ b/vortex-bench/sql/ssb/q7.sql @@ -0,0 +1,26 @@ +-- SSB Q3.1 +select + c_nation, + s_nation, + d_year, + sum(lo_revenue) as lo_revenue +from + customer, + lineorder, + supplier, + dwdate +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_orderdate = d_datekey + and c_region = 'ASIA' + and s_region = 'ASIA' + and d_year >= 1992 + and d_year <= 1997 +group by + c_nation, + s_nation, + d_year +order by + d_year asc, + lo_revenue desc diff --git a/vortex-bench/sql/ssb/q8.sql b/vortex-bench/sql/ssb/q8.sql new file mode 100644 index 00000000000..5a9b8734a42 --- /dev/null +++ b/vortex-bench/sql/ssb/q8.sql @@ -0,0 +1,26 @@ +-- SSB Q3.2 +select + c_city, + s_city, + d_year, + sum(lo_revenue) as lo_revenue +from + customer, + lineorder, + supplier, + dwdate +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_orderdate = d_datekey + and c_nation = 'UNITED STATES' + and s_nation = 'UNITED STATES' + and d_year >= 1992 + and d_year <= 1997 +group by + c_city, + s_city, + d_year +order by + d_year asc, + lo_revenue desc diff --git a/vortex-bench/sql/ssb/q9.sql b/vortex-bench/sql/ssb/q9.sql new file mode 100644 index 00000000000..17837aef0b9 --- /dev/null +++ b/vortex-bench/sql/ssb/q9.sql @@ -0,0 +1,26 @@ +-- SSB Q3.3 +select + c_city, + s_city, + d_year, + sum(lo_revenue) as lo_revenue +from + customer, + lineorder, + supplier, + dwdate +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_orderdate = d_datekey + and (c_city = 'UNITED KI1' or c_city = 'UNITED KI5') + and (s_city = 'UNITED KI1' or s_city = 'UNITED KI5') + and d_year >= 1992 + and d_year <= 1997 +group by + c_city, + s_city, + d_year +order by + d_year asc, + lo_revenue desc diff --git a/vortex-bench/src/datasets/mod.rs b/vortex-bench/src/datasets/mod.rs index 7eb341edbb7..5f316c23a87 100644 --- a/vortex-bench/src/datasets/mod.rs +++ b/vortex-bench/src/datasets/mod.rs @@ -73,6 +73,8 @@ pub enum BenchmarkDataset { PublicBi { name: String }, #[serde(rename = "spatialbench")] SpatialBench { scale_factor: String }, + #[serde(rename = "ssb")] + Ssb { scale_factor: String }, #[serde(rename = "statpopgen")] StatPopGen { n_rows: u64 }, #[serde(rename = "polarsignals")] @@ -95,6 +97,7 @@ impl BenchmarkDataset { BenchmarkDataset::ClickBenchSorted => "clickbench-sorted", BenchmarkDataset::PublicBi { .. } => "public-bi", BenchmarkDataset::SpatialBench { .. } => "spatialbench", + BenchmarkDataset::Ssb { .. } => "ssb", BenchmarkDataset::StatPopGen { .. } => "statpopgen", BenchmarkDataset::PolarSignals { .. } => "polarsignals", BenchmarkDataset::Fineweb => "fineweb", @@ -119,6 +122,7 @@ impl Display for BenchmarkDataset { BenchmarkDataset::SpatialBench { scale_factor } => { write!(f, "spatialbench(sf={scale_factor})") } + BenchmarkDataset::Ssb { scale_factor } => write!(f, "ssb(sf={scale_factor})"), BenchmarkDataset::StatPopGen { n_rows } => write!(f, "statpopgen(n_rows={n_rows})"), BenchmarkDataset::PolarSignals { n_rows } => { write!(f, "polarsignals(n_rows={n_rows})") @@ -130,6 +134,10 @@ impl Display for BenchmarkDataset { } } +/// SSB registers `date` as `dwdate`, because `date` is a reserved word in both engines' +/// parsers. Kept in sync with `ssb::datagen::TABLES` by a test there. +pub const SSB_TABLES: &[&str] = &["customer", "supplier", "part", "dwdate", "lineorder"]; + const APPIAN_TABLES: &[&str] = &[ "addressview", "categoryview", @@ -179,6 +187,7 @@ impl BenchmarkDataset { BenchmarkDataset::ClickBench { .. } | BenchmarkDataset::ClickBenchSorted => &["hits"], BenchmarkDataset::PublicBi { .. } => todo!(), BenchmarkDataset::SpatialBench { .. } => &["trip", "building", "customer", "zone"], + BenchmarkDataset::Ssb { .. } => SSB_TABLES, BenchmarkDataset::StatPopGen { .. } => &["statpopgen"], BenchmarkDataset::PolarSignals { .. } => &["stacktraces"], BenchmarkDataset::Fineweb => &["fineweb"], diff --git a/vortex-bench/src/lib.rs b/vortex-bench/src/lib.rs index bc077a64719..f0ea9a02c20 100644 --- a/vortex-bench/src/lib.rs +++ b/vortex-bench/src/lib.rs @@ -36,6 +36,7 @@ use vortex::file::WriteStrategyBuilder; use vortex::utils::aliases::hash_map::HashMap; use crate::spatialbench::SpatialBenchBenchmark; +use crate::ssb::SsbBenchmark; use crate::vortex_queries::VortexBenchmark; pub mod appian; @@ -56,6 +57,7 @@ pub mod random_access; pub mod realnest; pub mod runner; pub mod spatialbench; +pub mod ssb; pub mod statpopgen; pub mod tpcds; pub mod tpch; @@ -277,6 +279,8 @@ pub enum BenchmarkArg { PublicBi, #[clap(name = "spatialbench")] SpatialBench, + #[clap(name = "ssb")] + Ssb, #[clap(name = "vortex")] VortexQueries, } @@ -351,6 +355,12 @@ pub fn create_benchmark(b: BenchmarkArg, opts: &Opts) -> anyhow::Result { + let scale_factor = opts.get(SCALE_FACTOR_KEY).unwrap_or(DEFAULT_SCALE_FACTOR); + let remote_data_dir = opts.get_as::(REMOTE_DATA_KEY); + let benchmark = SsbBenchmark::new(scale_factor.to_string(), remote_data_dir)?; + Ok(Box::new(benchmark) as _) + } BenchmarkArg::VortexQueries => { let mut benchmark = VortexBenchmark::new()?; if let Some(query) = opts.get("query") { diff --git a/vortex-bench/src/ssb/datagen.rs b/vortex-bench/src/ssb/datagen.rs new file mode 100644 index 00000000000..e458732b610 --- /dev/null +++ b/vortex-bench/src/ssb/datagen.rs @@ -0,0 +1,321 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Star Schema Benchmark data generation. +//! +//! There is no Rust SSB generator, and SSB is *not* derivable from TPC-H output — its +//! cardinalities differ (`customer` is SF x 30k rather than SF x 150k, `supplier` SF x 2k rather +//! than SF x 10k, `part` is `200000 * floor(1 + log2(SF))`, and `dwdate` is a 2557-row calendar +//! that TPC-H has no analogue for). So we build the reference C `dbgen` from source, run it, and +//! convert its pipe-delimited `.tbl` output into Parquet with the `duckdb` CLI — the same +//! shell-out the Appian suite uses. +//! +//! ## Which `dbgen` +//! +//! SSB has no official upstream, only a tree of unsynchronized `dbgen` forks. We pin +//! [`eyalroz/ssb-dbgen`][fork], which unifies them and is the fork ClickHouse's own SSB docs +//! use. The alternatives are not interchangeable: `lemire/StarSchemaBenchmark`, for instance, +//! carries a `/*bug!*/`-annotated `gen_city()` that draws from random stream 98 when +//! `MAX_STREAM` is 47 — `UnifInt()` clamps the stream for the *value* but `dss_random()` then +//! increments `Seed[98].usage` out of bounds, corrupting an adjacent global. That is a `SIGBUS` +//! at SF 10 and, below the crash threshold, data whose contents depend on global layout. The +//! pinned fork fixes it properly (a real `P_CITY_SD` stream, `MAX_STREAM` raised to 49, and an +//! assert in `dss_random()`), and also emits the full 2557-day calendar rather than 2556. +//! +//! ## Prerequisites +//! +//! A C compiler, `cmake`, `git`, and the `duckdb` CLI (the last already required by Appian). +//! +//! [fork]: https://github.com/eyalroz/ssb-dbgen + +use std::fs; +use std::path::Path; +use std::path::PathBuf; +use std::process::Command; + +use anyhow::Context; +use anyhow::bail; +use itertools::Itertools; +use tracing::info; + +use crate::Format; +use crate::utils::file::data_dir; + +/// Maintained fork of the SSB `dbgen` published alongside O'Neil, O'Neil & Chen, +/// *The Star Schema Benchmark, Revision 3* (2009). See the module docs for why this fork. +const DBGEN_REPO: &str = "https://github.com/eyalroz/ssb-dbgen.git"; + +/// Pinned so the generated data is reproducible across runs and machines. +const DBGEN_REV: &str = "ae1e254aa4d603d8ef1f44078e5abed011634b23"; + +/// One SSB table: the registered name, the `.tbl` `dbgen` writes it to, and its columns as +/// `(name, DuckDB type)` in file order. +pub struct Table { + /// Name registered with the query engines, and the Parquet file stem. + pub name: &'static str, + /// Stem of the `.tbl` file `dbgen` emits. Only `dwdate` differs from [`Table::name`]. + tbl_stem: &'static str, + columns: &'static [(&'static str, &'static str)], +} + +impl Table { + /// `read_csv` column spec, plus a trailing `dummy` for the line-terminating `|` that every + /// `.tbl` row carries. The `dummy` is dropped again by [`Table::copy_stmt`]. + fn read_csv_columns(&self) -> String { + self.columns + .iter() + .map(|(name, ty)| format!("'{name}':'{ty}'")) + .chain(std::iter::once("'dummy':'VARCHAR'".to_string())) + .join(",") + } + + fn copy_stmt(&self, tbl_dir: &Path, parquet_dir: &Path) -> String { + let tbl = tbl_dir.join(format!("{}.tbl", self.tbl_stem)); + let parquet = parquet_dir.join(format!("{}.parquet", self.name)); + format!( + "COPY (SELECT * EXCLUDE (dummy) FROM read_csv('{}', delim='|', header=false, \ + columns={{{}}})) TO '{}' (FORMAT PARQUET);\n", + tbl.display(), + self.read_csv_columns(), + parquet.display(), + ) + } +} + +/// The five SSB tables, typed to match the reference DDL in the SSB paper. Every numeric column +/// is a 32-bit `INTEGER`: at SF 100 the widest values are `lo_orderkey` (6e8) and +/// `lo_ordtotalprice` (~5e7), both comfortably inside `i32`. +pub const TABLES: &[Table] = &[ + Table { + name: "customer", + tbl_stem: "customer", + columns: &[ + ("c_custkey", "INTEGER"), + ("c_name", "VARCHAR"), + ("c_address", "VARCHAR"), + ("c_city", "VARCHAR"), + ("c_nation", "VARCHAR"), + ("c_region", "VARCHAR"), + ("c_phone", "VARCHAR"), + ("c_mktsegment", "VARCHAR"), + ], + }, + Table { + name: "supplier", + tbl_stem: "supplier", + columns: &[ + ("s_suppkey", "INTEGER"), + ("s_name", "VARCHAR"), + ("s_address", "VARCHAR"), + ("s_city", "VARCHAR"), + ("s_nation", "VARCHAR"), + ("s_region", "VARCHAR"), + ("s_phone", "VARCHAR"), + ], + }, + Table { + name: "part", + tbl_stem: "part", + columns: &[ + ("p_partkey", "INTEGER"), + ("p_name", "VARCHAR"), + ("p_mfgr", "VARCHAR"), + ("p_category", "VARCHAR"), + ("p_brand1", "VARCHAR"), + ("p_color", "VARCHAR"), + ("p_type", "VARCHAR"), + ("p_size", "INTEGER"), + ("p_container", "VARCHAR"), + ], + }, + // `date` is a reserved word in both DataFusion's and DuckDB's parsers, so the table is + // registered as `dwdate` — the name the reference SSB load scripts use for the same reason. + Table { + name: "dwdate", + tbl_stem: "date", + columns: &[ + ("d_datekey", "INTEGER"), + ("d_date", "VARCHAR"), + ("d_dayofweek", "VARCHAR"), + ("d_month", "VARCHAR"), + ("d_year", "INTEGER"), + ("d_yearmonthnum", "INTEGER"), + ("d_yearmonth", "VARCHAR"), + ("d_daynuminweek", "INTEGER"), + ("d_daynuminmonth", "INTEGER"), + ("d_daynuminyear", "INTEGER"), + ("d_monthnuminyear", "INTEGER"), + ("d_weeknuminyear", "INTEGER"), + ("d_sellingseason", "VARCHAR"), + ("d_lastdayinweekfl", "INTEGER"), + ("d_lastdayinmonthfl", "INTEGER"), + ("d_holidayfl", "INTEGER"), + ("d_weekdayfl", "INTEGER"), + ], + }, + Table { + name: "lineorder", + tbl_stem: "lineorder", + columns: &[ + ("lo_orderkey", "INTEGER"), + ("lo_linenumber", "INTEGER"), + ("lo_custkey", "INTEGER"), + ("lo_partkey", "INTEGER"), + ("lo_suppkey", "INTEGER"), + ("lo_orderdate", "INTEGER"), + ("lo_orderpriority", "VARCHAR"), + ("lo_shippriority", "VARCHAR"), + ("lo_quantity", "INTEGER"), + ("lo_extendedprice", "INTEGER"), + ("lo_ordtotalprice", "INTEGER"), + ("lo_discount", "INTEGER"), + ("lo_revenue", "INTEGER"), + ("lo_supplycost", "INTEGER"), + ("lo_tax", "INTEGER"), + ("lo_commitdate", "INTEGER"), + ("lo_shipmode", "VARCHAR"), + ], + }, +]; + +/// Generate the SSB Parquet base data for `scale_factor` under `base_dir/parquet/`. +/// +/// Idempotent: returns immediately once every table's Parquet is in place. The `.tbl` +/// intermediates are deleted after conversion (SF 10 alone is ~6.5 GB of text). +pub fn generate_tables(scale_factor: &str, base_dir: &Path) -> anyhow::Result<()> { + let parquet_dir = base_dir.join(Format::Parquet.name()); + fs::create_dir_all(&parquet_dir)?; + + if TABLES + .iter() + .all(|t| parquet_dir.join(format!("{}.parquet", t.name)).exists()) + { + info!( + "ssb: {} Parquet shards already present in {}", + TABLES.len(), + parquet_dir.display(), + ); + return Ok(()); + } + + let tbl_dir = base_dir.join("tbl"); + write_tbl_files(scale_factor, &tbl_dir)?; + convert_tbl_to_parquet(&tbl_dir, &parquet_dir)?; + + fs::remove_dir_all(&tbl_dir)?; + info!( + "ssb base data generated in {} ({} Parquet shards)", + parquet_dir.display(), + TABLES.len(), + ); + Ok(()) +} + +/// Run `dbgen` into `tbl_dir`, then verify it actually produced every table. +fn write_tbl_files(scale_factor: &str, tbl_dir: &Path) -> anyhow::Result<()> { + let dbgen = build_dbgen()?; + fs::create_dir_all(tbl_dir)?; + + // `dbgen` reads `dists.dss` from `DSS_CONFIG` and writes its `.tbl` files into `DSS_PATH`, + // so neither needs to be the process working directory. Omitting `-T` generates every + // table (this fork has no `-T a`). + info!(scale_factor, "ssb: generating .tbl files with dbgen"); + run(Command::new(&dbgen) + .env("DSS_CONFIG", dbgen_dir()) + .env("DSS_PATH", tbl_dir) + .args(["-s", scale_factor, "-f"]))?; + + // `dbgen` exits 0 even when it rejects its arguments, so check its output rather than + // its status. + for table in TABLES { + let tbl = tbl_dir.join(format!("{}.tbl", table.tbl_stem)); + if !tbl.exists() { + bail!("ssb: dbgen did not produce {}", tbl.display()); + } + } + Ok(()) +} + +/// Convert every `.tbl` into its Parquet counterpart in one `duckdb` invocation. +fn convert_tbl_to_parquet(tbl_dir: &Path, parquet_dir: &Path) -> anyhow::Result<()> { + info!("ssb: converting .tbl files to Parquet"); + let script = TABLES + .iter() + .map(|t| t.copy_stmt(tbl_dir, parquet_dir)) + .join(""); + run(Command::new("duckdb").arg("-c").arg(&script)) +} + +/// Checkout of the pinned upstream generator, shared by every scale factor. +fn dbgen_dir() -> PathBuf { + data_dir().join("ssb-dbgen") +} + +/// Clone (once) and build the pinned `dbgen`, returning the path to the binary. Idempotent. +fn build_dbgen() -> anyhow::Result { + let dir = dbgen_dir(); + let build_dir = dir.join("build"); + let binary = build_dir.join("dbgen"); + if binary.exists() { + return Ok(binary); + } + + if !dir.join(".git").exists() { + if let Some(parent) = dir.parent() { + fs::create_dir_all(parent)?; + } + info!("ssb: cloning {DBGEN_REPO} at {DBGEN_REV}"); + run(Command::new("git").args(["clone", DBGEN_REPO]).arg(&dir))?; + } + run(Command::new("git") + .arg("-C") + .arg(&dir) + .args(["checkout", "--quiet", DBGEN_REV]))?; + + info!("ssb: building dbgen"); + run(Command::new("cmake") + .arg("-S") + .arg(&dir) + .arg("-B") + .arg(&build_dir) + .arg("-DCMAKE_BUILD_TYPE=Release"))?; + run(Command::new("cmake") + .arg("--build") + .arg(&build_dir) + .args(["--target", "dbgen"]))?; + + if !binary.exists() { + bail!("ssb: cmake succeeded but {} is missing", binary.display()); + } + Ok(binary) +} + +fn run(command: &mut Command) -> anyhow::Result<()> { + let program = format!("{:?}", command.get_program()); + let output = command + .output() + .with_context(|| format!("ssb: failed to spawn {program}"))?; + if !output.status.success() { + bail!( + "ssb: {program} failed ({}): stdout={:?} stderr={:?}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::TABLES; + use crate::datasets::SSB_TABLES; + + /// `datasets::SSB_TABLES` drives table registration while [`TABLES`] drives generation; a + /// divergence would silently register a table nothing writes (or vice versa). + #[test] + fn table_lists_agree() { + let generated = TABLES.iter().map(|t| t.name).collect::>(); + assert_eq!(generated.as_slice(), SSB_TABLES); + } +} diff --git a/vortex-bench/src/ssb/mod.rs b/vortex-bench/src/ssb/mod.rs new file mode 100644 index 00000000000..eb38ac6a05c --- /dev/null +++ b/vortex-bench/src/ssb/mod.rs @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Star Schema Benchmark (SSB). +//! +//! A denormalized redesign of TPC-H into a classic star schema: one wide `lineorder` fact table +//! joined against four dimensions (`customer`, `supplier`, `part`, `dwdate`). The 13 queries are +//! organized into four "flights" of progressively more selective dimension filters, which makes +//! the suite a direct test of filter pushdown, zone-map pruning, and dimension-join throughput — +//! the axes on which a columnar format is supposed to win. +//! +//! Data generation lives in [`datagen`]. + +use std::fs; +use std::path::Path; + +use glob::Pattern; +use url::Url; +use vortex::error::VortexExpect; + +use crate::Benchmark; +use crate::BenchmarkDataset; +use crate::Format; +use crate::TableSpec; +use crate::datasets::SSB_TABLES; +use crate::utils::file::resolve_data_url; + +pub mod datagen; + +/// The 13 SSB queries, stored as `q1.sql` ... `q13.sql`. The framework keys queries on a plain +/// index, so the flight-and-query numbering from the paper (Q1.1 ... Q4.3) maps onto 1 ... 13 in +/// order; each file names its SSB query in a leading comment, and `sql/ssb/README.md` carries the +/// full table. +pub fn ssb_queries() -> impl Iterator { + (1..=13).map(|q| (q, ssb_query(q))) +} + +fn ssb_query(query_idx: usize) -> String { + let path = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("sql") + .join("ssb") + .join(format!("q{query_idx}")) + .with_extension("sql"); + fs::read_to_string(path).vortex_expect("cannot load ssb query from file") +} + +/// Benchmark over the [Star Schema Benchmark][ssb]. +/// +/// [ssb]: https://www.cs.umb.edu/~poneil/StarSchemaB.PDF +pub struct SsbBenchmark { + pub scale_factor: String, + pub data_url: Url, +} + +impl SsbBenchmark { + pub fn new(scale_factor: String, use_remote_data_dir: Option) -> anyhow::Result { + Ok(Self { + data_url: resolve_data_url( + use_remote_data_dir.as_deref(), + &format!("ssb/{scale_factor}"), + )?, + scale_factor, + }) + } +} + +#[async_trait::async_trait] +impl Benchmark for SsbBenchmark { + fn doc_path(&self) -> &'static str { + "vortex-bench/sql/ssb/README.md" + } + + fn queries(&self) -> anyhow::Result> { + Ok(ssb_queries().collect()) + } + + async fn generate_base_data(&self) -> anyhow::Result<()> { + if self.data_url.scheme() != "file" { + return Ok(()); + } + let base_dir = self.data_url.to_file_path().map_err(|()| { + anyhow::anyhow!( + "Failed to convert data URL to filesystem path - ensure data_url uses 'file://' scheme" + ) + })?; + datagen::generate_tables(&self.scale_factor, &base_dir) + } + + fn expected_row_counts(&self) -> Option> { + // Indexed by `query_idx` (1-based), so index 0 is a dummy and Q1's count is at index 1 + // (TPC-H convention). Only the scale factors CI runs are validated; anything else + // returns `None`. Measured with DuckDB over the generated Parquet, and consistent with + // the group cardinalities the SSB schema implies — e.g. Q4.3 at SF 10 is 2 years x 10 + // US cities x 40 `MFGR#14` brands = 800. + match self.scale_factor.as_str() { + "1.0" => Some(vec![0, 1, 1, 1, 280, 56, 7, 150, 600, 24, 3, 35, 100, 725]), + "10.0" => Some(vec![0, 1, 1, 1, 280, 56, 7, 150, 600, 24, 4, 35, 100, 800]), + _ => None, + } + } + + fn dataset(&self) -> BenchmarkDataset { + BenchmarkDataset::Ssb { + scale_factor: self.scale_factor.clone(), + } + } + + fn dataset_name(&self) -> &str { + "ssb" + } + + fn dataset_display(&self) -> String { + format!("ssb(sf={})", self.scale_factor) + } + + fn data_url(&self) -> &Url { + &self.data_url + } + + fn table_specs(&self) -> Vec { + SSB_TABLES + .iter() + .map(|name| TableSpec::new(name, None)) + .collect() + } + + /// Scope each table to its own file; the default globs every file in the format dir, which + /// would conflate the five schemas. + #[expect(clippy::expect_used)] + fn pattern(&self, table_name: &str, format: Format) -> Option { + Some( + format!("{}.{}", table_name, format.ext()) + .parse() + .expect("valid glob pattern"), + ) + } +} diff --git a/vortex-bench/src/v3.rs b/vortex-bench/src/v3.rs index 735f0f4dc88..337993c9dce 100644 --- a/vortex-bench/src/v3.rs +++ b/vortex-bench/src/v3.rs @@ -281,6 +281,7 @@ fn canonical_tpc_scale_factor(scale_factor: &str) -> String { /// | `Appian` | `appian` | `None` | `None` | Static dataset; no scale factor. | /// | `PublicBi { name }` | `public-bi` | dataset name (e.g. `cms-provider`) | `None` | Sub-dataset name lives in `dataset_variant`. | /// | `SpatialBench { scale_factor }` | `spatialbench` | `None` | SF as string | Same canonicalization as TPC-H; no historical v2 records to merge with. | +/// | `Ssb { scale_factor }` | `ssb` | `None` | SF as string | Same canonicalization as TPC-H; no historical v2 records to merge with. | /// | `VortexQueries` | `vortex` | `None` | `None` | Own microbenchmarks | pub fn benchmark_dataset_dims(d: &BenchmarkDataset) -> (String, Option, Option) { match d { @@ -312,6 +313,11 @@ pub fn benchmark_dataset_dims(d: &BenchmarkDataset) -> (String, Option, None, Some(canonical_tpc_scale_factor(scale_factor)), ), + BenchmarkDataset::Ssb { scale_factor } => ( + "ssb".to_string(), + None, + Some(canonical_tpc_scale_factor(scale_factor)), + ), BenchmarkDataset::StatPopGen { .. } => ("statpopgen".to_string(), None, None), BenchmarkDataset::PolarSignals { .. } => ("polarsignals".to_string(), None, None), BenchmarkDataset::Fineweb => ("fineweb".to_string(), None, None), @@ -761,6 +767,17 @@ mod tests { assert_eq!(size.dataset, "cmsprovider"); } + #[test] + fn ssb_dims_carry_a_canonicalized_scale_factor() { + let (dataset, variant, scale_factor) = benchmark_dataset_dims(&BenchmarkDataset::Ssb { + scale_factor: "10.0".to_string(), + }); + + assert_eq!(dataset, "ssb"); + assert_eq!(variant, None); + assert_eq!(scale_factor.as_deref(), Some("10")); + } + #[test] fn tpc_scale_factors_are_canonicalized_for_query_dims() { // Bench-orchestrator passes raw TPC scale factors like `"1.0"` and `"100.0"`,