diff --git a/README.md b/README.md index 9bbd5765..f48e88d6 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,9 @@ In this case, `hll` takes a `lg_k` which describes the size of register list. **`--zipf-s`:** Some distribution requires parameter, like `zipf-s`. `zipf-s` means skewness, a parameter for zipf distribution. +`--dataset pareto` instead takes `--pareto-alpha` (shape) and `--pareto-scale` +(minimum value, default 1.0); it is unbounded, so it needs `--dtype f64` (or +`i64`, which floors each draw) and ignores `--cardinality`. **`--cardinality`:** Another parameter required by zipf distribution. `cardinality` means the size of key-space of this synthesized zipf dataset. diff --git a/aqpbm-cli/src/cli.rs b/aqpbm-cli/src/cli.rs index b3f65789..743885e2 100644 --- a/aqpbm-cli/src/cli.rs +++ b/aqpbm-cli/src/cli.rs @@ -89,7 +89,7 @@ pub struct SketchbenchArgs { /// A *list* of specs is a multi-column stream, which only `hydra-*` targets take. #[arg(long, help_heading = "Dataset")] pub spec: Option, - /// Inline shape: "uniform" or "zipf". Ignored when `--spec` is + /// Inline shape: "uniform", "zipf" or "pareto". Ignored when `--spec` is /// set. Required unless `--spec` or `--list-impls` is given: a run either /// names its shape or names its file, never falls back to one unasked. #[arg( @@ -106,7 +106,8 @@ pub struct SketchbenchArgs { help_heading = "Dataset" )] pub size: Option, - /// Cardinality (uniform: max key; zipf: key-space size). Required unless + /// Cardinality (uniform: max key; zipf: key-space size; pareto: ignored, + /// it is unbounded). Required unless /// `--spec` or `--list-impls` is given. #[arg( long, @@ -118,6 +119,13 @@ pub struct SketchbenchArgs { /// otherwise. #[arg(long, required_if_eq("dataset", "zipf"), help_heading = "Dataset")] pub zipf_s: Option, + /// Pareto shape `alpha`. Required when `--dataset pareto` is given; + /// ignored otherwise. + #[arg(long, required_if_eq("dataset", "pareto"), help_heading = "Dataset")] + pub pareto_alpha: Option, + /// Pareto scale (the minimum value). Only read by `--dataset pareto`. + #[arg(long, default_value_t = 1.0, help_heading = "Dataset")] + pub pareto_scale: f64, /// Item type the value column is generated at: `i64`, `u64`, `f64` or /// `string` /// Required unless `--list-impls` is given: read regardless of `--spec`. diff --git a/aqpbm-cli/src/main.rs b/aqpbm-cli/src/main.rs index 6e20711f..8334d4dd 100644 --- a/aqpbm-cli/src/main.rs +++ b/aqpbm-cli/src/main.rs @@ -36,8 +36,8 @@ use aqpbm_core::measure::MeasureConfig; use aqpbm_core::measure::MIN_MERGE_SHARDS; use aqpbm_core::metrics::{Metric, MetricsMask, Operation, OperationMask}; use aqpbm_datagen::{ - ColumnSpec, DataDistribution, GeneratedTable, StringOpts, TableDescription, UniformParameter, - ZipfParameter, RULE_NONE, + ColumnSpec, DataDistribution, GeneratedTable, ParetoParameter, StringOpts, TableDescription, + UniformParameter, ZipfParameter, RULE_NONE, }; use clap::Parser; use sketch_bench::params::ParamSet; @@ -205,7 +205,25 @@ fn dataset_spec(args: &SketchbenchArgs) -> Result { population_size: cardinality, seed: args.seed, }), - other => bail!("unknown dataset shape: {other} (expected uniform|zipf, or use --spec)"), + // Unbounded and continuous, so `--cardinality` has nothing to name. An + // f64 column keeps the draws as drawn; an i64 column floors them (every + // draw is >= scale > 0, so the renderer's truncation is a floor), which + // is what the i64-only exact quantile baseline needs. + "pareto" => { + if !matches!(args.dtype.as_deref(), Some("f64" | "i64")) { + bail!("--dataset pareto needs --dtype f64 or i64 (i64 floors each draw)"); + } + DataDistribution::Pareto(ParetoParameter { + alpha: args + .pareto_alpha + .expect("clap requires --pareto-alpha when --dataset pareto"), + scale: args.pareto_scale, + seed: args.seed, + }) + } + other => { + bail!("unknown dataset shape: {other} (expected uniform|zipf|pareto, or use --spec)") + } }; // Only the rows that ingest text read this. Left `None` at the defaults // so a run that did not ask for the axis keeps the descriptor — and so diff --git a/aqpbm-core/schema/merged_record.schema.json b/aqpbm-core/schema/merged_record.schema.json index 617315c1..3fd038c0 100644 --- a/aqpbm-core/schema/merged_record.schema.json +++ b/aqpbm-core/schema/merged_record.schema.json @@ -344,6 +344,38 @@ ], "x-schema-version": 5, "$defs": { + "BurstSpec": { + "type": "object", + "properties": { + "burst_intervals": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "extra_fraction": { + "description": "Additional rows per selected interval, relative to its base size.", + "type": "number", + "format": "double" + }, + "interval_rows": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "seed": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "interval_rows", + "burst_intervals", + "extra_fraction", + "seed" + ] + }, "ColumnSpec": { "description": "One column of a [`crate::TableDescription`].", "type": "object", @@ -453,6 +485,19 @@ "required": [ "kind" ] + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "pareto" + } + }, + "$ref": "#/$defs/ParetoParameter", + "required": [ + "kind" + ] } ] }, @@ -607,6 +652,30 @@ "seed" ] }, + "ParetoParameter": { + "description": "Pareto: heavy-tailed, `P(X > x) = (scale / x)^alpha` for `x >= scale`.\nUnbounded above, so like Normal it has no domain.", + "type": "object", + "properties": { + "alpha": { + "type": "number", + "format": "double" + }, + "scale": { + "type": "number", + "format": "double" + }, + "seed": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "alpha", + "scale", + "seed" + ] + }, "RunStats": { "description": "Aggregate across N samples: mean / stddev / optional 95% CI, plus the raw\nsamples the aggregate was computed from.", "type": "object", @@ -781,6 +850,16 @@ "synthetic": { "type": "object", "properties": { + "burst": { + "anyOf": [ + { + "$ref": "#/$defs/BurstSpec" + }, + { + "type": "null" + } + ] + }, "description": { "$ref": "#/$defs/TableDescription" } diff --git a/aqpbm-core/src/erp.rs b/aqpbm-core/src/erp.rs index c023d317..556fbb18 100644 --- a/aqpbm-core/src/erp.rs +++ b/aqpbm-core/src/erp.rs @@ -79,6 +79,13 @@ fn distribution_descriptor(workload: &WorkloadDescription) -> serde_json::Value ("standard_deviation".into(), value.standard_deviation), ]), ), + DataDistribution::Pareto(value) => ( + "pareto", + BTreeMap::from([ + ("alpha".into(), value.alpha), + ("scale".into(), value.scale), + ]), + ), }; Some(ErpDataShape { cardinality, diff --git a/aqpbm-core/src/lib.rs b/aqpbm-core/src/lib.rs index be224076..da048d69 100644 --- a/aqpbm-core/src/lib.rs +++ b/aqpbm-core/src/lib.rs @@ -16,8 +16,8 @@ pub mod metrics; pub use accuracy::GroundTruth; pub use aqpbm_datagen::{ ColumnData, ColumnItem, ColumnSpec, DataDistribution, DataGenError, GeneratedTable, - NormalParameter, StringOpts, TableDescription, UniformParameter, ZipfParameter, - RULE_MONOTONIC_INCREASE, RULE_NONE, + NormalParameter, ParetoParameter, StringOpts, TableDescription, UniformParameter, + ZipfParameter, RULE_MONOTONIC_INCREASE, RULE_NONE, }; pub use atomic_costs::{reduce_all, reduce_one, AtomicCostEntry, AtomicCostTable, SkipReason}; pub use benchmark_result::{ diff --git a/aqpbm-datagen/src/dist.rs b/aqpbm-datagen/src/dist.rs index 23855f0f..48596961 100644 --- a/aqpbm-datagen/src/dist.rs +++ b/aqpbm-datagen/src/dist.rs @@ -2,7 +2,7 @@ //! fixes it — so two columns are independent unless given the same one. Every //! distribution draws an `f64`, so [`crate::column`]'s pipeline is written once. -use rand_distr::{Distribution as _, Normal, Uniform, Zipf}; +use rand_distr::{Distribution as _, Normal, Pareto, Uniform, Zipf}; use rand_xoshiro::Xoshiro256PlusPlus; use serde::{Deserialize, Serialize}; @@ -40,6 +40,16 @@ pub struct NormalParameter { pub seed: u64, } +/// Pareto: heavy-tailed, `P(X > x) = (scale / x)^alpha` for `x >= scale`. +/// Unbounded above, so like Normal it has no domain. +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ParetoParameter { + pub alpha: f64, + pub scale: f64, + pub seed: u64, +} + /// A column's distribution. Serialises internally tagged, so a spec file reads /// `distribution: {kind: zipf, skewness: 1.1, population_size: 200, seed: 1}`. #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] @@ -49,6 +59,7 @@ pub enum DataDistribution { Zipf(ZipfParameter), Uniform(UniformParameter), Normal(NormalParameter), + Pareto(ParetoParameter), } /// The half-open span a bounded distribution draws over: the lowest value it can @@ -68,6 +79,7 @@ impl DataDistribution { DataDistribution::Zipf(_) => "zipf", DataDistribution::Uniform(_) => "uniform", DataDistribution::Normal(_) => "normal", + DataDistribution::Pareto(_) => "pareto", } } @@ -77,10 +89,11 @@ impl DataDistribution { DataDistribution::Zipf(p) => p.seed, DataDistribution::Uniform(p) => p.seed, DataDistribution::Normal(p) => p.seed, + DataDistribution::Pareto(p) => p.seed, } } - /// The bounded span, or `None` for Normal — which is unbounded, so a + /// The bounded span, or `None` for Normal and Pareto — which are unbounded, so a /// `cardinality` over it would be a claim this crate cannot keep. pub fn domain(&self) -> Option { match self { @@ -93,7 +106,7 @@ impl DataDistribution { lower: p.lower_bound, size: (p.upper_bound - p.lower_bound) as u64, }), - DataDistribution::Normal(_) => None, + DataDistribution::Normal(_) | DataDistribution::Pareto(_) => None, } } @@ -139,6 +152,18 @@ impl DataDistribution { .map_err(|e| bad(format!("normal: {e}")))?, )) } + DataDistribution::Pareto(p) => { + if !p.alpha.is_finite() || p.alpha <= 0.0 || !p.scale.is_finite() || p.scale <= 0.0 + { + return Err(bad(format!( + "pareto: alpha and scale must be finite and > 0, got alpha {} scale {}", + p.alpha, p.scale + ))); + } + Ok(Sampler::Pareto( + Pareto::new(p.scale, p.alpha).map_err(|e| bad(format!("pareto: {e}")))?, + )) + } } } } @@ -149,6 +174,7 @@ pub enum Sampler { Zipf(Zipf), Uniform(Uniform), Normal(Normal), + Pareto(Pareto), } impl Sampler { @@ -158,6 +184,7 @@ impl Sampler { Sampler::Zipf(d) => d.sample(rng), Sampler::Uniform(d) => d.sample(rng), Sampler::Normal(d) => d.sample(rng), + Sampler::Pareto(d) => d.sample(rng), } } } diff --git a/aqpbm-datagen/src/lib.rs b/aqpbm-datagen/src/lib.rs index 10004b9c..867131a3 100644 --- a/aqpbm-datagen/src/lib.rs +++ b/aqpbm-datagen/src/lib.rs @@ -13,7 +13,8 @@ pub mod value; pub use burst::{inject_interval_bursts, BurstSpec}; pub use column::{ColumnSpec, DATA_TYPES}; pub use dist::{ - DataDistribution, Domain, NormalParameter, Sampler, UniformParameter, ZipfParameter, + DataDistribution, Domain, NormalParameter, ParetoParameter, Sampler, UniformParameter, + ZipfParameter, }; pub use error::DataGenError; pub use rule::{RULE_MASK_ALL, RULE_MONOTONIC_INCREASE, RULE_NONE}; @@ -48,6 +49,10 @@ mod tests { }) } + fn pareto(alpha: f64, scale: f64, seed: u64) -> DataDistribution { + DataDistribution::Pareto(ParetoParameter { alpha, scale, seed }) + } + fn column(distribution: DataDistribution, data_type: &str) -> ColumnSpec { ColumnSpec { distribution, @@ -126,6 +131,45 @@ mod tests { assert!((mean - 1000.0).abs() < 0.2, "mean {mean} not near 1000"); } + /// The MLE of the shape, `n / sum(ln(x / scale))`, recovers `alpha` from a + /// large fixed-seed sample. + #[test] + fn pareto_mle_recovers_alpha() { + let (alpha, scale) = (1.5, 2.0); + let t = one(column(pareto(alpha, scale, 13), "f64"), 200_000) + .generate() + .unwrap(); + let values = t.into_column(0).unwrap().into_f64().unwrap(); + assert!(values.iter().all(|&x| x >= scale)); + let log_sum: f64 = values.iter().map(|x| (x / scale).ln()).sum(); + let alpha_hat = values.len() as f64 / log_sum; + assert!( + (alpha_hat - alpha).abs() / alpha < 0.05, + "alpha_hat {alpha_hat} not within 5% of {alpha}" + ); + } + + /// An i64 column is the f64 draws floored: every draw is >= scale > 0, so + /// the renderer's `as` truncation is a floor. The i64-only exact quantile + /// baseline scores a Pareto stream through this. + #[test] + fn pareto_i64_floors_the_f64_draws() { + let draw = |data_type| { + one(column(pareto(1.1, 1000.0, 7), data_type), 50_000) + .generate() + .unwrap() + .into_column(0) + .unwrap() + }; + let floats = draw("f64").into_f64().unwrap(); + let ints = draw("i64").into_i64().unwrap(); + assert!(ints.iter().all(|&v| v >= 1000)); + assert!(ints + .iter() + .zip(&floats) + .all(|(&i, f)| i == f.floor() as i64)); + } + // ---------- rendering ---------- /// `data_type` picks the variant, and nothing else about the description diff --git a/docs/aqpbm-datagen.md b/docs/aqpbm-datagen.md index 630dd5f6..fcccfa78 100644 --- a/docs/aqpbm-datagen.md +++ b/docs/aqpbm-datagen.md @@ -41,6 +41,7 @@ enum DataDistribution { Zipf(ZipfParameter), Uniform(UniformParameter), Normal(NormalParameter), + Pareto(ParetoParameter), } struct ZipfParameter { @@ -60,6 +61,12 @@ struct NormalParameter { pub standard_deviation: f64, pub seed: u64, } + +struct ParetoParameter { + pub alpha: f64, // shape + pub scale: f64, // minimum value + pub seed: u64, +} ``` `special_rule` bit map contains the following at this moment: diff --git a/docs/figures/saturation/grid_1e9/README.md b/docs/figures/saturation/grid_1e9/README.md new file mode 100644 index 00000000..219dd09b --- /dev/null +++ b/docs/figures/saturation/grid_1e9/README.md @@ -0,0 +1,8 @@ +# Error vs N up to 1e9 + +Drawn by `scripts/plot_saturation_curves.py` from `out_1e9/saturation_curve.csv` +(the 1e9 subset in `docs/saturation_study.md`): one config per sketch from the +grid (rows=3 cols=1024 for CMS/CountSketch/top-k, lg_k=12, k=200, alpha=0.01), +N from 1e3 to 1e9, 3 seeds, Pareto values floored to i64 (scale 1000). Panels, +bands, bounds and N_sat markers read as in `../reduced_run/README.md`; the +1e9 findings are summarised in the doc's "Results (1e9 subset)" section. diff --git a/docs/figures/saturation/grid_1e9/error_vs_N__cms-fastpath-vector2d.png b/docs/figures/saturation/grid_1e9/error_vs_N__cms-fastpath-vector2d.png new file mode 100644 index 00000000..7d5ab207 Binary files /dev/null and b/docs/figures/saturation/grid_1e9/error_vs_N__cms-fastpath-vector2d.png differ diff --git a/docs/figures/saturation/grid_1e9/error_vs_N__cms-heap-topk-fastpath-vector2d.png b/docs/figures/saturation/grid_1e9/error_vs_N__cms-heap-topk-fastpath-vector2d.png new file mode 100644 index 00000000..5d4d98fc Binary files /dev/null and b/docs/figures/saturation/grid_1e9/error_vs_N__cms-heap-topk-fastpath-vector2d.png differ diff --git a/docs/figures/saturation/grid_1e9/error_vs_N__countsketch-fastpath-vector2d.png b/docs/figures/saturation/grid_1e9/error_vs_N__countsketch-fastpath-vector2d.png new file mode 100644 index 00000000..080d5232 Binary files /dev/null and b/docs/figures/saturation/grid_1e9/error_vs_N__countsketch-fastpath-vector2d.png differ diff --git a/docs/figures/saturation/grid_1e9/error_vs_N__dd.png b/docs/figures/saturation/grid_1e9/error_vs_N__dd.png new file mode 100644 index 00000000..67441668 Binary files /dev/null and b/docs/figures/saturation/grid_1e9/error_vs_N__dd.png differ diff --git a/docs/figures/saturation/grid_1e9/error_vs_N__hll.png b/docs/figures/saturation/grid_1e9/error_vs_N__hll.png new file mode 100644 index 00000000..75a84cf0 Binary files /dev/null and b/docs/figures/saturation/grid_1e9/error_vs_N__hll.png differ diff --git a/docs/figures/saturation/grid_1e9/error_vs_N__kll-percall.png b/docs/figures/saturation/grid_1e9/error_vs_N__kll-percall.png new file mode 100644 index 00000000..c5a325f1 Binary files /dev/null and b/docs/figures/saturation/grid_1e9/error_vs_N__kll-percall.png differ diff --git a/docs/figures/saturation/grid_cost/README.md b/docs/figures/saturation/grid_cost/README.md new file mode 100644 index 00000000..55fe3179 --- /dev/null +++ b/docs/figures/saturation/grid_cost/README.md @@ -0,0 +1,17 @@ +# Grid cost, exact-vs-sketch crossover, recommendations + +From `--phase cost --n-max 1e7 --seeds 3 --cost-rows 3` (serial, idle machine) +and `--phase crossover`; see "Grid cost and crossover N*" in +`docs/saturation_study.md`. + +- `cost_cpu_memory_grid.png` (`scripts/plot_saturation_cost.py`): per-item + insert, per-fold merge and per-query CPU and `memory_bytes` at N = 1e7 vs + theta / alpha, one config per sketch. +- `crossover_exact_vs_sketch.png` (`scripts/plot_saturation_crossover.py`): + memory (top) and CPU to ingest N items and answer one query (bottom) for the + exact polars baseline (solid, one line per K) and every sketch config + (dashed), at theta = 1 / Pareto 1.5. The vertical gap is the saving. +- `crossover.csv`: N* per point (exact >= 10x / 100x the sketch from that N on). +- `recommendations.csv` (`scripts/recommend_config.py` on ASAPQuery#746's + `skew_summary.csv`): smallest-memory config per query, range and sketch + family that meets the query's accuracy target, with flags. diff --git a/docs/figures/saturation/grid_cost/cost_cpu_memory_grid.png b/docs/figures/saturation/grid_cost/cost_cpu_memory_grid.png new file mode 100644 index 00000000..98e3f399 Binary files /dev/null and b/docs/figures/saturation/grid_cost/cost_cpu_memory_grid.png differ diff --git a/docs/figures/saturation/grid_cost/crossover.csv b/docs/figures/saturation/grid_cost/crossover.csv new file mode 100644 index 00000000..e6020a55 --- /dev/null +++ b/docs/figures/saturation/grid_cost/crossover.csv @@ -0,0 +1,344 @@ +family,sketch,config,dist,param,cardinality,n_sat,sketch_memory_bytes,sketch_cpu_secs,memory_n_star_10x,memory_n_star_100x,cpu_n_star_10x,cpu_n_star_100x +frequency,cms-fastpath-vector2d,rows=3 cols=256,zipf,0.0,1000,177828,3072,0.22340969499760074,1778,56234,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=256,zipf,0.0,100000,not_saturated,3072,0.22373268887992448,1000,10000,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=256,zipf,0.0,10000000,not_saturated,3072,0.22520769471268345,1000,10000,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=256,zipf,0.5,1000,5623,3072,0.2238116926655127,1778,56234,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=256,zipf,0.5,100000,56234,3072,0.2250510222398812,1000,10000,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=256,zipf,0.5,10000000,562341,3072,0.2248930281867483,1000,10000,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=256,zipf,0.8,1000,5623,3072,0.2238306933306062,1778,56234,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=256,zipf,0.8,100000,17783,3072,0.22362368900131566,1778,17783,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=256,zipf,0.8,10000000,56234,3072,0.22475736151948988,1000,10000,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=256,zipf,1.0,1000,5623,3072,0.22361635926550827,1778,56234,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=256,zipf,1.0,100000,10000,3072,0.22364402222058133,1778,17783,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=256,zipf,1.0,10000000,17783,3072,0.2238143610591677,1778,17783,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=256,zipf,1.2,1000,5623,3072,0.22370002632005415,3162,56234,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=256,zipf,1.2,100000,17783,3072,0.2237063555407867,1778,17783,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=256,zipf,1.2,10000000,10000,3072,0.22518068986540568,1778,17783,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=256,zipf,1.5,1000,17783,3072,0.22407102599862078,3162,56234,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=256,zipf,1.5,100000,10000,3072,0.22405468898572214,3162,56234,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=256,zipf,1.5,10000000,17783,3072,0.22385335566214257,3162,56234,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=256,zipf,2.0,1000,31623,3072,0.22363169266040483,3162,56234,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=256,zipf,2.0,100000,100000,3072,0.22318469119412043,3162,56234,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=256,zipf,2.0,10000000,31623,3072,0.22374502536719193,3162,56234,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=1024,zipf,0.0,1000,316228,12288,0.22440869999826352,10000,177828,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=1024,zipf,0.0,100000,not_saturated,12288,0.2248110222498946,5623,56234,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=1024,zipf,0.0,10000000,not_saturated,12288,0.22600702778837548,5623,56234,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=1024,zipf,0.5,1000,5623,12288,0.22383069599611638,10000,177828,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=1024,zipf,0.5,100000,100000,12288,0.2241316888865772,5623,56234,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=1024,zipf,0.5,10000000,562341,12288,0.2239013612452328,5623,56234,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=1024,zipf,0.8,1000,100000,12288,0.2236493623324249,10000,177828,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=1024,zipf,0.8,100000,31623,12288,0.22366368891327537,5623,100000,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=1024,zipf,0.8,10000000,100000,12288,0.22511136148660513,5623,56234,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=1024,zipf,1.0,1000,17783,12288,0.22398602832320008,10000,177828,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=1024,zipf,1.0,100000,56234,12288,0.2233003555806129,5623,100000,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=1024,zipf,1.0,10000000,31623,12288,0.2248363623884641,5623,100000,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=1024,zipf,1.2,1000,5623,12288,0.22423802966296452,10000,177828,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=1024,zipf,1.2,100000,56234,12288,0.22369168890504568,10000,100000,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=1024,zipf,1.2,10000000,56234,12288,0.22391135657295652,10000,100000,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=1024,zipf,1.5,1000,56234,12288,0.22374202999514878,10000,177828,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=1024,zipf,1.5,100000,177828,12288,0.2241053559760855,10000,177828,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=1024,zipf,1.5,10000000,177828,12288,0.2237756891269328,10000,177828,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=1024,zipf,2.0,1000,316228,12288,0.2233940312087956,10000,177828,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=1024,zipf,2.0,100000,1778279,12288,0.2234553582519393,10000,177828,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=1024,zipf,2.0,10000000,1778279,12288,0.22325735814005834,10000,177828,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=4096,zipf,0.0,1000,1000000,49152,0.22336470633090913,56234,562341,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=4096,zipf,0.0,100000,not_saturated,49152,0.22394502234988445,17783,316228,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=4096,zipf,0.0,10000000,not_saturated,49152,0.22304702778376206,17783,177828,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=4096,zipf,0.5,1000,17783,49152,0.22427370689286863,56234,562341,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=4096,zipf,0.5,100000,177828,49152,0.22346168906645886,17783,316228,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=4096,zipf,0.5,10000000,1000000,49152,0.22352536140566617,17783,177828,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=4096,zipf,0.8,1000,31623,49152,0.22433370599332297,56234,562341,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=4096,zipf,0.8,100000,56234,49152,0.22361802233993638,17783,316228,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=4096,zipf,0.8,10000000,177828,49152,0.22508936133532748,17783,177828,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=4096,zipf,1.0,1000,17783,49152,0.22466070763901597,56234,562341,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=4096,zipf,1.0,100000,177828,49152,0.22334835566395841,31623,316228,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=4096,zipf,1.0,10000000,100000,49152,0.2234593610764241,17783,316228,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=4096,zipf,1.2,1000,56234,49152,0.22354737366124314,56234,562341,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=4096,zipf,1.2,100000,316228,49152,0.2237556889982697,56234,562341,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=4096,zipf,1.2,10000000,316228,49152,0.22419168999930847,56234,316228,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=4096,zipf,1.5,1000,not_saturated,49152,0.22360504066179496,56234,562341,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=4096,zipf,1.5,100000,1000000,49152,0.22326635598719127,56234,562341,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=4096,zipf,1.5,10000000,1000000,49152,0.22369468912769871,56234,562341,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=4096,zipf,2.0,1000,1778279,49152,0.2237907062348167,56234,562341,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=4096,zipf,2.0,100000,1778279,49152,0.2235463608308551,56234,562341,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=4096,zipf,2.0,10000000,not_saturated,49152,0.2239116940249524,56234,562341,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=16384,zipf,0.0,1000,1000,196608,0.22474374498828734,177828,3162278,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=16384,zipf,0.0,100000,not_saturated,196608,0.22800002542207062,100000,3162278,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=16384,zipf,0.0,10000000,not_saturated,196608,0.22506836282630457,100000,562341,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=16384,zipf,0.5,1000,1000,196608,0.22516474599838818,177828,3162278,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=16384,zipf,0.5,100000,177828,196608,0.22619135776469781,100000,3162278,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=16384,zipf,0.5,10000000,1000000,196608,0.22942936287405655,100000,562341,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=16384,zipf,0.8,1000,1000,196608,0.22427207866444906,177828,3162278,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=16384,zipf,0.8,100000,316228,196608,0.22540835752689076,100000,3162278,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=16384,zipf,0.8,10000000,316228,196608,0.2261180292164598,100000,1000000,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=16384,zipf,1.0,1000,1000,196608,0.2232494106643059,177828,3162278,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=16384,zipf,1.0,100000,316228,196608,0.22613069151061632,177828,3162278,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=16384,zipf,1.0,10000000,562341,196608,0.2253713623601004,100000,1778279,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=16384,zipf,1.2,1000,1000,196608,0.22381307733124917,177828,3162278,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=16384,zipf,1.2,100000,1000000,196608,0.22423502407344412,177828,3162278,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=16384,zipf,1.2,10000000,1000000,196608,0.22422869075806906,177828,1778279,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=16384,zipf,1.5,1000,1000,196608,0.22388674332112432,177828,3162278,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=16384,zipf,1.5,100000,1000000,196608,0.2235990262802503,177828,3162278,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=16384,zipf,1.5,10000000,1778279,196608,0.22451035822079143,177828,3162278,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=16384,zipf,2.0,1000,1000,196608,0.22315541132680905,177828,3162278,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=16384,zipf,2.0,100000,not_saturated,196608,0.2236373815961558,177828,3162278,1000,not_reached +frequency,cms-fastpath-vector2d,rows=3 cols=16384,zipf,2.0,10000000,not_saturated,196608,0.2244497144220971,177828,3162278,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=256,zipf,0.0,1000,100000,3072,0.2630963889929759,1778,56234,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=256,zipf,0.0,100000,not_saturated,3072,0.26194138429663744,1000,10000,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=256,zipf,0.0,10000000,not_saturated,3072,0.2629817238276065,1000,10000,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=256,zipf,0.5,1000,1000,3072,0.2623910591622642,1778,56234,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=256,zipf,0.5,100000,177828,3072,0.26252471765659796,1000,10000,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=256,zipf,0.5,10000000,not_saturated,3072,0.26434672637638895,1000,10000,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=256,zipf,0.8,1000,56234,3072,0.26284905499673195,1778,56234,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=256,zipf,0.8,100000,100000,3072,0.26103305066301663,1778,17783,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=256,zipf,0.8,10000000,1000000,3072,0.26404605740931947,1000,10000,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=256,zipf,1.0,1000,31623,3072,0.2628327213326638,1778,56234,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=256,zipf,1.0,100000,316228,3072,0.2629950510747098,1778,17783,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=256,zipf,1.0,10000000,316228,3072,0.2630210569644152,1778,17783,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=256,zipf,1.2,1000,316228,3072,0.26239139393904165,3162,56234,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=256,zipf,1.2,100000,316228,3072,0.26269971768389516,1778,17783,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=256,zipf,1.2,10000000,177828,3072,0.2649173861505078,1778,17783,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=256,zipf,1.5,1000,31623,3072,0.26301638799031685,3162,56234,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=256,zipf,1.5,100000,316228,3072,0.26246638451657434,3162,56234,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=256,zipf,1.5,10000000,100000,3072,0.26320905110839704,3162,56234,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=256,zipf,2.0,1000,562341,3072,0.2616700553298029,3162,56234,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=256,zipf,2.0,100000,17783,3072,0.262511387544216,3162,56234,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=256,zipf,2.0,10000000,177828,3072,0.2628727209208812,3162,56234,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=1024,zipf,0.0,1000,not_saturated,12288,0.2628790593313301,10000,177828,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=1024,zipf,0.0,100000,not_saturated,12288,0.2638793844866524,5623,56234,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=1024,zipf,0.0,10000000,not_saturated,12288,0.2645120566238367,5623,56234,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=1024,zipf,0.5,1000,31623,12288,0.26313072842850177,10000,177828,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=1024,zipf,0.5,100000,100000,12288,0.2636910512260784,5623,56234,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=1024,zipf,0.5,10000000,not_saturated,12288,0.2652090571475893,5623,56234,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=1024,zipf,0.8,1000,1778,12288,0.2626233923309551,10000,177828,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=1024,zipf,0.8,100000,100000,12288,0.26228571786596855,5623,100000,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=1024,zipf,0.8,10000000,1000000,12288,0.2652060571443927,5623,56234,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=1024,zipf,1.0,1000,3162,12288,0.2620523896657587,10000,177828,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=1024,zipf,1.0,100000,56234,12288,0.26301605081148943,5623,100000,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=1024,zipf,1.0,10000000,56234,12288,0.263090723246495,5623,100000,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=1024,zipf,1.2,1000,1000,12288,0.26283639166112716,10000,177828,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=1024,zipf,1.2,100000,100000,12288,0.2628140510516626,10000,100000,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=1024,zipf,1.2,10000000,100000,12288,0.26342838546429465,10000,100000,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=1024,zipf,1.5,1000,177828,12288,0.26213639266505684,10000,177828,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=1024,zipf,1.5,100000,316228,12288,0.2630830522291848,10000,177828,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=1024,zipf,1.5,10000000,562341,12288,0.2625907178857184,10000,177828,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=1024,zipf,2.0,1000,3162,12288,0.26140872773270357,10000,177828,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=1024,zipf,2.0,100000,100000,12288,0.26275872095595565,10000,177828,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=1024,zipf,2.0,10000000,1778279,12288,0.26220138961384065,10000,177828,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=4096,zipf,0.0,1000,not_saturated,49152,0.2638607372398125,56234,562341,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=4096,zipf,0.0,100000,not_saturated,49152,0.26293838455301394,17783,316228,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=4096,zipf,0.0,10000000,not_saturated,49152,0.26519805664656954,17783,177828,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=4096,zipf,0.5,1000,100000,49152,0.26315473857848143,56234,562341,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=4096,zipf,0.5,100000,100000,49152,0.2632927177596056,17783,316228,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=4096,zipf,0.5,10000000,not_saturated,49152,0.2627207234541104,17783,177828,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=4096,zipf,0.8,1000,56234,49152,0.2633450715498726,56234,562341,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=4096,zipf,0.8,100000,562341,49152,0.2637130511166468,17783,316228,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=4096,zipf,0.8,10000000,not_saturated,49152,0.2651627238004014,17783,177828,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=4096,zipf,1.0,1000,1778279,49152,0.26360907453710497,56234,562341,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=4096,zipf,1.0,100000,316228,49152,0.2639867177515303,31623,316228,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=4096,zipf,1.0,10000000,1000000,49152,0.26381105673124394,17783,316228,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=4096,zipf,1.2,1000,1000000,49152,0.26369007188632465,56234,562341,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=4096,zipf,1.2,100000,1000000,49152,0.26329838453566273,56234,562341,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=4096,zipf,1.2,10000000,1000000,49152,0.2627937203169069,56234,316228,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=4096,zipf,1.5,1000,1778279,49152,0.26292173699656707,56234,562341,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=4096,zipf,1.5,100000,1000000,49152,0.2628760513990628,56234,562341,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=4096,zipf,1.5,10000000,1778279,49152,0.26300471799580616,56234,562341,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=4096,zipf,2.0,1000,1000000,49152,0.2622694128799939,56234,562341,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=4096,zipf,2.0,100000,not_saturated,49152,0.2625573907409045,56234,562341,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=4096,zipf,2.0,10000000,not_saturated,49152,0.2634627247080246,56234,562341,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=16384,zipf,0.0,1000,562341,196608,0.263394797664844,177828,3162278,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=16384,zipf,0.0,100000,not_saturated,196608,0.26570005311664596,100000,3162278,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=16384,zipf,0.0,10000000,not_saturated,196608,0.2670623911332022,100000,562341,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=16384,zipf,0.5,1000,not_saturated,196608,0.26460980205308393,177828,3162278,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=16384,zipf,0.5,100000,56234,196608,0.2656713873506246,100000,3162278,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=16384,zipf,0.5,10000000,not_saturated,196608,0.2653547244441192,100000,562341,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=16384,zipf,0.8,1000,17783,196608,0.26436446599556535,177828,3162278,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=16384,zipf,0.8,100000,10000,196608,0.26727138743624296,100000,3162278,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=16384,zipf,0.8,10000000,1778279,196608,0.2655683913114466,100000,1000000,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=16384,zipf,1.0,1000,10000,196608,0.2645151341779694,177828,3162278,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=16384,zipf,1.0,100000,10000,196608,0.2671177212581169,177828,3162278,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=16384,zipf,1.0,10000000,1000000,196608,0.2653050576272487,100000,1778279,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=16384,zipf,1.2,1000,5623,196608,0.2646288013327755,177828,3162278,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=16384,zipf,1.2,100000,177828,196608,0.26411905305178207,177828,3162278,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=16384,zipf,1.2,10000000,1778279,196608,0.26396071985145647,177828,1778279,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=16384,zipf,1.5,1000,562341,196608,0.2639141323331722,177828,3162278,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=16384,zipf,1.5,100000,177828,196608,0.2633730560147171,177828,3162278,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=16384,zipf,1.5,10000000,56234,196608,0.2636520547284122,177828,3162278,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=16384,zipf,2.0,1000,100000,196608,0.26294146833238985,177828,3162278,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=16384,zipf,2.0,100000,31623,196608,0.26287608999041817,177828,3162278,1000,not_reached +frequency,countsketch-fastpath-vector2d,rows=3 cols=16384,zipf,2.0,10000000,100000,196608,0.26311375930378644,177828,3162278,1000,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=256,zipf,0.0,1000,5623,3072,1.3908710729618186,1778,56234,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=256,zipf,0.0,100000,3162,3072,0.8581848496117855,1000,10000,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=256,zipf,0.0,10000000,1000,3072,0.9197436019652696,1000,10000,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=256,zipf,0.5,1000,177828,3072,2.901321585075735,1778,56234,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=256,zipf,0.5,100000,1778279,3072,0.9362224008322528,1000,10000,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=256,zipf,0.5,10000000,1000000,3072,0.9000285705428639,1000,10000,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=256,zipf,0.8,1000,1000,3072,6.831607389892326,1778,56234,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=256,zipf,0.8,100000,1778279,3072,2.3020159632392865,1778,17783,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=256,zipf,0.8,10000000,not_saturated,3072,1.4753418403196508,1000,10000,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=256,zipf,1.0,1000,1000,3072,10.152391065234832,1778,56234,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=256,zipf,1.0,100000,not_saturated,3072,5.45679485119314,1778,17783,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=256,zipf,1.0,10000000,not_saturated,3072,3.823714284027799,1778,17783,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=256,zipf,1.2,1000,1000,3072,12.964425921548308,3162,56234,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=256,zipf,1.2,100000,not_saturated,3072,10.077077106781065,1778,17783,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=256,zipf,1.2,10000000,not_saturated,3072,9.273247797332388,1778,17783,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=256,zipf,1.5,1000,1000,3072,16.286239836288328,3162,56234,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=256,zipf,1.5,100000,not_saturated,3072,15.160266795740975,3162,56234,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=256,zipf,1.5,10000000,not_saturated,3072,15.981890712149053,3162,56234,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=256,zipf,2.0,1000,1000,3072,17.522831428652495,3162,56234,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=256,zipf,2.0,100000,562341,3072,17.604389271599544,3162,56234,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=256,zipf,2.0,10000000,177828,3072,17.509910150044313,3162,56234,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=1024,zipf,0.0,1000,not_saturated,12288,1.3984460705613642,10000,177828,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=1024,zipf,0.0,100000,17783,12288,0.8416608240343245,5623,56234,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=1024,zipf,0.0,10000000,1000,12288,0.87050581781694,5623,56234,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=1024,zipf,0.5,1000,1000,12288,3.561902436267761,10000,177828,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=1024,zipf,0.5,100000,56234,12288,1.0299201546892602,5623,56234,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=1024,zipf,0.5,10000000,not_saturated,12288,0.8678293110348151,5623,56234,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=1024,zipf,0.8,1000,1000,12288,7.088884837287234,10000,177828,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=1024,zipf,0.8,100000,5623,12288,2.897371218463476,5623,100000,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=1024,zipf,0.8,10000000,not_saturated,12288,1.584556917411267,5623,56234,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=1024,zipf,1.0,1000,1000,12288,10.337386151036176,10000,177828,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=1024,zipf,1.0,100000,1778,12288,6.601278270774769,5623,100000,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=1024,zipf,1.0,10000000,not_saturated,12288,4.777097557506799,5623,100000,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=1024,zipf,1.2,1000,1000,12288,13.579583280284282,10000,177828,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=1024,zipf,1.2,100000,1000,12288,11.171082245585954,10000,100000,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=1024,zipf,1.2,10000000,316228,12288,10.609070529699103,10000,100000,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=1024,zipf,1.5,1000,1000,12288,16.17468818099226,10000,177828,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=1024,zipf,1.5,100000,1000,12288,16.203072609966558,10000,177828,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=1024,zipf,1.5,10000000,1000,12288,15.559546283366998,10000,177828,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=1024,zipf,2.0,1000,1778,12288,17.583894310123544,10000,177828,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=1024,zipf,2.0,100000,3162,12288,17.824266860049406,10000,177828,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=1024,zipf,2.0,10000000,3162,12288,17.47469618882686,10000,177828,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=4096,zipf,0.0,1000,17783,49152,1.3898649547009194,56234,562341,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=4096,zipf,0.0,100000,not_saturated,49152,0.8329409696090867,17783,316228,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=4096,zipf,0.0,10000000,1000,49152,0.8448663840159043,17783,177828,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=4096,zipf,0.5,1000,1778,49152,3.5924358868884845,56234,562341,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=4096,zipf,0.5,100000,56234,49152,1.1089910152108626,17783,316228,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=4096,zipf,0.5,10000000,not_saturated,49152,0.8527334519215907,17783,177828,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=4096,zipf,0.8,1000,1000,49152,7.041857260899079,56234,562341,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=4096,zipf,0.8,100000,3162,49152,2.9812884265553987,17783,316228,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=4096,zipf,0.8,10000000,5623,49152,1.6260888728131888,17783,177828,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=4096,zipf,1.0,1000,1000,49152,10.230336236054946,56234,562341,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=4096,zipf,1.0,100000,1778,49152,6.556363376616289,31623,316228,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=4096,zipf,1.0,10000000,1778,49152,5.072269888384958,17783,316228,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=4096,zipf,1.2,1000,1000,49152,13.073194419210184,56234,562341,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=4096,zipf,1.2,100000,1000,49152,11.345731439423396,56234,562341,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=4096,zipf,1.2,10000000,1000,49152,11.011504851682217,56234,316228,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=4096,zipf,1.5,1000,1000,49152,16.368706457356293,56234,562341,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=4096,zipf,1.5,100000,1000,49152,15.635576388221573,56234,562341,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=4096,zipf,1.5,10000000,1000,49152,15.587351452428258,56234,562341,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=4096,zipf,2.0,1000,1778,49152,17.465140020979874,56234,562341,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=4096,zipf,2.0,100000,3162,49152,17.678481575469192,56234,562341,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=4096,zipf,2.0,10000000,3162,49152,17.745061224664084,56234,562341,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=16384,zipf,0.0,1000,5623,196608,1.4309840830656757,177828,3162278,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=16384,zipf,0.0,100000,316228,196608,0.9522807894518874,100000,3162278,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=16384,zipf,0.0,10000000,not_saturated,196608,0.8396358492695036,100000,562341,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=16384,zipf,0.5,1000,1778,196608,3.5730175450160186,177828,3162278,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=16384,zipf,0.5,100000,56234,196608,1.0967535980202776,100000,3162278,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=16384,zipf,0.5,10000000,1778279,196608,0.8599335460372204,100000,562341,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=16384,zipf,0.8,1000,1000,196608,7.0450818659026915,177828,3162278,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=16384,zipf,0.8,100000,3162,196608,2.959008925467571,100000,3162278,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=16384,zipf,0.8,10000000,5623,196608,1.6653930316789975,100000,1000000,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=16384,zipf,1.0,1000,1000,196608,10.269664220193304,177828,3162278,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=16384,zipf,1.0,100000,1778,196608,6.612525669218897,177828,3162278,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=16384,zipf,1.0,10000000,1778,196608,6.439837430930308,100000,1778279,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=16384,zipf,1.2,1000,1000,196608,13.02341303750842,177828,3162278,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=16384,zipf,1.2,100000,1000,196608,11.413335099402278,177828,3162278,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=16384,zipf,1.2,10000000,1000,196608,10.792642957285858,177828,1778279,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=16384,zipf,1.5,1000,1000,196608,16.042167013724516,177828,3162278,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=16384,zipf,1.5,100000,1000,196608,15.715061826003915,177828,3162278,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=16384,zipf,1.5,10000000,1000,196608,15.610345718124718,177828,3162278,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=16384,zipf,2.0,1000,1778,196608,17.634805645942286,177828,3162278,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=16384,zipf,2.0,100000,3162,196608,17.497365478324117,177828,3162278,not_reached,not_reached +topk,cms-heap-topk-fastpath-vector2d,rows=3 cols=16384,zipf,2.0,10000000,3162,196608,17.43076592188998,177828,3162278,not_reached,not_reached +cardinality,hll,lg_k=12,zipf,0.0,1000,1000,4096,0.03456938778384721,5623,56234,1000,not_reached +cardinality,hll,lg_k=12,zipf,0.0,100000,316228,4096,0.0341625228133254,5623,56234,1778,not_reached +cardinality,hll,lg_k=12,zipf,0.0,10000000,316228,4096,0.0341353125138905,5623,56234,1778,not_reached +cardinality,hll,lg_k=12,zipf,0.5,1000,1000,4096,0.03405811532007991,5623,56234,1000,not_reached +cardinality,hll,lg_k=12,zipf,0.5,100000,100000,4096,0.03417279131474886,5623,56234,1778,not_reached +cardinality,hll,lg_k=12,zipf,0.5,10000000,1000000,4096,0.03429672592737485,5623,56234,1000,not_reached +cardinality,hll,lg_k=12,zipf,0.8,1000,1778,4096,0.0339885227823519,5623,56234,1000,not_reached +cardinality,hll,lg_k=12,zipf,0.8,100000,17783,4096,0.034779403627594026,5623,56234,1000,not_reached +cardinality,hll,lg_k=12,zipf,0.8,10000000,1000000,4096,0.03412204152450459,5623,56234,1778,not_reached +cardinality,hll,lg_k=12,zipf,1.0,1000,31623,4096,0.03407380663136848,5623,56234,1000,not_reached +cardinality,hll,lg_k=12,zipf,1.0,100000,1000000,4096,0.034334335110593646,5623,56234,3162,not_reached +cardinality,hll,lg_k=12,zipf,1.0,10000000,3162,4096,0.034559230793605754,5623,56234,3162,not_reached +cardinality,hll,lg_k=12,zipf,1.2,1000,1778,4096,0.03420807889342874,5623,56234,1000,not_reached +cardinality,hll,lg_k=12,zipf,1.2,100000,1000000,4096,0.03485466218829547,5623,56234,1778,not_reached +cardinality,hll,lg_k=12,zipf,1.2,10000000,not_saturated,4096,0.03409009101151759,5623,56234,3162,not_reached +cardinality,hll,lg_k=12,zipf,1.5,1000,31623,4096,0.03396446693607941,5623,56234,1000,not_reached +cardinality,hll,lg_k=12,zipf,1.5,100000,177828,4096,0.03407142597798571,5623,56234,3162,not_reached +cardinality,hll,lg_k=12,zipf,1.5,10000000,562341,4096,0.034059463408834714,5623,56234,5623,not_reached +cardinality,hll,lg_k=12,zipf,2.0,1000,1778279,4096,0.033971117702591744,5623,56234,1000,not_reached +cardinality,hll,lg_k=12,zipf,2.0,100000,56234,4096,0.03394672236606599,5623,56234,1000,not_reached +cardinality,hll,lg_k=12,zipf,2.0,10000000,not_saturated,4096,0.034655868142839365,5623,56234,3162,not_reached +cardinality,hll,lg_k=14,zipf,0.0,1000,1000,16384,0.05013098343447049,17783,177828,1778,not_reached +cardinality,hll,lg_k=14,zipf,0.0,100000,10000,16384,0.034897253519359656,17783,177828,10000,not_reached +cardinality,hll,lg_k=14,zipf,0.0,10000000,not_saturated,16384,0.03521189888687531,17783,177828,10000,not_reached +cardinality,hll,lg_k=14,zipf,0.5,1000,3162,16384,0.034062081217746606,17783,177828,3162,not_reached +cardinality,hll,lg_k=14,zipf,0.5,100000,177828,16384,0.03476551575044991,17783,177828,10000,not_reached +cardinality,hll,lg_k=14,zipf,0.5,10000000,not_saturated,16384,0.03530568442038534,17783,177828,10000,not_reached +cardinality,hll,lg_k=14,zipf,0.8,1000,5623,16384,0.034311704940267355,17783,177828,1778,not_reached +cardinality,hll,lg_k=14,zipf,0.8,100000,562341,16384,0.03457527927720129,17783,177828,10000,not_reached +cardinality,hll,lg_k=14,zipf,0.8,10000000,1000000,16384,0.03503941522290749,17783,177828,10000,not_reached +cardinality,hll,lg_k=14,zipf,1.0,1000,1000,16384,0.03430626066209151,17783,177828,3162,not_reached +cardinality,hll,lg_k=14,zipf,1.0,100000,not_saturated,16384,0.034708000416694425,17783,177828,10000,not_reached +cardinality,hll,lg_k=14,zipf,1.0,10000000,not_saturated,16384,0.035024137076020756,17783,177828,17783,not_reached +cardinality,hll,lg_k=14,zipf,1.2,1000,1000,16384,0.034575996920892485,17783,177828,3162,not_reached +cardinality,hll,lg_k=14,zipf,1.2,100000,not_saturated,16384,0.03551601813159453,17783,177828,10000,not_reached +cardinality,hll,lg_k=14,zipf,1.2,10000000,1778279,16384,0.03489353958718226,17783,177828,10000,not_reached +cardinality,hll,lg_k=14,zipf,1.5,1000,100000,16384,0.03461038560301957,17783,177828,3162,not_reached +cardinality,hll,lg_k=14,zipf,1.5,100000,not_saturated,16384,0.034387001090355716,17783,177828,17783,not_reached +cardinality,hll,lg_k=14,zipf,1.5,10000000,not_saturated,16384,0.034508124475354854,17783,177828,10000,not_reached +cardinality,hll,lg_k=14,zipf,2.0,1000,not_saturated,16384,0.034075440850246594,17783,177828,10000,not_reached +cardinality,hll,lg_k=14,zipf,2.0,100000,not_saturated,16384,0.05016551754501836,17783,177828,not_reached,not_reached +cardinality,hll,lg_k=14,zipf,2.0,10000000,not_saturated,16384,0.05002259054038068,17783,177828,1000000,not_reached +cardinality,hll,lg_k=16,zipf,0.0,1000,1000,65536,0.03529006625748267,100000,562341,10000,not_reached +cardinality,hll,lg_k=16,zipf,0.0,100000,1000000,65536,0.037201888485096826,100000,562341,31623,not_reached +cardinality,hll,lg_k=16,zipf,0.0,10000000,562341,65536,0.038623352613449735,100000,562341,31623,not_reached +cardinality,hll,lg_k=16,zipf,0.5,1000,1000,65536,0.035162186978054615,100000,562341,5623,not_reached +cardinality,hll,lg_k=16,zipf,0.5,100000,not_saturated,65536,0.037292552894140185,100000,562341,31623,not_reached +cardinality,hll,lg_k=16,zipf,0.5,10000000,562341,65536,0.03938997056662186,100000,562341,31623,not_reached +cardinality,hll,lg_k=16,zipf,0.8,1000,17783,65536,0.03556487690624495,100000,562341,10000,not_reached +cardinality,hll,lg_k=16,zipf,0.8,100000,1778279,65536,0.03694726520104524,100000,562341,31623,not_reached +cardinality,hll,lg_k=16,zipf,0.8,10000000,1778279,65536,0.03839993014024537,100000,562341,31623,not_reached +cardinality,hll,lg_k=16,zipf,1.0,1000,17783,65536,0.03503348767012128,100000,562341,17783,not_reached +cardinality,hll,lg_k=16,zipf,1.0,100000,not_saturated,65536,0.036528352204811924,100000,562341,31623,not_reached +cardinality,hll,lg_k=16,zipf,1.0,10000000,1778279,65536,0.03824793121731535,100000,562341,31623,not_reached +cardinality,hll,lg_k=16,zipf,1.2,1000,56234,65536,0.03500634788378672,100000,562341,10000,not_reached +cardinality,hll,lg_k=16,zipf,1.2,100000,not_saturated,65536,0.03676501732212499,100000,562341,31623,not_reached +cardinality,hll,lg_k=16,zipf,1.2,10000000,not_saturated,65536,0.037311812953937984,100000,562341,56234,not_reached +cardinality,hll,lg_k=16,zipf,1.5,1000,31623,65536,0.03511086155248576,100000,562341,10000,not_reached +cardinality,hll,lg_k=16,zipf,1.5,100000,56234,65536,0.03578702850740185,100000,562341,31623,not_reached +cardinality,hll,lg_k=16,zipf,1.5,10000000,not_saturated,65536,0.035947640660649434,100000,562341,56234,not_reached +cardinality,hll,lg_k=16,zipf,2.0,1000,562341,65536,0.048007797275532935,100000,562341,10000000,not_reached +cardinality,hll,lg_k=16,zipf,2.0,100000,100000,65536,0.0504008686356227,100000,562341,not_reached,not_reached +cardinality,hll,lg_k=16,zipf,2.0,10000000,1778279,65536,0.03521706956654299,100000,562341,100000,not_reached +quantile,kll-percall,k=50,pareto,1.1,,31623,1600,0.25120694644911995,1778,17783,1000,562341 +quantile,kll-percall,k=50,pareto,1.5,,177828,1600,0.2523282559031779,1778,17783,1000,1000000 +quantile,kll-percall,k=50,pareto,2.0,,not_saturated,1600,0.2509616206341706,1778,17783,1000,562341 +quantile,kll-percall,k=50,pareto,3.0,,not_saturated,1600,0.25306907070983614,1778,17783,1000,562341 +quantile,kll-percall,k=200,pareto,1.1,,not_saturated,6400,0.25188603719336483,5623,100000,1000,562341 +quantile,kll-percall,k=200,pareto,1.5,,not_saturated,6400,0.2527151931260529,5623,100000,1000,1000000 +quantile,kll-percall,k=200,pareto,2.0,,not_saturated,6400,0.2564858164966189,5623,100000,1000,562341 +quantile,kll-percall,k=200,pareto,3.0,,177828,6400,0.2540536676941532,5623,100000,1000,562341 +quantile,kll-percall,k=800,pareto,1.1,,17783,25600,0.25917003822774515,17783,316228,1000,562341 +quantile,kll-percall,k=800,pareto,1.5,,not_saturated,25600,0.2584970124403127,17783,316228,1000,1000000 +quantile,kll-percall,k=800,pareto,2.0,,not_saturated,25600,0.2595164289340731,17783,316228,1000,562341 +quantile,kll-percall,k=800,pareto,3.0,,not_saturated,25600,0.2590892433356333,17783,316228,1000,562341 +quantile,dd,alpha=0.005,pareto,1.1,,316228,12576,0.15707980289951745,10000,177828,1000,177828 +quantile,dd,alpha=0.005,pareto,1.5,,10000,9224,0.15634895596013362,10000,100000,1000,177828 +quantile,dd,alpha=0.005,pareto,2.0,,17783,6920,0.1561266624256427,10000,100000,1000,177828 +quantile,dd,alpha=0.005,pareto,3.0,,1778,4616,0.16422183538878807,5623,56234,1000,177828 +quantile,dd,alpha=0.01,pareto,1.1,,3162,6288,0.15615399163253602,5623,100000,1000,177828 +quantile,dd,alpha=0.01,pareto,1.5,,3162,4616,0.1561373406885192,5623,56234,1000,177828 +quantile,dd,alpha=0.01,pareto,2.0,,1000,3464,0.15696688446237103,5623,56234,1000,177828 +quantile,dd,alpha=0.01,pareto,3.0,,1000,2312,0.15586612509807116,3162,17783,1000,177828 +quantile,dd,alpha=0.02,pareto,1.1,,1000,3144,0.15642865338737158,3162,56234,1000,177828 +quantile,dd,alpha=0.02,pareto,1.5,,1000,2312,0.15747882214517062,3162,17783,1000,177828 +quantile,dd,alpha=0.02,pareto,2.0,,1000,1736,0.1561453971307622,3162,17783,1000,177828 +quantile,dd,alpha=0.02,pareto,3.0,,1000,1160,0.1571013136097804,1778,10000,1000,177828 +quantile,dd,alpha=0.05,pareto,1.1,,1000,1264,0.15710788030143372,1778,10000,1000,177828 +quantile,dd,alpha=0.05,pareto,1.5,,1000,928,0.15605780221731944,1778,10000,1000,177828 +quantile,dd,alpha=0.05,pareto,2.0,,1000,696,0.15621183059928348,1000,10000,1000,177828 +quantile,dd,alpha=0.05,pareto,3.0,,1000,464,0.15620341759628867,1000,5623,1000,177828 diff --git a/docs/figures/saturation/grid_cost/crossover_exact_vs_sketch.png b/docs/figures/saturation/grid_cost/crossover_exact_vs_sketch.png new file mode 100644 index 00000000..e4164009 Binary files /dev/null and b/docs/figures/saturation/grid_cost/crossover_exact_vs_sketch.png differ diff --git a/docs/figures/saturation/grid_cost/recommendations.csv b/docs/figures/saturation/grid_cost/recommendations.csv new file mode 100644 index 00000000..3793e74e --- /dev/null +++ b/docs/figures/saturation/grid_cost/recommendations.csv @@ -0,0 +1,217 @@ +dataset,query_id,range,family,config,est_error,target,meets_target,n_sat,flags,memory_bytes,insert_ns_per_item,merge_us_per_fold,query_phase_us,grid_param,grid_K,N,shards +alibaba_v2022,mcr_by_msname,instant,cms,rows=3 cols=16384,0.020760829338153652,0.05,True,316228,,196608.0,22.54083333333333,28.933333333333334,2420.0,0.8,100000,521344,1 +alibaba_v2022,mcr_by_msname,instant,countsketch,rows=3 cols=4096,0.04153682919157856,0.05,True,562341,not saturated at min_N,49152.0,26.3713,5.977777777777778,5111.666666666666,0.8,100000,521344,1 +alibaba_v2022,mcr_by_msname,instant,topk,rows=5 cols=1024,0.9895833333333334,0.95,True,5623,cost scaled from rows=3,20480.0,482.89444444444445,152.5555555555556,8.333333333333334,0.8,100000,521344,1 +alibaba_v2022,mcr_by_msname,5m,cms,rows=3 cols=4096,0.044913144197755384,0.05,True,177828,extrapolated beyond N=1e+07,49152.0,22.33483333333333,5.977777777777778,2233.0,1.0,100000,20040209,5 +alibaba_v2022,mcr_by_msname,5m,countsketch,rows=3 cols=4096,0.018155593895840188,0.05,True,316228,extrapolated beyond N=1e+07,49152.0,26.39866666666667,6.177777777777778,5108.333333333334,1.0,100000,20040209,5 +alibaba_v2022,mcr_by_msname,5m,topk,rows=3 cols=1024,0.96875,0.95,True,1778,no merge curve for point; single-sketch curve optimistic,12288.0,660.1273666666667,94.97777777777777,5.333333333333333,1.0,100000,20040209,5 +alibaba_v2022,mcr_by_msname,1h,cms,rows=3 cols=4096,0.044913144197755384,0.05,True,177828,extrapolated beyond N=1e+07,49152.0,22.33483333333333,5.977777777777778,2233.0,1.0,100000,166100193,60 +alibaba_v2022,mcr_by_msname,1h,countsketch,rows=3 cols=4096,0.018155593895840188,0.05,True,316228,extrapolated beyond N=1e+07,49152.0,26.39866666666667,6.177777777777778,5108.333333333334,1.0,100000,166100193,60 +alibaba_v2022,mcr_by_msname,1h,topk,rows=3 cols=1024,0.96875,0.95,True,1778,no merge curve for point; single-sketch curve optimistic,12288.0,660.1273666666667,94.97777777777777,5.333333333333333,1.0,100000,166100193,60 +alibaba_v2022,mcr_by_nodeid,instant,cms,rows=5 cols=16384,1.1753430524102333,0.05,False,56234133,target not met; not saturated at min_N; cost scaled from rows=3,327680.0,38.0,53.370370370370374,4237.222222222223,0.0,100000,521344,1 +alibaba_v2022,mcr_by_nodeid,instant,countsketch,rows=5 cols=16384,0.40686253508409664,0.05,False,1778279,target not met; not saturated at min_N; cost scaled from rows=3,327680.0,44.283333333333324,48.85185185185186,8852.77777777778,0.0,100000,521344,1 +alibaba_v2022,mcr_by_nodeid,instant,topk,rows=3 cols=16384,0.01809371296620448,0.95,False,316228,target not met,196608.0,95.22756666666666,103.06666666666668,5.666666666666666,0.0,100000,521344,1 +alibaba_v2022,mcr_by_nodeid,5m,cms,rows=5 cols=16384,0.11674291755274485,0.05,False,316228,target not met; extrapolated beyond N=1e+07; cost scaled from rows=3,327680.0,37.69855555555556,52.296296296296305,4072.2222222222226,0.5,100000,20040209,5 +alibaba_v2022,mcr_by_nodeid,5m,countsketch,rows=5 cols=16384,0.048418503893649385,0.05,True,562341,extrapolated beyond N=1e+07; cost scaled from rows=3,327680.0,44.27855555555554,49.55555555555556,9004.444444444445,0.5,100000,20040209,5 +alibaba_v2022,mcr_by_nodeid,5m,topk,rows=3 cols=16384,0.9895833333333334,0.95,True,56234,no merge curve for point; single-sketch curve optimistic; extrapolated beyond N=1e+07,196608.0,109.67483333333332,117.08888888888889,6.0,0.5,100000,20040209,5 +alibaba_v2022,mcr_by_nodeid,1h,cms,rows=5 cols=16384,0.11674291755274485,0.05,False,316228,target not met; extrapolated beyond N=1e+07; cost scaled from rows=3,327680.0,37.69855555555556,52.296296296296305,4072.2222222222226,0.5,100000,166100193,60 +alibaba_v2022,mcr_by_nodeid,1h,countsketch,rows=5 cols=16384,0.048418503893649385,0.05,True,562341,extrapolated beyond N=1e+07; cost scaled from rows=3,327680.0,44.27855555555554,49.55555555555556,9004.444444444445,0.5,100000,166100193,60 +alibaba_v2022,mcr_by_nodeid,1h,topk,rows=3 cols=16384,0.9895833333333334,0.95,True,56234,no merge curve for point; single-sketch curve optimistic; extrapolated beyond N=1e+07,196608.0,109.67483333333332,117.08888888888889,6.0,0.5,100000,166100193,60 +alibaba_v2022,rt_p99_by_msname,instant,cms,rows=3 cols=16384,0.020760829338153652,0.05,True,316228,,196608.0,22.54083333333333,28.933333333333334,2420.0,0.8,100000,521344,1 +alibaba_v2022,rt_p99_by_msname,instant,countsketch,rows=3 cols=4096,0.04153682919157856,0.05,True,562341,not saturated at min_N,49152.0,26.3713,5.977777777777778,5111.666666666666,0.8,100000,521344,1 +alibaba_v2022,rt_p99_by_msname,instant,topk,rows=5 cols=1024,0.9895833333333334,0.95,True,5623,cost scaled from rows=3,20480.0,482.89444444444445,152.5555555555556,8.333333333333334,0.8,100000,521344,1 +alibaba_v2022,rt_p99_by_msname,5m,cms,rows=3 cols=4096,0.014887852077571267,0.05,True,316228,extrapolated beyond N=1e+07,49152.0,22.375566666666668,5.933333333333334,2157.9999999999995,1.2,100000,20040209,5 +alibaba_v2022,rt_p99_by_msname,5m,countsketch,rows=3 cols=1024,0.04947613038114032,0.05,True,100000,,12288.0,26.281399999999998,1.9555555555555557,4933.333333333334,1.2,100000,20040209,5 +alibaba_v2022,rt_p99_by_msname,5m,topk,rows=3 cols=1024,0.96875,0.95,True,1000,no merge curve for point; single-sketch curve optimistic,12288.0,1117.1077666666665,93.28888888888889,5.0,1.2,100000,20040209,5 +alibaba_v2022,rt_p99_by_msname,1h,cms,rows=3 cols=4096,0.014887852077571267,0.05,True,316228,extrapolated beyond N=1e+07,49152.0,22.375566666666668,5.933333333333334,2157.9999999999995,1.2,100000,166100193,60 +alibaba_v2022,rt_p99_by_msname,1h,countsketch,rows=3 cols=1024,0.0497436737693786,0.05,True,100000,,12288.0,26.281399999999998,1.9555555555555557,4933.333333333334,1.2,100000,166100193,60 +alibaba_v2022,rt_p99_by_msname,1h,topk,rows=3 cols=1024,0.96875,0.95,True,1000,no merge curve for point; single-sketch curve optimistic,12288.0,1117.1077666666665,93.28888888888889,5.0,1.2,100000,166100193,60 +alibaba_v2022,rt_p99_by_msname,instant,kll,k=200,0.002470041815777119,0.01,True,not_saturated,not saturated,6400.0,25.645400000000002,5.177777777777778,3240.9999999999995,3.0,,401372,1 +alibaba_v2022,rt_p99_by_msname,instant,dd,alpha=0.01,0.007332698267151987,0.01,True,1000,,3464.0,15.694700000000001,1.1333333333333333,2008.3333333333333,3.0,,401372,1 +alibaba_v2022,rt_p99_by_msname,5m,kll,k=50,0.009129934662150238,0.01,True,not_saturated,"alpha 12.6717 beyond grid, used 3; no merge curve for point; single-sketch curve optimistic; not saturated",1600.0,25.094000000000005,2.666666666666667,2186.3333333333335,3.0,,18912878,5 +alibaba_v2022,rt_p99_by_msname,5m,dd,alpha=0.01,0.007349054272741779,0.01,True,1000,"alpha 12.6717 beyond grid, used 3",3464.0,15.694700000000001,1.1333333333333333,2008.3333333333333,3.0,,18912878,5 +alibaba_v2022,rt_p99_by_msname,1h,kll,k=200,0.0022262964207332685,0.01,True,not_saturated,"alpha 9.27532 beyond grid, used 3; no merge curve for point; single-sketch curve optimistic; not saturated",6400.0,25.645400000000002,5.177777777777778,3240.9999999999995,3.0,,152748684,60 +alibaba_v2022,rt_p99_by_msname,1h,dd,alpha=0.01,0.007350253064515069,0.01,True,1000,"alpha 9.27532 beyond grid, used 3",3464.0,15.694700000000001,1.1333333333333333,2008.3333333333333,3.0,,152748684,60 +alibaba_v2022,calls_by_rpctype,1m,cms,rows=3 cols=1024,0.012068843865983931,0.05,True,5623,not saturated at min_N,12288.0,22.423800000000004,1.9333333333333336,29.666666666666668,1.2,1000,5401423,1 +alibaba_v2022,calls_by_rpctype,1m,countsketch,rows=3 cols=1024,0.040474453524032145,0.05,True,1000,not saturated at min_N,12288.0,26.283633333333334,2.0444444444444443,58.333333333333336,1.2,1000,5401423,1 +alibaba_v2022,calls_by_rpctype,1m,topk,rows=5 cols=256,0.96875,0.95,True,1000,not saturated at min_N; cost scaled from rows=3,5120.0,2160.7369444444444,152.29629629629633,8.333333333333334,1.2,1000,5401423,1 +alibaba_v2022,calls_by_rpctype,5m,cms,rows=3 cols=1024,0.012062400730156707,0.05,True,5623,,12288.0,22.423800000000004,1.9333333333333336,29.666666666666668,1.2,1000,26594881,5 +alibaba_v2022,calls_by_rpctype,5m,countsketch,rows=3 cols=1024,0.040519028976739355,0.05,True,1000,,12288.0,26.283633333333334,2.0444444444444443,58.333333333333336,1.2,1000,26594881,5 +alibaba_v2022,calls_by_rpctype,5m,topk,rows=5 cols=256,0.96875,0.95,True,1000,no merge curve for point; single-sketch curve optimistic; extrapolated beyond N=1e+07; cost scaled from rows=3,5120.0,2160.7369444444444,152.29629629629633,8.333333333333334,1.2,1000,26594881,5 +alibaba_v2022,calls_by_rpctype,1h,cms,rows=3 cols=1024,0.012054748429688853,0.05,True,5623,,12288.0,22.423800000000004,1.9333333333333336,29.666666666666668,1.2,1000,260437245,60 +alibaba_v2022,calls_by_rpctype,1h,countsketch,rows=3 cols=1024,0.04039436310787848,0.05,True,1000,,12288.0,26.283633333333334,2.0444444444444443,58.333333333333336,1.2,1000,260437245,60 +alibaba_v2022,calls_by_rpctype,1h,topk,rows=5 cols=256,0.96875,0.95,True,1000,no merge curve for point; single-sketch curve optimistic; extrapolated beyond N=1e+07; cost scaled from rows=3,5120.0,2160.7369444444444,152.29629629629633,8.333333333333334,1.2,1000,260437245,60 +alibaba_v2022,calls_by_service,1m,cms,rows=3 cols=16384,0.02129176204509065,0.05,True,316228,not saturated at min_N,196608.0,22.54083333333333,28.933333333333334,2420.0,0.8,100000,5401423,1 +alibaba_v2022,calls_by_service,1m,countsketch,rows=3 cols=4096,0.03928169571169596,0.05,True,562341,not saturated at min_N,49152.0,26.3713,5.977777777777778,5111.666666666666,0.8,100000,5401423,1 +alibaba_v2022,calls_by_service,1m,topk,rows=5 cols=1024,0.9895833333333334,0.95,True,5623,not saturated at min_N; cost scaled from rows=3,20480.0,482.89444444444445,152.5555555555556,8.333333333333334,0.8,100000,5401423,1 +alibaba_v2022,calls_by_service,5m,cms,rows=3 cols=16384,0.019873590686755222,0.05,True,562341,extrapolated beyond N=1e+07,196608.0,22.537133333333333,30.955555555555556,56817.33333333332,1.0,10000000,26594881,5 +alibaba_v2022,calls_by_service,5m,countsketch,rows=3 cols=4096,0.01832341853528574,0.05,True,1000000,extrapolated beyond N=1e+07,49152.0,26.381099999999996,6.355555555555556,111033.66666666667,1.0,10000000,26594881,5 +alibaba_v2022,calls_by_service,5m,topk,rows=5 cols=1024,1.0,0.95,True,1778,no merge curve for point; single-sketch curve optimistic; extrapolated beyond N=1e+07; cost scaled from rows=3,20480.0,796.0973888888889,137.33333333333331,929.9999999999999,1.0,10000000,26594881,5 +alibaba_v2022,calls_by_service,1h,cms,rows=3 cols=16384,0.019873590686755222,0.05,True,562341,extrapolated beyond N=1e+07,196608.0,22.537133333333333,30.955555555555556,56817.33333333332,1.0,10000000,260437245,60 +alibaba_v2022,calls_by_service,1h,countsketch,rows=3 cols=4096,0.01832341853528574,0.05,True,1000000,extrapolated beyond N=1e+07,49152.0,26.381099999999996,6.355555555555556,111033.66666666667,1.0,10000000,260437245,60 +alibaba_v2022,calls_by_service,1h,topk,rows=5 cols=1024,1.0,0.95,True,1778,no merge curve for point; single-sketch curve optimistic; extrapolated beyond N=1e+07; cost scaled from rows=3,20480.0,796.0973888888889,137.33333333333331,929.9999999999999,1.0,10000000,260437245,60 +alibaba_v2022,calls_by_um,1m,cms,rows=3 cols=4096,0.04470475707962842,0.05,True,177828,not saturated at min_N,49152.0,22.33483333333333,5.977777777777778,2233.0,1.0,100000,5401423,1 +alibaba_v2022,calls_by_um,1m,countsketch,rows=3 cols=4096,0.0182543247070165,0.05,True,316228,not saturated at min_N,49152.0,26.39866666666667,6.177777777777778,5108.333333333334,1.0,100000,5401423,1 +alibaba_v2022,calls_by_um,1m,topk,rows=3 cols=1024,0.96875,0.95,True,1778,not saturated at min_N,12288.0,660.1273666666667,94.97777777777777,5.333333333333333,1.0,100000,5401423,1 +alibaba_v2022,calls_by_um,5m,cms,rows=3 cols=4096,0.044913144197755384,0.05,True,177828,extrapolated beyond N=1e+07,49152.0,22.33483333333333,5.977777777777778,2233.0,1.0,100000,26594881,5 +alibaba_v2022,calls_by_um,5m,countsketch,rows=3 cols=4096,0.018155593895840188,0.05,True,316228,extrapolated beyond N=1e+07,49152.0,26.39866666666667,6.177777777777778,5108.333333333334,1.0,100000,26594881,5 +alibaba_v2022,calls_by_um,5m,topk,rows=3 cols=1024,0.96875,0.95,True,1778,no merge curve for point; single-sketch curve optimistic,12288.0,660.1273666666667,94.97777777777777,5.333333333333333,1.0,100000,26594881,5 +alibaba_v2022,calls_by_um,1h,cms,rows=3 cols=4096,0.044913144197755384,0.05,True,177828,extrapolated beyond N=1e+07,49152.0,22.33483333333333,5.977777777777778,2233.0,1.0,100000,260437245,60 +alibaba_v2022,calls_by_um,1h,countsketch,rows=3 cols=4096,0.018155593895840188,0.05,True,316228,extrapolated beyond N=1e+07,49152.0,26.39866666666667,6.177777777777778,5108.333333333334,1.0,100000,260437245,60 +alibaba_v2022,calls_by_um,1h,topk,rows=3 cols=1024,0.96875,0.95,True,1778,no merge curve for point; single-sketch curve optimistic,12288.0,660.1273666666667,94.97777777777777,5.333333333333333,1.0,100000,260437245,60 +alibaba_v2022,calls_by_dm,1m,cms,rows=3 cols=4096,0.04470475707962842,0.05,True,177828,not saturated at min_N,49152.0,22.33483333333333,5.977777777777778,2233.0,1.0,100000,5401423,1 +alibaba_v2022,calls_by_dm,1m,countsketch,rows=3 cols=4096,0.0182543247070165,0.05,True,316228,not saturated at min_N,49152.0,26.39866666666667,6.177777777777778,5108.333333333334,1.0,100000,5401423,1 +alibaba_v2022,calls_by_dm,1m,topk,rows=3 cols=1024,0.96875,0.95,True,1778,not saturated at min_N,12288.0,660.1273666666667,94.97777777777777,5.333333333333333,1.0,100000,5401423,1 +alibaba_v2022,calls_by_dm,5m,cms,rows=3 cols=4096,0.044913144197755384,0.05,True,177828,extrapolated beyond N=1e+07,49152.0,22.33483333333333,5.977777777777778,2233.0,1.0,100000,26594881,5 +alibaba_v2022,calls_by_dm,5m,countsketch,rows=3 cols=4096,0.018155593895840188,0.05,True,316228,extrapolated beyond N=1e+07,49152.0,26.39866666666667,6.177777777777778,5108.333333333334,1.0,100000,26594881,5 +alibaba_v2022,calls_by_dm,5m,topk,rows=3 cols=1024,0.96875,0.95,True,1778,no merge curve for point; single-sketch curve optimistic,12288.0,660.1273666666667,94.97777777777777,5.333333333333333,1.0,100000,26594881,5 +alibaba_v2022,calls_by_dm,1h,cms,rows=3 cols=4096,0.044913144197755384,0.05,True,177828,extrapolated beyond N=1e+07,49152.0,22.33483333333333,5.977777777777778,2233.0,1.0,100000,260437245,60 +alibaba_v2022,calls_by_dm,1h,countsketch,rows=3 cols=4096,0.018155593895840188,0.05,True,316228,extrapolated beyond N=1e+07,49152.0,26.39866666666667,6.177777777777778,5108.333333333334,1.0,100000,260437245,60 +alibaba_v2022,calls_by_dm,1h,topk,rows=3 cols=1024,0.96875,0.95,True,1778,no merge curve for point; single-sketch curve optimistic,12288.0,660.1273666666667,94.97777777777777,5.333333333333333,1.0,100000,260437245,60 +alibaba_v2022,calls_by_interface,1m,cms,rows=5 cols=16384,0.11852205026294958,0.05,False,1000000,target not met; not saturated at min_N; cost scaled from rows=3,327680.0,37.68633333333333,51.18518518518519,187872.7777777778,0.8,10000000,5401423,1 +alibaba_v2022,calls_by_interface,1m,countsketch,rows=3 cols=4096,0.042239557359247955,0.05,True,5623413,not saturated at min_N,49152.0,26.516266666666667,6.133333333333334,220461.0,0.8,10000000,5401423,1 +alibaba_v2022,calls_by_interface,1m,topk,rows=5 cols=1024,0.9873968588599285,0.95,True,31623,not saturated at min_N; cost scaled from rows=3,20480.0,263.90122222222226,133.85185185185185,2103.888888888889,0.8,10000000,5401423,1 +alibaba_v2022,calls_by_interface,5m,cms,rows=3 cols=16384,0.019873590686755222,0.05,True,562341,extrapolated beyond N=1e+07,196608.0,22.537133333333333,30.955555555555556,56817.33333333332,1.0,10000000,26594881,5 +alibaba_v2022,calls_by_interface,5m,countsketch,rows=3 cols=4096,0.01832341853528574,0.05,True,1000000,extrapolated beyond N=1e+07,49152.0,26.381099999999996,6.355555555555556,111033.66666666667,1.0,10000000,26594881,5 +alibaba_v2022,calls_by_interface,5m,topk,rows=5 cols=1024,1.0,0.95,True,1778,no merge curve for point; single-sketch curve optimistic; extrapolated beyond N=1e+07; cost scaled from rows=3,20480.0,796.0973888888889,137.33333333333331,929.9999999999999,1.0,10000000,26594881,5 +alibaba_v2022,calls_by_interface,1h,cms,rows=3 cols=16384,0.019873590686755222,0.05,True,562341,extrapolated beyond N=1e+07,196608.0,22.537133333333333,30.955555555555556,56817.33333333332,1.0,10000000,260437245,60 +alibaba_v2022,calls_by_interface,1h,countsketch,rows=3 cols=4096,0.01832341853528574,0.05,True,1000000,extrapolated beyond N=1e+07,49152.0,26.381099999999996,6.355555555555556,111033.66666666667,1.0,10000000,260437245,60 +alibaba_v2022,calls_by_interface,1h,topk,rows=5 cols=1024,1.0,0.95,True,1778,no merge curve for point; single-sketch curve optimistic; extrapolated beyond N=1e+07; cost scaled from rows=3,20480.0,796.0973888888889,137.33333333333331,929.9999999999999,1.0,10000000,260437245,60 +alibaba_v2022,calls_by_um_dm,1m,cms,rows=3 cols=16384,0.02129176204509065,0.05,True,316228,not saturated at min_N,196608.0,22.54083333333333,28.933333333333334,2420.0,0.8,100000,5401423,1 +alibaba_v2022,calls_by_um_dm,1m,countsketch,rows=3 cols=4096,0.03928169571169596,0.05,True,562341,not saturated at min_N,49152.0,26.3713,5.977777777777778,5111.666666666666,0.8,100000,5401423,1 +alibaba_v2022,calls_by_um_dm,1m,topk,rows=5 cols=1024,0.9895833333333334,0.95,True,5623,not saturated at min_N; cost scaled from rows=3,20480.0,482.89444444444445,152.5555555555556,8.333333333333334,0.8,100000,5401423,1 +alibaba_v2022,calls_by_um_dm,5m,cms,rows=3 cols=16384,0.0213003939726821,0.05,True,316228,extrapolated beyond N=1e+07,196608.0,22.54083333333333,28.933333333333334,2420.0,0.8,100000,26594881,5 +alibaba_v2022,calls_by_um_dm,5m,countsketch,rows=3 cols=4096,0.03841192871816221,0.05,True,562341,extrapolated beyond N=1e+07,49152.0,26.3713,5.977777777777778,5111.666666666666,0.8,100000,26594881,5 +alibaba_v2022,calls_by_um_dm,5m,topk,rows=5 cols=1024,1.0,0.95,True,5623,no merge curve for point; single-sketch curve optimistic; extrapolated beyond N=1e+07; cost scaled from rows=3,20480.0,482.89444444444445,152.5555555555556,8.333333333333334,0.8,100000,26594881,5 +alibaba_v2022,calls_by_um_dm,1h,cms,rows=3 cols=16384,0.019873590686755222,0.05,True,562341,extrapolated beyond N=1e+07,196608.0,22.537133333333333,30.955555555555556,56817.33333333332,1.0,10000000,260437245,60 +alibaba_v2022,calls_by_um_dm,1h,countsketch,rows=3 cols=4096,0.01832341853528574,0.05,True,1000000,extrapolated beyond N=1e+07,49152.0,26.381099999999996,6.355555555555556,111033.66666666667,1.0,10000000,260437245,60 +alibaba_v2022,calls_by_um_dm,1h,topk,rows=5 cols=1024,1.0,0.95,True,1778,no merge curve for point; single-sketch curve optimistic; extrapolated beyond N=1e+07; cost scaled from rows=3,20480.0,796.0973888888889,137.33333333333331,929.9999999999999,1.0,10000000,260437245,60 +alibaba_v2022,calls_by_service_um_dm,1m,cms,rows=5 cols=16384,0.11852205026294958,0.05,False,1000000,target not met; not saturated at min_N; cost scaled from rows=3,327680.0,37.68633333333333,51.18518518518519,187872.7777777778,0.8,10000000,5401423,1 +alibaba_v2022,calls_by_service_um_dm,1m,countsketch,rows=3 cols=4096,0.042239557359247955,0.05,True,5623413,not saturated at min_N,49152.0,26.516266666666667,6.133333333333334,220461.0,0.8,10000000,5401423,1 +alibaba_v2022,calls_by_service_um_dm,1m,topk,rows=5 cols=1024,0.9873968588599285,0.95,True,31623,not saturated at min_N; cost scaled from rows=3,20480.0,263.90122222222226,133.85185185185185,2103.888888888889,0.8,10000000,5401423,1 +alibaba_v2022,calls_by_service_um_dm,5m,cms,rows=5 cols=16384,0.11895150520544058,0.05,False,1000000,target not met; extrapolated beyond N=1e+07; cost scaled from rows=3,327680.0,37.68633333333333,51.18518518518519,187872.7777777778,0.8,10000000,26594881,5 +alibaba_v2022,calls_by_service_um_dm,5m,countsketch,rows=3 cols=4096,0.03865929186904125,0.05,True,5623413,,49152.0,26.516266666666667,6.133333333333334,220461.0,0.8,10000000,26594881,5 +alibaba_v2022,calls_by_service_um_dm,5m,topk,rows=5 cols=1024,0.9791666666666666,0.95,True,31623,no merge curve for point; single-sketch curve optimistic; extrapolated beyond N=1e+07; cost scaled from rows=3,20480.0,263.90122222222226,133.85185185185185,2103.888888888889,0.8,10000000,26594881,5 +alibaba_v2022,calls_by_service_um_dm,1h,cms,rows=5 cols=16384,0.11895150520544058,0.05,False,1000000,target not met; extrapolated beyond N=1e+07; cost scaled from rows=3,327680.0,37.68633333333333,51.18518518518519,187872.7777777778,0.8,10000000,260437245,60 +alibaba_v2022,calls_by_service_um_dm,1h,countsketch,rows=3 cols=4096,0.03830386459325624,0.05,True,5623413,,49152.0,26.516266666666667,6.133333333333334,220461.0,0.8,10000000,260437245,60 +alibaba_v2022,calls_by_service_um_dm,1h,topk,rows=5 cols=1024,0.9791666666666666,0.95,True,31623,no merge curve for point; single-sketch curve optimistic; extrapolated beyond N=1e+07; cost scaled from rows=3,20480.0,263.90122222222226,133.85185185185185,2103.888888888889,0.8,10000000,260437245,60 +alibaba_v2022,rt_p99_by_service,1m,cms,rows=3 cols=16384,0.02129176204509065,0.05,True,316228,not saturated at min_N,196608.0,22.54083333333333,28.933333333333334,2420.0,0.8,100000,5401423,1 +alibaba_v2022,rt_p99_by_service,1m,countsketch,rows=3 cols=4096,0.03928169571169596,0.05,True,562341,not saturated at min_N,49152.0,26.3713,5.977777777777778,5111.666666666666,0.8,100000,5401423,1 +alibaba_v2022,rt_p99_by_service,1m,topk,rows=5 cols=1024,0.9895833333333334,0.95,True,5623,not saturated at min_N; cost scaled from rows=3,20480.0,482.89444444444445,152.5555555555556,8.333333333333334,0.8,100000,5401423,1 +alibaba_v2022,rt_p99_by_service,5m,cms,rows=3 cols=16384,0.019873590686755222,0.05,True,562341,extrapolated beyond N=1e+07,196608.0,22.537133333333333,30.955555555555556,56817.33333333332,1.0,10000000,26594881,5 +alibaba_v2022,rt_p99_by_service,5m,countsketch,rows=3 cols=4096,0.01832341853528574,0.05,True,1000000,extrapolated beyond N=1e+07,49152.0,26.381099999999996,6.355555555555556,111033.66666666667,1.0,10000000,26594881,5 +alibaba_v2022,rt_p99_by_service,5m,topk,rows=5 cols=1024,1.0,0.95,True,1778,no merge curve for point; single-sketch curve optimistic; extrapolated beyond N=1e+07; cost scaled from rows=3,20480.0,796.0973888888889,137.33333333333331,929.9999999999999,1.0,10000000,26594881,5 +alibaba_v2022,rt_p99_by_service,1h,cms,rows=3 cols=16384,0.019873590686755222,0.05,True,562341,extrapolated beyond N=1e+07,196608.0,22.537133333333333,30.955555555555556,56817.33333333332,1.0,10000000,260437245,60 +alibaba_v2022,rt_p99_by_service,1h,countsketch,rows=3 cols=4096,0.01832341853528574,0.05,True,1000000,extrapolated beyond N=1e+07,49152.0,26.381099999999996,6.355555555555556,111033.66666666667,1.0,10000000,260437245,60 +alibaba_v2022,rt_p99_by_service,1h,topk,rows=5 cols=1024,1.0,0.95,True,1778,no merge curve for point; single-sketch curve optimistic; extrapolated beyond N=1e+07; cost scaled from rows=3,20480.0,796.0973888888889,137.33333333333331,929.9999999999999,1.0,10000000,260437245,60 +alibaba_v2022,rt_p99_by_service,1m,kll,k=200,0.0023037840370404346,0.01,True,not_saturated,not saturated,6400.0,25.268599999999996,5.3999999999999995,2951.9999999999995,3.0,,3311360,1 +alibaba_v2022,rt_p99_by_service,1m,dd,alpha=0.01,0.007346637486156435,0.01,True,1000,not saturated at min_N,4616.0,15.611766666666668,1.266666666666667,1987.333333333333,3.0,,3311360,1 +alibaba_v2022,rt_p99_by_service,5m,kll,k=50,0.009225104455574413,0.01,True,not_saturated,no merge curve for point; single-sketch curve optimistic; not saturated,1600.0,25.230666666666668,2.466666666666667,2202.0,3.0,,16264263,5 +alibaba_v2022,rt_p99_by_service,5m,dd,alpha=0.01,0.007349125190843246,0.01,True,1000,,4616.0,15.611766666666668,1.266666666666667,1987.333333333333,3.0,,16264263,5 +alibaba_v2022,rt_p99_by_service,1h,kll,k=200,0.0021814989781444344,0.01,True,not_saturated,no merge curve for point; single-sketch curve optimistic; not saturated,6400.0,25.645400000000002,5.177777777777778,3240.9999999999995,3.0,,157226182,60 +alibaba_v2022,rt_p99_by_service,1h,dd,alpha=0.01,0.00735024368490511,0.01,True,1000,,3464.0,15.694700000000001,1.1333333333333333,2008.3333333333333,3.0,,157226182,60 +alibaba_v2022,ms_cpu_by_msname,instant,cms,rows=3 cols=16384,0.020573849606818375,0.05,True,316228,,196608.0,22.54083333333333,28.933333333333334,2420.0,0.8,100000,471603,1 +alibaba_v2022,ms_cpu_by_msname,instant,countsketch,rows=3 cols=4096,0.04190031679674842,0.05,True,562341,not saturated at min_N,49152.0,26.3713,5.977777777777778,5111.666666666666,0.8,100000,471603,1 +alibaba_v2022,ms_cpu_by_msname,instant,topk,rows=5 cols=1024,0.9895833333333334,0.95,True,5623,cost scaled from rows=3,20480.0,482.89444444444445,152.5555555555556,8.333333333333334,0.8,100000,471603,1 +alibaba_v2022,ms_cpu_by_msname,5m,cms,rows=3 cols=16384,0.021473512975033845,0.05,True,316228,,196608.0,22.54083333333333,28.933333333333334,2420.0,0.8,100000,2355461,5 +alibaba_v2022,ms_cpu_by_msname,5m,countsketch,rows=3 cols=4096,0.0398659549922401,0.05,True,562341,,49152.0,26.3713,5.977777777777778,5111.666666666666,0.8,100000,2355461,5 +alibaba_v2022,ms_cpu_by_msname,5m,topk,rows=5 cols=1024,0.9895833333333334,0.95,True,5623,no merge curve for point; single-sketch curve optimistic; cost scaled from rows=3,20480.0,482.89444444444445,152.5555555555556,8.333333333333334,0.8,100000,2355461,5 +alibaba_v2022,ms_cpu_by_msname,1h,cms,rows=3 cols=16384,0.0213003939726821,0.05,True,316228,extrapolated beyond N=1e+07,196608.0,22.54083333333333,28.933333333333334,2420.0,0.8,100000,28254558,60 +alibaba_v2022,ms_cpu_by_msname,1h,countsketch,rows=3 cols=4096,0.03841192871816221,0.05,True,562341,extrapolated beyond N=1e+07,49152.0,26.3713,5.977777777777778,5111.666666666666,0.8,100000,28254558,60 +alibaba_v2022,ms_cpu_by_msname,1h,topk,rows=5 cols=1024,1.0,0.95,True,5623,no merge curve for point; single-sketch curve optimistic; extrapolated beyond N=1e+07; cost scaled from rows=3,20480.0,482.89444444444445,152.5555555555556,8.333333333333334,0.8,100000,28254558,60 +alibaba_v2022,ms_cpu_p99,instant,kll,k=200,0.0026133499468463093,0.01,True,not_saturated,"alpha 10.154 beyond grid, used 3; light tail; not saturated",6400.0,25.402400000000004,5.2,2999.3333333333335,3.0,,471589,1 +alibaba_v2022,ms_cpu_p99,instant,dd,alpha=0.01,0.007332013494612215,0.01,True,1000,"alpha 10.154 beyond grid, used 3; light tail",2312.0,15.584666666666665,0.8222222222222221,1965.3333333333333,3.0,,471589,1 +alibaba_v2022,ms_cpu_p99,5m,kll,k=200,0.0024364095575409063,0.01,True,not_saturated,"alpha 10.0475 beyond grid, used 3; light tail; no merge curve for point; single-sketch curve optimistic; not saturated",6400.0,25.402400000000004,5.2,2999.3333333333335,3.0,,2355395,5 +alibaba_v2022,ms_cpu_p99,5m,dd,alpha=0.01,0.007347637516570892,0.01,True,1000,"alpha 10.0475 beyond grid, used 3; light tail",2312.0,15.584666666666665,0.8222222222222221,1965.3333333333333,3.0,,2355395,5 +alibaba_v2022,ms_cpu_p99,1h,kll,k=200,0.002922404604905079,0.01,True,not_saturated,"alpha 7.05221 beyond grid, used 3; light tail; no merge curve for point; single-sketch curve optimistic; not saturated",6400.0,25.402400000000004,5.2,2999.3333333333335,3.0,,28253774,60 +alibaba_v2022,ms_cpu_p99,1h,dd,alpha=0.01,0.0073488612347692535,0.01,True,1000,"alpha 7.05221 beyond grid, used 3; light tail",2312.0,15.584666666666665,0.8222222222222221,1965.3333333333333,3.0,,28253774,60 +alibaba_v2022,node_cpu_by_nodeid,instant,cms,rows=5 cols=16384,0.19044825832269802,0.05,False,56234133,target not met; not saturated at min_N; cost scaled from rows=3,327680.0,38.0,53.370370370370374,4237.222222222223,0.0,100000,42957,1 +alibaba_v2022,node_cpu_by_nodeid,instant,countsketch,rows=5 cols=16384,0.16009119472655217,0.05,False,1778279,target not met; not saturated at min_N; cost scaled from rows=3,327680.0,44.283333333333324,48.85185185185186,8852.77777777778,0.0,100000,42957,1 +alibaba_v2022,node_cpu_by_nodeid,instant,topk,rows=5 cols=16384,0.33705578919226736,0.95,False,316228,target not met; not saturated at min_N; cost scaled from rows=3,327680.0,158.7126111111111,171.7777777777778,9.444444444444445,0.0,100000,42957,1 +alibaba_v2022,node_cpu_by_nodeid,5m,cms,rows=5 cols=16384,0.7626094093044472,0.05,False,56234133,target not met; not saturated at min_N; cost scaled from rows=3,327680.0,38.0,53.370370370370374,4237.222222222223,0.0,100000,214454,5 +alibaba_v2022,node_cpu_by_nodeid,5m,countsketch,rows=5 cols=16384,0.3023531933877328,0.05,False,1778279,target not met; not saturated at min_N; cost scaled from rows=3,327680.0,44.283333333333324,48.85185185185186,8852.77777777778,0.0,100000,214454,5 +alibaba_v2022,node_cpu_by_nodeid,5m,topk,rows=5 cols=16384,0.06638876054821932,0.95,False,316228,target not met; no merge curve for point; single-sketch curve optimistic; not saturated at min_N; cost scaled from rows=3,327680.0,158.7126111111111,171.7777777777778,9.444444444444445,0.0,100000,214454,5 +alibaba_v2022,node_cpu_by_nodeid,1h,cms,rows=5 cols=16384,1.9636232471285444,0.05,False,56234133,target not met; not saturated at min_N; cost scaled from rows=3,327680.0,38.0,53.370370370370374,4237.222222222223,0.0,100000,2570715,60 +alibaba_v2022,node_cpu_by_nodeid,1h,countsketch,rows=5 cols=16384,0.5820675430071887,0.05,False,1778279,target not met; not saturated at min_N; cost scaled from rows=3,327680.0,44.283333333333324,48.85185185185186,8852.77777777778,0.0,100000,2570715,60 +alibaba_v2022,node_cpu_by_nodeid,1h,topk,rows=3 cols=256,0.0037477525918789625,0.95,False,3162,target not met; no merge curve for point; single-sketch curve optimistic,3072.0,85.81806666666665,54.53333333333333,4.666666666666667,0.0,100000,2570715,60 +alibaba_v2022,node_cpu_p99,instant,kll,k=50,0.009710050811872485,0.01,True,not_saturated,"alpha 6.44342 beyond grid, used 3; not saturated",1600.0,25.304666666666662,2.3111111111111113,2280.6666666666665,3.0,,42955,1 +alibaba_v2022,node_cpu_p99,instant,dd,alpha=0.01,0.0073115486162062615,0.01,True,1000,"alpha 6.44342 beyond grid, used 3",2312.0,15.584666666666665,0.8222222222222221,1965.3333333333333,3.0,,42955,1 +alibaba_v2022,node_cpu_p99,5m,kll,k=200,0.0021151738322659846,0.01,True,not_saturated,"alpha 6.36875 beyond grid, used 3; no merge curve for point; single-sketch curve optimistic; not saturated",6400.0,25.402400000000004,5.2,2999.3333333333335,3.0,,214444,5 +alibaba_v2022,node_cpu_p99,5m,dd,alpha=0.01,0.007342681163634055,0.01,True,1000,"alpha 6.36875 beyond grid, used 3",2312.0,15.584666666666665,0.8222222222222221,1965.3333333333333,3.0,,214444,5 +alibaba_v2022,node_cpu_p99,1h,kll,k=200,0.0023959816287722905,0.01,True,not_saturated,"alpha 6.24654 beyond grid, used 3; no merge curve for point; single-sketch curve optimistic; not saturated",6400.0,25.402400000000004,5.2,2999.3333333333335,3.0,,2570595,60 +alibaba_v2022,node_cpu_p99,1h,dd,alpha=0.01,0.0073472721923670626,0.01,True,1000,"alpha 6.24654 beyond grid, used 3",2312.0,15.584666666666665,0.8222222222222221,1965.3333333333333,3.0,,2570595,60 +google_2011,cpu_by_user,instant,cms,rows=3 cols=1024,0.02010047453674985,0.05,True,17783,,12288.0,22.3986,1.9333333333333336,28.333333333333336,1.0,1000,212533,1 +google_2011,cpu_by_user,instant,countsketch,rows=5 cols=1024,0.03100871637085652,0.05,True,17783,cost scaled from rows=3,20480.0,43.67538888888889,3.111111111111111,93.88888888888889,1.0,1000,212533,1 +google_2011,cpu_by_user,instant,topk,rows=5 cols=256,0.9719760905645167,0.95,True,1000,cost scaled from rows=3,5120.0,1692.064388888889,153.33333333333337,8.888888888888888,1.0,1000,212533,1 +google_2011,cpu_by_user,5m,cms,rows=3 cols=1024,0.020121154813470713,0.05,True,17783,,12288.0,22.3986,1.9333333333333336,28.333333333333336,1.0,1000,379702,1 +google_2011,cpu_by_user,5m,countsketch,rows=5 cols=1024,0.031219370129271302,0.05,True,17783,cost scaled from rows=3,20480.0,43.67538888888889,3.111111111111111,93.88888888888889,1.0,1000,379702,1 +google_2011,cpu_by_user,5m,topk,rows=5 cols=256,0.9758565522801315,0.95,True,1000,cost scaled from rows=3,5120.0,1692.064388888889,153.33333333333337,8.888888888888888,1.0,1000,379702,1 +google_2011,cpu_by_user,1h,cms,rows=3 cols=1024,0.019373032639913173,0.05,True,17783,,12288.0,22.3986,1.9333333333333336,28.333333333333336,1.0,1000,4179680,12 +google_2011,cpu_by_user,1h,countsketch,rows=5 cols=1024,0.030256153260673116,0.05,True,17783,cost scaled from rows=3,20480.0,43.67538888888889,3.111111111111111,93.88888888888889,1.0,1000,4179680,12 +google_2011,cpu_by_user,1h,topk,rows=5 cols=256,0.96875,0.95,True,1000,no merge curve for point; single-sketch curve optimistic; cost scaled from rows=3,5120.0,1692.064388888889,153.33333333333337,8.888888888888888,1.0,1000,4179680,12 +google_2011,cpu_by_user_priority,instant,cms,rows=3 cols=1024,0.01169498334419633,0.05,True,5623,,12288.0,22.423800000000004,1.9333333333333336,29.666666666666668,1.2,1000,212533,1 +google_2011,cpu_by_user_priority,instant,countsketch,rows=3 cols=1024,0.03829416534185969,0.05,True,1000,,12288.0,26.283633333333334,2.0444444444444443,58.333333333333336,1.2,1000,212533,1 +google_2011,cpu_by_user_priority,instant,topk,rows=5 cols=256,0.96875,0.95,True,1000,cost scaled from rows=3,5120.0,2160.7369444444444,152.29629629629633,8.333333333333334,1.2,1000,212533,1 +google_2011,cpu_by_user_priority,5m,cms,rows=3 cols=1024,0.012121078260562697,0.05,True,5623,,12288.0,22.423800000000004,1.9333333333333336,29.666666666666668,1.2,1000,379702,1 +google_2011,cpu_by_user_priority,5m,countsketch,rows=3 cols=1024,0.039443381165268285,0.05,True,1000,,12288.0,26.283633333333334,2.0444444444444443,58.333333333333336,1.2,1000,379702,1 +google_2011,cpu_by_user_priority,5m,topk,rows=5 cols=256,0.96875,0.95,True,1000,cost scaled from rows=3,5120.0,2160.7369444444444,152.29629629629633,8.333333333333334,1.2,1000,379702,1 +google_2011,cpu_by_user_priority,1h,cms,rows=3 cols=1024,0.012067113093819451,0.05,True,5623,,12288.0,22.423800000000004,1.9333333333333336,29.666666666666668,1.2,1000,4179680,12 +google_2011,cpu_by_user_priority,1h,countsketch,rows=3 cols=1024,0.0402749337344935,0.05,True,1000,,12288.0,26.283633333333334,2.0444444444444443,58.333333333333336,1.2,1000,4179680,12 +google_2011,cpu_by_user_priority,1h,topk,rows=5 cols=256,0.96875,0.95,True,1000,no merge curve for point; single-sketch curve optimistic; cost scaled from rows=3,5120.0,2160.7369444444444,152.29629629629633,8.333333333333334,1.2,1000,4179680,12 +google_2011,cpu_by_priority,instant,cms,rows=3 cols=1024,0.01169498334419633,0.05,True,5623,,12288.0,22.423800000000004,1.9333333333333336,29.666666666666668,1.2,1000,212533,1 +google_2011,cpu_by_priority,instant,countsketch,rows=3 cols=1024,0.03829416534185969,0.05,True,1000,,12288.0,26.283633333333334,2.0444444444444443,58.333333333333336,1.2,1000,212533,1 +google_2011,cpu_by_priority,instant,topk,rows=5 cols=256,0.96875,0.95,True,1000,cost scaled from rows=3,5120.0,2160.7369444444444,152.29629629629633,8.333333333333334,1.2,1000,212533,1 +google_2011,cpu_by_priority,5m,cms,rows=3 cols=1024,0.012121078260562697,0.05,True,5623,,12288.0,22.423800000000004,1.9333333333333336,29.666666666666668,1.2,1000,379702,1 +google_2011,cpu_by_priority,5m,countsketch,rows=3 cols=1024,0.039443381165268285,0.05,True,1000,,12288.0,26.283633333333334,2.0444444444444443,58.333333333333336,1.2,1000,379702,1 +google_2011,cpu_by_priority,5m,topk,rows=5 cols=256,0.96875,0.95,True,1000,cost scaled from rows=3,5120.0,2160.7369444444444,152.29629629629633,8.333333333333334,1.2,1000,379702,1 +google_2011,cpu_by_priority,1h,cms,rows=3 cols=1024,0.012067113093819451,0.05,True,5623,,12288.0,22.423800000000004,1.9333333333333336,29.666666666666668,1.2,1000,4179680,12 +google_2011,cpu_by_priority,1h,countsketch,rows=3 cols=1024,0.0402749337344935,0.05,True,1000,,12288.0,26.283633333333334,2.0444444444444443,58.333333333333336,1.2,1000,4179680,12 +google_2011,cpu_by_priority,1h,topk,rows=5 cols=256,0.96875,0.95,True,1000,no merge curve for point; single-sketch curve optimistic; cost scaled from rows=3,5120.0,2160.7369444444444,152.29629629629633,8.333333333333334,1.2,1000,4179680,12 +google_2011,cpu_by_job_id,instant,cms,rows=3 cols=4096,0.04232304463607425,0.05,True,177828,not saturated at min_N,49152.0,22.33483333333333,5.977777777777778,2233.0,1.0,100000,212533,1 +google_2011,cpu_by_job_id,instant,countsketch,rows=3 cols=4096,0.020720519823678972,0.05,True,316228,not saturated at min_N,49152.0,26.39866666666667,6.177777777777778,5108.333333333334,1.0,100000,212533,1 +google_2011,cpu_by_job_id,instant,topk,rows=3 cols=1024,0.9583333333333334,0.95,True,1778,,12288.0,660.1273666666667,94.97777777777777,5.333333333333333,1.0,100000,212533,1 +google_2011,cpu_by_job_id,5m,cms,rows=3 cols=4096,0.04301674918691996,0.05,True,177828,not saturated at min_N,49152.0,22.33483333333333,5.977777777777778,2233.0,1.0,100000,379702,1 +google_2011,cpu_by_job_id,5m,countsketch,rows=3 cols=4096,0.019333083136004733,0.05,True,316228,not saturated at min_N,49152.0,26.39866666666667,6.177777777777778,5108.333333333334,1.0,100000,379702,1 +google_2011,cpu_by_job_id,5m,topk,rows=3 cols=1024,0.9616434477198685,0.95,True,1778,,12288.0,660.1273666666667,94.97777777777777,5.333333333333333,1.0,100000,379702,1 +google_2011,cpu_by_job_id,1h,cms,rows=3 cols=4096,0.04488667446961157,0.05,True,177828,,49152.0,22.33483333333333,5.977777777777778,2233.0,1.0,100000,4179680,12 +google_2011,cpu_by_job_id,1h,countsketch,rows=3 cols=4096,0.018399250683173572,0.05,True,316228,,49152.0,26.39866666666667,6.177777777777778,5108.333333333334,1.0,100000,4179680,12 +google_2011,cpu_by_job_id,1h,topk,rows=3 cols=1024,0.96875,0.95,True,1778,no merge curve for point; single-sketch curve optimistic,12288.0,660.1273666666667,94.97777777777777,5.333333333333333,1.0,100000,4179680,12 +google_2011,cpu_by_logical_job_name,instant,cms,rows=3 cols=4096,0.04232304463607425,0.05,True,177828,not saturated at min_N,49152.0,22.33483333333333,5.977777777777778,2233.0,1.0,100000,212533,1 +google_2011,cpu_by_logical_job_name,instant,countsketch,rows=3 cols=4096,0.020720519823678972,0.05,True,316228,not saturated at min_N,49152.0,26.39866666666667,6.177777777777778,5108.333333333334,1.0,100000,212533,1 +google_2011,cpu_by_logical_job_name,instant,topk,rows=3 cols=1024,0.9583333333333334,0.95,True,1778,,12288.0,660.1273666666667,94.97777777777777,5.333333333333333,1.0,100000,212533,1 +google_2011,cpu_by_logical_job_name,5m,cms,rows=3 cols=4096,0.04301674918691996,0.05,True,177828,not saturated at min_N,49152.0,22.33483333333333,5.977777777777778,2233.0,1.0,100000,379702,1 +google_2011,cpu_by_logical_job_name,5m,countsketch,rows=3 cols=4096,0.019333083136004733,0.05,True,316228,not saturated at min_N,49152.0,26.39866666666667,6.177777777777778,5108.333333333334,1.0,100000,379702,1 +google_2011,cpu_by_logical_job_name,5m,topk,rows=3 cols=1024,0.9616434477198685,0.95,True,1778,,12288.0,660.1273666666667,94.97777777777777,5.333333333333333,1.0,100000,379702,1 +google_2011,cpu_by_logical_job_name,1h,cms,rows=3 cols=4096,0.04488667446961157,0.05,True,177828,,49152.0,22.33483333333333,5.977777777777778,2233.0,1.0,100000,4179680,12 +google_2011,cpu_by_logical_job_name,1h,countsketch,rows=3 cols=4096,0.018399250683173572,0.05,True,316228,,49152.0,26.39866666666667,6.177777777777778,5108.333333333334,1.0,100000,4179680,12 +google_2011,cpu_by_logical_job_name,1h,topk,rows=3 cols=1024,0.96875,0.95,True,1778,no merge curve for point; single-sketch curve optimistic,12288.0,660.1273666666667,94.97777777777777,5.333333333333333,1.0,100000,4179680,12 +google_2011,cpu_by_scheduling_class,instant,cms,rows=3 cols=4096,0.016760841459045682,0.05,True,1000000,not saturated at min_N,49152.0,22.33646666666667,6.177777777777778,39.666666666666664,0.0,1000,212533,1 +google_2011,cpu_by_scheduling_class,instant,countsketch,rows=5 cols=4096,0.028718639298722665,0.05,True,562341,not saturated at min_N; cost scaled from rows=3,81920.0,43.976777777777784,10.185185185185185,117.77777777777779,0.0,1000,212533,1 +google_2011,cpu_by_scheduling_class,instant,topk,rows=5 cols=4096,0.9655239094354833,0.95,True,5623,cost scaled from rows=3,81920.0,231.64333333333335,122.81481481481482,8.888888888888888,0.0,1000,212533,1 +google_2011,cpu_by_scheduling_class,5m,cms,rows=3 cols=4096,0.013143994005858778,0.05,True,1000000,not saturated at min_N,49152.0,22.33646666666667,6.177777777777778,39.666666666666664,0.0,1000,379702,1 +google_2011,cpu_by_scheduling_class,5m,countsketch,rows=5 cols=4096,0.02563823434458546,0.05,True,562341,not saturated at min_N; cost scaled from rows=3,81920.0,43.976777777777784,10.185185185185185,117.77777777777779,0.0,1000,379702,1 +google_2011,cpu_by_scheduling_class,5m,topk,rows=5 cols=4096,0.971573790879474,0.95,True,5623,cost scaled from rows=3,81920.0,231.64333333333335,122.81481481481482,8.888888888888888,0.0,1000,379702,1 +google_2011,cpu_by_scheduling_class,1h,cms,rows=3 cols=4096,0.013026943752789035,0.05,True,1000000,,49152.0,22.33646666666667,6.177777777777778,39.666666666666664,0.0,1000,4179680,12 +google_2011,cpu_by_scheduling_class,1h,countsketch,rows=5 cols=4096,0.017628240253642864,0.05,True,562341,cost scaled from rows=3,81920.0,43.976777777777784,10.185185185185185,117.77777777777779,0.0,1000,4179680,12 +google_2011,cpu_by_scheduling_class,1h,topk,rows=5 cols=4096,0.9845357075646095,0.95,True,5623,no merge curve for point; single-sketch curve optimistic; cost scaled from rows=3,81920.0,231.64333333333335,122.81481481481482,8.888888888888888,0.0,1000,4179680,12 +google_2011,cpu_by_machine_id,instant,cms,rows=5 cols=16384,0.7582599880170395,0.05,False,56234133,target not met; not saturated at min_N; cost scaled from rows=3,327680.0,38.0,53.370370370370374,4237.222222222223,0.0,100000,212533,1 +google_2011,cpu_by_machine_id,instant,countsketch,rows=5 cols=16384,0.30138674217296635,0.05,False,1778279,target not met; not saturated at min_N; cost scaled from rows=3,327680.0,44.283333333333324,48.85185185185186,8852.77777777778,0.0,100000,212533,1 +google_2011,cpu_by_machine_id,instant,topk,rows=5 cols=16384,0.06720288051074944,0.95,False,316228,target not met; not saturated at min_N; cost scaled from rows=3,327680.0,158.7126111111111,171.7777777777778,9.444444444444445,0.0,100000,212533,1 +google_2011,cpu_by_machine_id,5m,cms,rows=5 cols=16384,1.0326638561629764,0.05,False,56234133,target not met; not saturated at min_N; cost scaled from rows=3,327680.0,38.0,53.370370370370374,4237.222222222223,0.0,100000,379702,1 +google_2011,cpu_by_machine_id,5m,countsketch,rows=5 cols=16384,0.36704286832769284,0.05,False,1778279,target not met; not saturated at min_N; cost scaled from rows=3,327680.0,44.283333333333324,48.85185185185186,8852.77777777778,0.0,100000,379702,1 +google_2011,cpu_by_machine_id,5m,topk,rows=5 cols=16384,0.024629771226929636,0.95,False,316228,target not met; not saturated at min_N; cost scaled from rows=3,327680.0,158.7126111111111,171.7777777777778,9.444444444444445,0.0,100000,379702,1 +google_2011,cpu_by_machine_id,1h,cms,rows=5 cols=16384,2.1662410274416684,0.05,False,56234133,target not met; not saturated at min_N; cost scaled from rows=3,327680.0,38.0,53.370370370370374,4237.222222222223,0.0,100000,4179680,12 +google_2011,cpu_by_machine_id,1h,countsketch,rows=5 cols=16384,0.6104741039391377,0.05,False,1778279,target not met; not saturated at min_N; cost scaled from rows=3,327680.0,44.283333333333324,48.85185185185186,8852.77777777778,0.0,100000,4179680,12 +google_2011,cpu_by_machine_id,1h,topk,rows=3 cols=4096,0.005047625768723855,0.95,False,56234,target not met; no merge curve for point; single-sketch curve optimistic,49152.0,83.29363333333333,65.57777777777778,5.333333333333333,0.0,100000,4179680,12 +google_2011,cpu_p99,instant,kll,k=200,0.002039960011051183,0.01,True,not_saturated,"alpha 21.4807 beyond grid, used 3; not saturated",6400.0,25.645400000000002,5.177777777777778,3240.9999999999995,3.0,,138139,1 +google_2011,cpu_p99,instant,dd,alpha=0.01,0.007329712681478714,0.01,True,1000,"alpha 21.4807 beyond grid, used 3",3464.0,15.694700000000001,1.1333333333333333,2008.3333333333333,3.0,,138139,1 +google_2011,cpu_p99,5m,kll,k=200,0.0022097160036066922,0.01,True,not_saturated,"alpha 20.2786 beyond grid, used 3; not saturated",6400.0,25.645400000000002,5.177777777777778,3240.9999999999995,3.0,,277263,1 +google_2011,cpu_p99,5m,dd,alpha=0.01,0.007336747808287365,0.01,True,1000,"alpha 20.2786 beyond grid, used 3",3464.0,15.694700000000001,1.1333333333333333,2008.3333333333333,3.0,,277263,1 +google_2011,cpu_p99,1h,kll,k=200,0.002321227973282705,0.01,True,not_saturated,"alpha 16.2919 beyond grid, used 3; no merge curve for point; single-sketch curve optimistic; not saturated",6400.0,25.645400000000002,5.177777777777778,3240.9999999999995,3.0,,3021637,12 +google_2011,cpu_p99,1h,dd,alpha=0.01,0.007346596686089427,0.01,True,1000,"alpha 16.2919 beyond grid, used 3",3464.0,15.694700000000001,1.1333333333333333,2008.3333333333333,3.0,,3021637,12 +google_2011,memory_p99,instant,kll,k=200,0.002039654412650209,0.01,True,not_saturated,"alpha 17.2179 beyond grid, used 3; light tail; not saturated",6400.0,25.268599999999996,5.3999999999999995,2951.9999999999995,3.0,,136461,1 +google_2011,memory_p99,instant,dd,alpha=0.01,0.007328875873422717,0.01,True,1000,"alpha 17.2179 beyond grid, used 3; light tail",4616.0,15.611766666666668,1.266666666666667,1987.333333333333,3.0,,136461,1 +google_2011,memory_p99,5m,kll,k=200,0.0021330932787594088,0.01,True,not_saturated,"alpha 17.4546 beyond grid, used 3; light tail; not saturated",6400.0,25.268599999999996,5.3999999999999995,2951.9999999999995,3.0,,225145,1 +google_2011,memory_p99,5m,dd,alpha=0.01,0.007341556560272891,0.01,True,1000,"alpha 17.4546 beyond grid, used 3; light tail",4616.0,15.611766666666668,1.266666666666667,1987.333333333333,3.0,,225145,1 +google_2011,memory_p99,1h,kll,k=200,0.002418813325501822,0.01,True,not_saturated,"alpha 16.0911 beyond grid, used 3; light tail; no merge curve for point; single-sketch curve optimistic; not saturated",6400.0,25.268599999999996,5.3999999999999995,2951.9999999999995,3.0,,2446753,12 +google_2011,memory_p99,1h,dd,alpha=0.01,0.007347478509427665,0.01,True,1000,"alpha 16.0911 beyond grid, used 3; light tail",4616.0,15.611766666666668,1.266666666666667,1987.333333333333,3.0,,2446753,12 diff --git a/docs/figures/saturation/reduced_run/README.md b/docs/figures/saturation/reduced_run/README.md new file mode 100644 index 00000000..2095ac3c --- /dev/null +++ b/docs/figures/saturation/reduced_run/README.md @@ -0,0 +1,136 @@ +# How to read the error-vs-N figures + +These figures come from the **reduced run** in `docs/saturation_study.md`: one +config per sketch (CMS/CountSketch/top-k rows=3 cols=1024, HLL lg_k=12, +KLL k=200, DDSketch alpha=0.01), N up to 1e7, 3 seeds (10 seeds for KLL and +DDSketch), Pareto values at f64. `cost_cpu_memory.png` (see `README_cost.md`) +is the serial cost of the same run. The config-grid / 1e9 figures are in +`../grid_1e9/`. + +Each figure has one sketch at one config. Each panel shows one data distribution: +- **Rows** are the number of distinct keys, K = 1K / 100K / 10M. +- **Columns** are the Zipf skew θ = 0 … 2. For quantile sketches the columns are the Pareto tail index α instead. + +What the marks mean: +- The x-axis is the number of items inserted, N, on a log scale. +- The blue line is the mean error over 3 seeds, and the band is ±2 standard errors. +- The dotted gray line is the theoretical bound, whose value is printed in the panel title. +- The dashed orange line is N_saturation: the first N after which every later error stays within max(10% of the plateau, 2 SE) of the mean of the last three points. + +Every panel has its own y-scale. + +**Common theme.** None of the six theoretical bounds depends on N. For every sketch, the error normalized by N (or by the key's frequency) is bounded by a quantity that depends only on the sketch configuration and on the distribution. That distribution is θ and K, or α. Where the empirical error still moves with N, the cause is one of two finite-N effects: +- **Sampling noise in the ground truth.** When N is small, the "true" ranks and counts are themselves noisy. +- **The tail filling in.** While N ≲ K, new keys keep arriving, and they add collision mass the sketch has not seen yet. + +N_saturation measures how long these effects last. + +Caveat on N_sat: the rule compares against the mean of the last three points. When a curve is still trending at N = 10⁷, the marker can land too early. Read the curve, not just the marker, and treat "not saturated" as "not by 10⁷", not as "never". + +--- + +## CMS (rows=3, cols=1024) · `are_top100` + +**Metric.** The mean relative error |f̂ − f| / f over the true top-100 keys. Lower is better. + +**Bound.** CMS guarantees f̂ − f ≤ εN with ε = e/w (probability ≥ 1 − e⁻³). Since f_i ≈ N·p_i, key i's relative error is ≤ ε/p_i, and the bound is the mean of that over the top 100. It is **N-independent** and depends on θ and K. + +**What the panels show:** +1. **Rise, then plateau.** CMS only ever over-counts, and the over-count comes from all the other keys hashed into the same cell. At small N few keys have arrived, so there are few collisions. The error climbs as the other keys fill the cells, then flattens once the collision mass grows in proportion to N. N_sat marks that point, typically 10⁴–10⁶. +2. **The plateau falls steeply with θ.** + - K = 100K: about 3.6 at θ = 0.5, 0.25 at θ = 1, and about 0.005 at θ = 2. + - With more skew, the top-100 keys hold more of the mass, so the same collision noise is small relative to their counts. + - **For CMS, the worst case is the lower θ bound.** +3. **At θ = 0 with K ≥ 100K the error never saturates.** It reaches 70 (K = 100K) and 1,200 (K = 10M) at N = 10⁷ and is still rising. Under a uniform distribution the true top-100 counts only grow once N ≫ K, while the collision noise grows with N from the start. The sketch cannot saturate before N is several times K. +4. **At high θ the error is ≈ 0 at small N and rises late.** For example, N_sat ≈ 2×10⁶ at θ = 2. Very few distinct keys have appeared, so nothing collides until the rare tail keys start arriving. +5. **The bound is very loose at high θ.** It is 14.8 at θ = 2 against a measured 0.005, because it is set by the rank-100 key's tiny p_i. Use it as a worst-case ceiling, not as a predictor. + +**Takeaway for configuration.** Size w for the lowest θ the workload can show. Also make sure the per-window N is well past N_sat. For near-uniform keys with large K, CMS with w = 1024 is unusable. + +--- + +## CountSketch (rows=3, cols=1024) · `are_top100` + +**Metric.** Same as CMS. + +**Bound.** |f̂ − f| ≤ ε·‖f‖₂ with ε = √(3/w). The relative bound is ε·‖p‖₂/p_i, averaged over the top 100. It is **N-independent** and depends on θ and K. + +**What the panels show:** +1. **A different shape from CMS: a hump, then decline to the plateau.** This is most visible at K = 10M, θ = 0.5–1. + - CountSketch uses signed counters, so collisions cancel on average. Its error scales with the L2 norm of the counts, not their sum. + - With finite N, E[‖f‖₂²] ≈ N²·Σp² + N. The relative error therefore behaves like √(Σp² + 1/N)/p_i. + - While 1/N dominates Σp², which happens at small N and large K, the error **falls as more data arrives**. It settles on the N-independent plateau once N ≫ 1/Σp². + - The initial rise at the smallest N is the same few-keys-seen effect as CMS. +2. **θ = 0 grows with N and never saturates.** The error is 0.35 at K = 1K, 2.4 at 100K and 5 at 10M by N = 10⁷. Under a uniform distribution the top-100 counts stay near the Poisson noise level until N ≫ K, while ‖f‖₂ grows like √N. +3. **The plateau is much lower than CMS for low θ.** At θ = 0.5, K = 100K it is about 0.33 against CMS's 3.6. This is the L2-vs-L1 advantage on flat distributions. For θ ≥ 1.5 both sketches sit near 0.01–0.03. +4. **Measured values stay well below the bound** everywhere (the bound is off-scale at θ = 2). + +**Takeaway.** For low-skew, high-cardinality keys, CountSketch beats CMS at the same memory. It also needs a *minimum* N (of order 1/Σp²) before its error drops. A short window x can therefore hurt CountSketch in a way it does not hurt CMS. + +--- + +## CMS-heap top-k (rows=3, cols=1024, heap = 32) · `precision@32` + +**Metric.** The share of the 32 reported keys that are in the true top 32. **Higher is better.** + +**Bound (lower bound).** The share of true top-32 keys with p_i − p₃₃ > e/w. Such a key cannot be pushed below the 33rd key by CMS over-estimation. The bound is **N-independent**, because both the gap and the CMS error scale with N. The curves should stay above it, and they all do. + +**What the panels show:** +1. **θ = 0: precision ≈ 0 and the bound is 0.** Under a uniform distribution the "true top 32" is just random fluctuation, far below the CMS noise. Top-k is ill-defined here; the sketch is not at fault. The 0.6 at K = 1K and small N comes from ties among keys seen once or twice. +2. **θ ≥ 0.8 with K ≤ 100K: precision ≈ 1 almost immediately** (N_sat ≈ 10³–10⁴). The heads stand far above rank 33. +3. **θ = 0.5, or K = 10M: precision rises with N.** For example, θ = 0.5, K = 10M goes from 0 below 10⁵ to 0.33 at 10⁷ and is not saturated. When N ≪ K most keys have been seen 0–1 times, so the true ranking is itself noise, and the head only emerges with more data. +4. **K = 10M, θ = 0.8–1.2: precision rises, then falls after N ≈ 10⁶** (θ = 1: 0.97 → 0.72). The 10M-key tail only fills in as N approaches K. Its growing collision mass inflates tail keys' estimates, and those tail keys evict the true rank 25–32 keys, whose gap to rank 33 is small. These curves are not saturated by 10⁷, even where an N_sat marker is drawn (θ = 1.2). +5. **θ ≥ 1.5: ≈ 1 everywhere.** The lower bound (0.41–0.56) is loose. + +**Takeaway.** +- Use the lower θ bound. +- Top-k with θ ≤ 0.5 is not meaningful. +- For large K near θ ≈ 1, precision degrades as the per-window N approaches K. Either widen w or shorten the window. +- Separately, insert CPU rises steeply with θ: about 150 → 2,100 ns/item (see the cost figure). + +--- + +## HLL (lg_k = 12, m = 4096) · `relative_error` + +**Metric.** |estimate − true distinct count| / true distinct count. + +**Bound.** Standard error σ = 1.04/√m = 1.63%, so the dotted line is at 2σ = 3.25% (about 95% coverage). It is **independent of N**. HLL's error depends only on the number of distinct keys D seen so far. Below about 2.5m ≈ 10K distinct keys, HLL switches to linear counting, which is more accurate. + +**What the panels show:** +1. **K = 1K: the error becomes exactly constant (≈ 1.1%, band collapses to 0) once all 1,000 keys have been seen.** HLL is deterministic in the key set, and every seed eventually inserts the same set. More N changes nothing. +2. **K ≥ 100K: the error wanders within the 2σ band at every N.** D keeps growing, and each new D gives a different, essentially random estimate error of about σ. There is no real trend with N. The N_sat markers here mostly reflect noise, not saturation. +3. **θ = 2: the error is 0 at small N.** Only a handful of distinct keys have appeared, and linear counting is exact. +4. All points are within the 3.25% bound except one point, which touches it. That is expected at 95% coverage. + +**Takeaway.** For HLL, "N_saturation" should be read as "the distinct count has stopped growing". The accuracy is set by lg_k alone, and the question for configuration is only the D per window (memory is fixed at 4 KB here). + +--- + +## KLL (k = 200) · `mean_rank_err` (10 seeds) + +**Metric.** Mean over the 101 quantiles 0.00…1.00 of |rank(estimate) − q·N| / N. + +**Bound.** Normalized rank error ε ≈ 2.296/k^0.9723 = 1.33% (DataSketches formula, 99% confidence, single quantile) — **N-independent** (the error is already normalized by N) and **distribution-independent** (KLL is comparison-based). + +**What the panels show.** +1. **Lower at the smallest N** (few compactions), then **flat at ≈ 3×10⁻³ from about N ≈ 5×10³** — roughly 4× below the bound. +2. **No dependence on α**: all four columns sit at the same level, as theory predicts — the tail shape does not matter to KLL's rank error. +3. **The N_sat markers (2×10⁵ – 2×10⁶) are later than the eye would place them (≈ 5×10³).** Even with 10 seeds the curve wiggles ±15% around the plateau — KLL's compaction randomness at each N is larger than 2 SE of the seed mean — so the "every later point in band" rule keeps getting broken until near the end. For KLL, read the plateau from the curve; the rule's N_sat is conservative. + +**Takeaway.** KLL's accuracy is set by k and is reached after a few thousand items; Pareto α affects neither its error nor (see the cost figure) its memory or CPU. + +--- + +## DDSketch (α_dd = 0.01) · `mean_rank_err` (10 seeds) + +**Metric.** Same as KLL. Note DDSketch's own guarantee is on **value** error (relative error ≤ α_dd), not rank error. + +**Bound (derived here, not quoted from a paper).** A relative value error α_dd at value x shifts rank by ≈ pdf(x)·α_dd·x; for Pareto(a), pdf(x)·x = a·(1 − q), so averaged over the quantile grid: mean rank error ≤ α_dd·a/2. **N-independent**, grows with a. + +**What the panels show.** +1. **Error falls slightly with N and is flat by 10³–10⁴**; N_sat = 10³ – 3×10³, the earliest of all sketches. The 10-seed band is very tight. +2. **Error increases with α**: 2.4e-3 (α = 1.1), 3.8e-3 (1.5), 4.9e-3 (2), 7.2e-3 (3), tracking the bound's ∝ a. A lighter tail packs more mass near the minimum, so the same relative value error spans more ranks. **For DDSketch rank error, heavier tails are easier.** +3. **Memory goes the other way** (cost figure): 6.3 KB at α = 1.1 down to 2.3 KB at α = 3 — a heavier tail spans a wider value range and needs more buckets. +4. Measured values are ≈ 2× below the bound everywhere. + +**Takeaway.** DDSketch saturates almost immediately. For configuration, use the *upper* α bound (lightest tail) from task 2 for rank error and the *lower* α bound (heaviest tail) for memory. diff --git a/docs/figures/saturation/reduced_run/README_cost.md b/docs/figures/saturation/reduced_run/README_cost.md new file mode 100644 index 00000000..78326dc8 --- /dev/null +++ b/docs/figures/saturation/reduced_run/README_cost.md @@ -0,0 +1,28 @@ +# How to read `cost_cpu_memory.png` + +Serial measurements at N = 10⁷ (one process at a time, 3 runs + 1 warm-up, 16 merge shards, seed 1). Columns are sketches; rows are: + +1. **insert CPU (ns / item)** — phase user+sys CPU / items inserted. +2. **merge CPU (µs / fold)** — one fold merges one shard into the accumulator (15 folds for 16 shards). +3. **query CPU (µs / query)** — per point query (frequency / top-k / cardinality) or per quantile probe. +4. **memory_bytes** — the sketch's own size formula for one instance. + +x-axis is Zipf θ (one line per K) or Pareto α. + +## What it shows + +| sketch | insert | merge | query | memory | depends on θ / K / α? | +|---|---|---|---|---|---| +| CMS 3×1024 | 27 ns | ≈ 2 µs | ≈ 0.03 µs | 12 KB | **no** (flat; merge spikes of ~1 µs are timer noise at µs scale) | +| CountSketch 3×1024 | 31 ns | ≈ 2 µs | ≈ 0.06 µs | 12 KB | **no** | +| CMS-heap top-k 3×1024, heap 32 | **100 → 2,100 ns** | 70 – 145 µs | ≈ 0 – 2,300 µs | 12 KB | **yes, strongly**: insert cost grows ~20× with θ (more heap updates as heavy keys keep changing rank); query cost is large only at K = 10M with low θ (heap churn), ~0 otherwise | +| HLL lg_k=12 | 4 ns | 3 – 8 µs | 20 µs (K=1K) / 45 – 65 µs (K ≥ 100K) | 4 KB | insert **no**; query/merge depend on K (register fill), not θ | +| KLL k=200 | 32 ns | ≈ 7 µs | ≈ 33 µs | 6.4 KB | **no** (comparison-based) | +| DDSketch α=0.01 | 20 → 27 ns | 1 – 3 µs | 21 – 27 µs | **6.3 → 2.3 KB** | memory **yes**: heavier tail (small α) → wider value range → more buckets | + +## Using it with the error figures + +- Per-item insert cost is N-independent for every sketch, so ingest CPU for a deployment is `arrival rate × (x / y) × insert ns`, as in PR #129's cost model. +- Merge cost per fold is set by the instance size, not N; a query over lookback S merging m = S/x instances costs `(m − 1) × merge µs` plus one query. +- The two distribution-sensitive costs to plan for are **top-k insert at high θ** (use the *upper* θ bound for CPU sizing — the opposite end from accuracy) and **DDSketch memory at heavy tails** (use the *lower* α bound). +- Four top-k points (θ = 1.0 at all K, θ = 1.2 at K = 1K) were re-measured serially after an overlap with another job; KLL/DDSketch costs come from the 10-seed run. diff --git a/docs/figures/saturation/reduced_run/cost_cpu_memory.png b/docs/figures/saturation/reduced_run/cost_cpu_memory.png new file mode 100644 index 00000000..18372847 Binary files /dev/null and b/docs/figures/saturation/reduced_run/cost_cpu_memory.png differ diff --git a/docs/figures/saturation/reduced_run/error_vs_N__cms-fastpath-vector2d.png b/docs/figures/saturation/reduced_run/error_vs_N__cms-fastpath-vector2d.png new file mode 100644 index 00000000..ebf3f36c Binary files /dev/null and b/docs/figures/saturation/reduced_run/error_vs_N__cms-fastpath-vector2d.png differ diff --git a/docs/figures/saturation/reduced_run/error_vs_N__cms-heap-topk-fastpath-vector2d.png b/docs/figures/saturation/reduced_run/error_vs_N__cms-heap-topk-fastpath-vector2d.png new file mode 100644 index 00000000..f0821bf1 Binary files /dev/null and b/docs/figures/saturation/reduced_run/error_vs_N__cms-heap-topk-fastpath-vector2d.png differ diff --git a/docs/figures/saturation/reduced_run/error_vs_N__countsketch-fastpath-vector2d.png b/docs/figures/saturation/reduced_run/error_vs_N__countsketch-fastpath-vector2d.png new file mode 100644 index 00000000..ad399be0 Binary files /dev/null and b/docs/figures/saturation/reduced_run/error_vs_N__countsketch-fastpath-vector2d.png differ diff --git a/docs/figures/saturation/reduced_run/error_vs_N__dd.png b/docs/figures/saturation/reduced_run/error_vs_N__dd.png new file mode 100644 index 00000000..f482481a Binary files /dev/null and b/docs/figures/saturation/reduced_run/error_vs_N__dd.png differ diff --git a/docs/figures/saturation/reduced_run/error_vs_N__hll.png b/docs/figures/saturation/reduced_run/error_vs_N__hll.png new file mode 100644 index 00000000..c2b61ae1 Binary files /dev/null and b/docs/figures/saturation/reduced_run/error_vs_N__hll.png differ diff --git a/docs/figures/saturation/reduced_run/error_vs_N__kll-percall.png b/docs/figures/saturation/reduced_run/error_vs_N__kll-percall.png new file mode 100644 index 00000000..15fea6b2 Binary files /dev/null and b/docs/figures/saturation/reduced_run/error_vs_N__kll-percall.png differ diff --git a/docs/saturation_conclusions.md b/docs/saturation_conclusions.md new file mode 100644 index 00000000..ed7baa55 --- /dev/null +++ b/docs/saturation_conclusions.md @@ -0,0 +1,195 @@ +# Saturation study: conclusions and why + +This page has two parts: + +- **Conclusions** across three studies: + - Task 1–2: worst-case data parameters from real traces (ProjectASAP/ASAPQuery#746). + - Task 3: the sketch saturation and cost study (this PR). + - Accuracy after merging (#131). +- **The math** behind three of the results. + +Method, tables and figures: [`saturation_study.md`](saturation_study.md) and +[`figures/saturation/`](figures/saturation/). + +Notation: + +| Symbol | Meaning | +|---|---| +| θ | Zipf skew of the keys | +| K | number of distinct keys | +| N | number of items inserted | +| a | Pareto tail index of the values | +| α_dd | DDSketch's relative-accuracy parameter (`alpha=` in configs) | +| k | KLL size parameter | +| w | CMS / CountSketch width (cols) | +| m | HLL register count (2^lg_k) when talking about HLL; otherwise the number of shards merged | + +## Conclusions + +### Real data (tasks 1–2) + +1. **Real group-by keys are skewed, with θ ≈ 1.0–1.5.** This covers Google user / job / priority and Alibaba service / msname / um / dm. Only per-machine or per-node keys, where one key is one series, are near-uniform (θ ≈ 0–0.3). +2. **The worst case is much worse than the average, and longer samples expose it.** For Google `by (user)`, the lower θ bound is 1.22 on a 1.4 h sample but 1.10 on a 7-day sample with 5-minute evaluations. Fitting on a short sample, or on average behaviour, undersizes the sketch. +3. **Instant and range queries see different distributions.** An instant query counts series per group; a range query counts samples per group. For MSRTMCR `by (msname)`, θ is 0.83 for the instant query and 1.44 for 5m/1h ranges, and the instant query needs a 4× larger CMS. Fits must follow the query's semantics. +4. **K per window depends on the time scale.** CallGraph `by (service)` has about 40K keys per minute and 2.77M over 6 h. Google `by (user)` stays at about 500–700. +5. **Most value tails are lognormal or light; few are truly heavy.** The clear heavy tails are call latency `rt` (a ≈ 2) and a few BOOM series. Power law vs lognormal is often undecidable, so the analysis reports a tail class (`light` / `power_law` / `lognormal` / `heavy_inconclusive`) rather than a pass/fail power-law test. + +### Sketches (task 3) + +6. **No sketch's theoretical error bound depends on N.** Error that still moves with N comes from two small-N effects: + - the ground truth itself is noisy when N is small; + - while N < K, new tail keys keep arriving. + + Past N_sat, error is set by the config alone, so a profile from a small window extrapolates to larger windows. +7. **N_sat is mostly 1e3–1e6, with two exceptions:** + - With uniform keys and large K, CMS and CountSketch never saturate: at K = 1e7 the error is still rising at 1e9. + - With large K, top-k precision keeps improving up to 1e9. + + So an N_sat measured only to 1e7 is a lower bound when K is large. +8. **Error scales with config size as theory predicts, so configs can be interpolated:** + + | Sketch | Error scales as | + |---|---| + | CMS | 1/w (steeper with skew) | + | CountSketch | 1/√w (about w^(−θ) with skew) | + | HLL | 1/√m | + | KLL | 1/k | + | DDSketch | α_dd | + +9. **The data distribution decides which sketch to use:** + - Low θ is the worst case for CMS and top-k, so size them from the lower θ bound. + - At low skew and large K, CountSketch beats CMS at the same memory. For CallGraph, CountSketch needs 48 KB where CMS needs 192 KB. + - Top-k is unreliable for θ ≤ 0.5. + - KLL's error does not depend on the distribution (why: section 1 below). + - DDSketch's rank error grows with a while its memory shrinks with a. Size the error from the upper a bound and the memory from the lower a bound (why: section 2 below). +10. **Merging (#131):** + - CMS, CountSketch, HLL and DDSketch merge exactly. + - KLL's error rises after merging, more for larger k: about 1.0–1.1× at k = 50/200 and 1.12–1.32× at k = 800 at N = 1e7, and up to 3–4× at N = 1e4–1e5 for k = 800 merged from 4 shards. + - Top-k is unchanged in the median, but loses up to 40% precision at large K, for every width. + + So single-sketch curves are optimistic for top-k and KLL on merged windows (why: section 3 below). +11. **Cost:** + - Per-item insert CPU is flat in N and in the distribution: CMS ≈ 22 ns, CountSketch ≈ 26 ns, HLL ≈ 3.4 ns, KLL ≈ 25 ns, DDSketch ≈ 16 ns. + - The exception is CMS-heap top-k, which grows from about 100 ns at θ = 0 to 1,750 ns at θ = 2. + - Memory is fixed by the config, except DDSketch's, which grows with heavier tails. +12. **Sketch vs exact computation.** The exact baseline buffers the samples and computes at query time. + - **Memory:** sketches save 10× after a few thousand items and 100× after 1e4–1e6 items, depending on sketch size. + - **CPU:** + - CMS and CountSketch are about 13× cheaper, HLL about 40×; neither reaches 100×. + - Quantile sketches pass 100× from 1e5–1e6 items, because exact computation sorts. + - Top-k at high skew is slower per item than exact; its benefit is memory only. + - Real windows hold millions of items, so sketches save at least one to two orders of magnitude of memory. +13. **Worst-case parameters → configuration is automated** in `scripts/recommend_config.py`. Examples: + + | Query | Recommendation | + |---|---| + | Google `by (user)` | CMS 3×1024 (12 KB) | + | CallGraph `by (service)` | CountSketch 3×4096 (48 KB) | + | MSRTMCR, instant query | 192 KB | + | p99 latency | DDSketch α_dd = 0.01 (3–5 KB) | + +**Limits:** +- Some recommendations extrapolate beyond N = 1e7. +- Merged accuracy covers every top-k and KLL config, but only up to N = 1e7 and m = 64 shards. +- The exact baseline buffers the stream. A streaming hash-map counter would shrink the frequency sketches' memory advantage. + +## Why: the math + +### 1. KLL's rank error does not depend on the value distribution + +**KLL only compares items.** Each compactor sorts its buffer and keeps the odd- or even-indexed half, chosen by a coin flip; the kept half moves up one level. Every step depends only on the relative order of items and on the coins, never on their values. + +**So a strictly increasing transform g leaves the sketch unchanged.** Replace every x by g(x) and every comparison, sort and discard happens the same way. The item returned for a quantile is g of the item that would have been returned before. + +**Every continuous distribution therefore behaves like the uniform one.** Let the X_i be i.i.d. with continuous CDF F, and take g = F. Then U_i = F(X_i) ~ Uniform(0, 1), and the sketch state on X equals the sketch state on U. + +**Rank error is defined on ranks**, err(q) = |rank(x̂) − qN| / N, and a monotone transform preserves ranks. So the rank error on any continuous F has exactly the same distribution as on uniform data. It depends only on k, N and the coins. + +**The bound has no F in it:** with probability at least 1 − δ, err ≤ ε with ε = O((1/k)·√log(1/δ)). DataSketches' empirical form is ε ≈ 2.296 / k^0.9723, which is 1.33% at k = 200. + +**Measured:** at k = 200 the mean rank error is about 3×10⁻³ for all four Pareto tails (a = 1.1, 1.5, 2, 3). + +**Assumptions:** +- *No ties.* Discrete data creates ties; we floor Pareto values with scale 1000, so ties are rare. +- *Non-adversarial arrival order.* i.i.d. order is exchangeable. + +KLL's memory, O(k·log(N/k)), is also distribution-free. + +### 2. DDSketch: rank error ∝ a, memory ∝ ln N / a + +**How DDSketch works.** It puts x > 0 into bucket ⌈log_γ x⌉ with γ = (1 + α_dd) / (1 − α_dd), and answers a quantile with its bucket's representative x̂. This guarantees |x̂ − x_q| ≤ α_dd·x_q, a bound on the *value* error. + +**From value error to rank error.** Near x_q, rank changes at rate N·f(x_q), where f is the density. So, normalized by N: + + rank error(q) ≈ f(x_q)·|x̂ − x_q| ≤ α_dd·x_q·f(x_q). + +For Pareto(a, x_m), F(x) = 1 − (x_m/x)^a and f(x) = a·x_m^a·x^(−a−1). Therefore + + **x·f(x) = a·(x_m/x)^a = a·(1 − F(x)) = a·(1 − q)**, + +so rank error(q) ≤ α_dd·a·(1 − q). Averaged over the 101-point grid q ∈ [0, 1]: + + **mean rank error ≤ α_dd·a / 2.** + +Intuition: a larger a means a lighter tail. More of the data sits near x_m, and a bucket whose width is proportional to x then spans more ranks. + +Measured at α_dd = 0.01 (bound in parentheses): + +| a | 1.1 | 1.5 | 2 | 3 | +|---|---|---|---|---| +| mean rank error | 2.4e-3 (≤ 5.5e-3) | 3.8e-3 (≤ 7.5e-3) | 4.9e-3 (≤ 1.0e-2) | 7.2e-3 (≤ 1.5e-2) | + +The fitted log-log slope against α_dd is 0.98–1.08, as the bound predicts. + +**Memory: the bucket count follows the value range.** + + buckets ≈ log_γ(x_max / x_min) = ln(x_max / x_min) / ln γ ≈ ln(x_max / x_min) / (2·α_dd). + +- **Minimum:** x_min ≈ x_m. +- **Maximum:** the maximum of N Pareto draws has P(max ≤ x) = (1 − (x_m/x)^a)^N, so typically x_max ≈ x_m·N^(1/a). + +Substituting: + + **buckets ≈ ln N / (2·a·α_dd).** + +At N = 1e7 and α_dd = 0.01 this predicts about 733 buckets at a = 1.1 and 269 at a = 3, a ratio of 2.7. The measured memory is 6.3 KB vs 2.3 KB, also 2.7. + +**Which bound to use for sizing:** + +| Concern | Dependence on a | Worst case | Use | +|---|---|---|---| +| Rank error | ∝ a | largest a (lightest tail) | upper a bound | +| Memory | ∝ ln N / a | smallest a (heaviest tail) | lower a bound | + +Memory also grows like ln N, so a longer window needs slightly more of it. + +### 3. Why merging is exact for CMS / HLL / DDSketch but not for KLL (or top-k) + +**CMS, CountSketch, HLL and DDSketch:** the sketch state is a deterministic function of the multiset of items, and merging is associative. +- CMS and CountSketch merge by adding counters (with the same hash functions). +- HLL takes the register-wise max. +- DDSketch adds bucket counts. + +Merging m shards therefore produces exactly the sketch you'd get from feeding all N items into one. Measured: 0 of 1,479 cells differ. + +**KLL: randomized compaction.** + +Where its error comes from: at level h, each item stands for 2^h inputs. A compaction at level h moves any rank by at most 2^h, with a random ± sign of mean 0. The total error is a sum of independent zero-mean terms: + + err(x) = Σ_compactions ±2^h·𝟙[the compaction affects x], + **Var(err) ≈ Σ_h C_h·(2^h)²**, + +where C_h is the number of compactions at level h. KLL's geometrically shrinking capacities keep this sum ≲ (N/k)², i.e. normalized error ≈ 1/k. Most of the variance comes from the top levels, where 2^h is largest. + +What merging adds: +1. Each shard ran its own compactions while it held only N/m items, and all of those stay in the result. A single stream might have done them later, at a lower level, or not at all. +2. Concatenating the shards' compactors overfills the levels, especially the top ones, so the merged sketch must compact again there. Those extra compactions carry the largest weights 2^h. + +So the merged C_h is larger at the top levels, and the variance Σ C_h·4^h grows by a constant factor. The order of the error does not change: KLL's ε guarantee holds under any merge order, with a somewhat larger constant. More shards (larger m = S/x) mean more extra high-level compactions. The full grid (10 seeds) matches this and adds a dependence on k: about 1.0–1.1× at k = 50/200 but 1.12–1.32× at k = 800 at N = 1e7. With a large k, each shard stays exact (uncompacted) longer, so merging introduces compactions the single stream would have done at lower levels or not at all. That shows up most at intermediate N: up to 3–4× at N = 1e4–1e5 for k = 800 with m = 4, converging by 1e6–1e7. + +**Top-k: a different and larger effect.** Each shard's heap keeps only that shard's top 32 keys. A key that is heavy globally but never in any one shard's top 32 is lost at merge time. That is lost information, not added noise. Across all 8 configs the median precision is unchanged, but at large K it drops by up to 40%, and widening the sketch does not remove the loss. + +**What this means for configuration:** +- For KLL on merged windows, use the merged curves. The recommender now does this: p99 latency over 5m windows moves from k = 50 to k = 200 to stay under the 1% target. +- For top-k, use the merged curves (#131), not the single-sketch ones. +- KLL's compaction randomness is unseeded. The merge grid uses 10 seeds for that reason. diff --git a/docs/saturation_study.md b/docs/saturation_study.md new file mode 100644 index 00000000..f64fb765 --- /dev/null +++ b/docs/saturation_study.md @@ -0,0 +1,543 @@ +# Saturation Study + +How long a stream does each sketch need before its accuracy stops changing? +This study measures that length, N_sat, per sketch, config and data shape, +records the cost of the sketch at the largest N measured, and finds the stream +length N* past which an exact answer costs 10x (100x) more than the sketch. + +It is stacked on PR #129 (`dev-milind-turboprom`) and does not change +`rqe-optimizer` or its atomic-cost table; it only adds a Pareto generator +(`--dataset pareto`, at `--dtype f64` or `i64`) and +`scripts/study_saturation.py`. + +## Definition of N_sat + +Accuracy is scored once, after the full stream, so each checkpoint N is a +separate `approxbench sketchbench --size N` run. For checkpoints +`N_1 < ... < N_m` (log-spaced, `--per-decade` per decade) the error at each +N is averaged over `--seeds` seeds, giving `e_1 .. e_m` with standard errors +`se_1 .. se_m`. + +- plateau `P` = mean of the last `--plateau-tail` (T) values of `e` +- checkpoint i is *in band* when `|e_i - P| <= max(tolerance * |P|, 2 * se_i, 1e-9)` + (`--tolerance`, default 0.10). The `2 * se_i` term keeps seed noise alone + from breaking the plateau when the error itself is small; the absolute floor + only matters for P = 0 +- **N_sat** = `N_i` for the smallest i such that every checkpoint `i..m` is in band +- **not_saturated** when `m < T`, or when no checkpoint before the tail is in + band (the tail agreeing with its own mean says nothing) + +Because the band is on magnitude, direction does not matter: `precision_at_k` +rises toward 1 while the other metrics fall. N_sat is bounded by `--n-max`: a +sketch whose error is still moving at the top of the grid is reported +`not_saturated`, not "never saturates". + +| family | sketches | data | error metric | +|---|---|---|---| +| frequency | `cms-fastpath-vector2d`, `countsketch-fastpath-vector2d` | Zipf(theta) over K keys, i64 | `are_top100` (falls back to `are_all` when fewer than 100 keys were seen) | +| topk | `cms-heap-topk-fastpath-vector2d` | Zipf(theta) over K keys, i64 | `precision_at_k` (k = 32) | +| cardinality | `hll` | Zipf(theta) over K keys, i64 | `relative_error` | +| quantile | `kll-percall`, `dd` | floor(Pareto(alpha, scale 1000)), i64 | `mean_rank_err` | + +Zipf with `--zipf-s 0` is uniform over the K keys (rand_distr accepts s = 0), +so theta = 0 needs no separate dataset. The quantile streams are Pareto draws +with minimum 1000 floored to i64 (`--dtype i64 --pareto-scale 1000`: the +datagen renderer truncates, which is a floor for positive draws). i64 because +the polars exact quantile baseline is i64 only; scale 1000 keeps three +significant digits, so ties from flooring are rare. Rank error is scale-free, +so this changes nothing for KLL; for DDSketch the floor adds at most 0.1% +value error, below its alpha. + +### Config grid + +Each sketch's size knob steps 4x over two decades (theory predicts the plateau +as a power of it): + +| sketch | configs | predicted plateau | +|---|---|---| +| CMS, CountSketch, CMS-heap top-k | cols 256, 1024, 4096, 16384 x rows 3, 5 | CMS ∝ 1/w, CountSketch ∝ 1/sqrt(w) | +| HLL (`lib`) | lg_k 12, 14, 16 | ∝ 1/sqrt(2^lg_k) | +| KLL | k 50, 200, 800 | ∝ ~1/k | +| DDSketch | alpha 0.005, 0.01, 0.02, 0.05 | ∝ alpha | + +asap HLL is compiled at lg_k 12, 14 and 16 only and refuses 10; its nearest +valid value, 12, is already in the grid, so HLL has three configs (one and a +half decades of m). Every other value is accepted (Vector2D takes any +positive shape, KLL k in [8, 26602], DDSketch alpha in (0, 1)). +`--one-config` keeps the first config of each list (rows=3 cols=256, lg_k=12, +k=50, alpha=0.005). + +At the final N of each point one extra run +(`--operations insert,query,merge --metrics throughput,cpu,memory +--merge-shards 16 --runs 3 --warmup-runs 1 --flat`) records insert, merge and +query CPU seconds (user + sys) and `memory_bytes`. `BENCH_WARMUP_SECS` +defaults to 0, so these costs are indicative; +`scripts/export_rqe_optimizer_costs.sh` remains the source of optimizer costs. + +`--cost-rows 3` times only the rows=3 configs of CMS, CountSketch and +CMS-heap top-k (all four cols); HLL, KLL and DDSketch are timed at every +config. The rows=5 points keep blank cost columns in `saturation.csv` and +get no `crossover.csv` row. Their cost is extrapolated: a Vector2D sketch +updates and reads one counter per row and stores rows x cols counters, so +insert CPU, query CPU and `memory_bytes` at rows=5 are ≈ 5/3 of rows=3 at +the same cols. For insert and query 5/3 is an upper bound: the fast path +hashes each item once, and CMS-heap top-k's heap update does not scale with +rows. Merge is a counter-wise add, also ≈ 5/3. + +## Exact baseline and crossover N* + +The cost phase also runs, at every checkpoint N, the exact polars baseline of +each family over the same distributions (every theta x K for Zipf, every +alpha for Pareto; frequency and top-k share one run): + +| family | exact baseline (`--library polars`) | what it holds | +|---|---|---| +| frequency, topk | `cms`: `group_by(v).agg(len)` | the buffered stream, then a key→count table (grows with distinct keys seen, so with K) | +| cardinality | `hll`: `n_unique()` | the buffered stream | +| quantile | `kll-cdf`: 101-point quantile grid | the buffered stream | + +Two serial runs per (baseline, distribution, N), 3 runs + 1 warmup each: +`--operations insert,query --metrics throughput,cpu,memory` and +`--operations prepare --metrics latency,cpu,memory` (prepare is the polars +pass and is only measurable under latency, where it is one timed call). +Exact CPU = insert + prepare + one query's CPU seconds (user + sys, all +polars threads); exact memory = `memory_bytes` of the prepare record. + +Both sides are charged **one** query: the benchmark's query phase repeats the +query over a probe set (every key seen, 101 quantiles, or a repeated HLL +estimate) whose size differs between sketch and exact, so the per-query CPU +(phase CPU / operations) is used. This is "ingest N items, answer one query". + +For each sketch point, with its cost run at the final N: + +- sketch memory = its `memory_bytes`, taken as constant in N (it is fixed + by the config except for KLL's log-N levels and DDSketch's bucket range); +- sketch CPU at N = insert CPU x N / N_final + one query's CPU (insert is a + per-item cost, measured N-independent in the earlier run; a query is not a + function of N); +- **N\*_mem(f)** = smallest checkpoint N **from which** exact memory stays + >= f x sketch memory at every larger checkpoint, and **N\*_cpu(f)** + likewise for CPU, for f = 10 and 100; `not_reached` when even the last + checkpoint does not qualify. "From which it stays" rather than "first time": + polars' fixed start-up cost (~30–50 ms) makes exact CPU look high at the + smallest N and comparatively cheap further up. + +`--phase crossover` recomputes `crossover.csv` from the cost phase's +`saturation_cost.jsonl` and `exact_cost.jsonl` without running anything. + +`crossover.csv` has one row per point: `n_sat`, `sketch_memory_bytes`, +`sketch_cpu_secs` (insert + query at N_final) and the four N\* columns. Since +both costs grow linearly in N, the CPU ratio tends to a constant (per-item +exact cost / per-item sketch cost) and N\*_cpu may never be reached; the +memory ratio grows without bound, so N\*_mem always exists given a long +enough stream. + +**Which memory metric.** Both rows report `memory_bytes`, a formula each row +computes over the structure it keeps: the sketch's counters or registers, and +for polars the buffered stream (`Vec` capacity x 8 bytes, so up to 2x N x 8 +from doubling) plus the count table. `heap_bytes_peak` (the tracking +allocator) is not comparable here: it only exists with the `heap-track` +feature, which the default `approxbench` build does not enable, and it counts +allocations inside the timed pass only. A sketch allocates its table at build +time, outside the pass, so its peak would read near zero, while the polars +buffer grows inside the insert pass and polars' transient group_by/sort +buffers land in prepare. Using `memory_bytes` for both compares retained +state with retained state. It leaves out polars' transient working memory, so +exact memory is a lower bound and N\*_mem is conservative (the exact answer +really gets expensive a bit sooner). + +## From data parameters to a configuration + +`scripts/recommend_config.py` turns the worst case each query's data shows +into a recommended config per sketch family, writing `out/recommendations.csv` +(`dataset, query_id, range, family, config, est_error, target, meets_target, +n_sat, flags`, the cost columns `memory_bytes, insert_ns_per_item, +merge_us_per_fold, query_phase_us`, and the grid point used: `grid_param, grid_K, N, +shards`). + +Inputs: + +- ASAPQuery `asap-tools/dataset-analysis/results/skew_summary.csv`: per + (dataset, query, range) `worst_theta_cms` (lowest θ of the weight a per-key + counter sees), `worst_K`, `min_N`/`max_N` (items per evaluation), + `worst_alpha_rank`/`worst_alpha_memory`, `tail_class` and `target_*` + (defaults: `are_top100` ≤ 0.05, precision@k ≥ 0.95, rank error ≤ 0.01). + The older per-window schema is read too (`lower`/`upper`, `K_win_max`, + `rows_win_*`, default targets, finest window as the step). +- `out_grid_1e7/saturation_curve.csv` (error per N) and, for N above 1e7, + `out_1e9/saturation_curve.csv` where it has the point; the two + `saturation.csv` files for n_sat (the 1e9 one wins) and, once the cost + phase has run, cost. Without cost the cost columns stay empty and configs + are ordered by nominal size (rows·cols, k, 1/α); rows=5 without cost takes + 5/3 of rows=3. +- Optional `--merge-curves saturation_merge_curve.csv` (branch + `merged-accuracy`). + +Families: key queries → CMS, CountSketch, CMS-heap top-k; value queries → +KLL, DDSketch. No query is a count-distinct, so HLL is not a candidate. + +Rounding (always toward the harder side, so estimates are conservative): θ +down to the grid, K up, α for rank error (`worst_alpha_rank`, steepest tail) +up, α for DDSketch cost (`worst_alpha_memory`) down. The error is read at +N = `max_N`, linear in log N between checkpoints; a merged window of a +CMS/CountSketch/DDSketch has the same error as one sketch over it. For top-k +and KLL, with `--merge-curves` the error comes from the merged curve at the +shard count nearest (in log scale) to m = range / step; without it the +single-sketch curve is optimistic. The smallest config meeting the target is +recommended; if none does, the best one. + +Flags: `θ/K/alpha ... beyond grid` or `below grid` (clamped to the grid edge), +`extrapolated beyond N=...` (N above the largest measured N; the last error is +used), `N below grid`, `not saturated` (never saturated in the grid), +`not saturated at min_N` (min_N < n_sat), `light tail` (α not meaningful), +`single-sketch curve optimistic` / `merged curve at m=...`, `cost scaled from +rows=3`, `target not met`. + +```bash +python3 scripts/recommend_config.py \ + --merge-curves ../sketch-bench-merged/out_merge/saturation_merge_curve.csv +python3 scripts/test_recommend_config.py +``` + +## Running + +`--phase accuracy|cost|all` (default all) splits the study. `accuracy` runs +only the parallel accuracy runs and writes `saturation_curve.csv` and a +`saturation.csv` with blank cost columns. `cost` reads +`saturation_curve.csv` from `--out`, recomputes N_sat and the final error from +it, and runs only the serial cost runs (sketch at the final N, exact baseline +at every N), writing `saturation.csv` and `crossover.csv`; give it the same +grid arguments as the accuracy run (it exits if a point's curve is missing or +covers other sizes). Run the cost phase on an otherwise idle machine. + +`--resume` (accuracy) keeps every point whose curve over the requested sizes +is already in `--out`'s `saturation_curve.csv`, writes those curves back +first, and runs only the remaining points; a point cut off mid-write is +dropped and rerun, and raw records are appended to +`saturation_accuracy.jsonl`. The grid run below was killed at 498 of 595 +points and finished this way. + +`--points-from FILE` restricts the grid to the points listed in a CSV with +`sketch,config,dist,param,cardinality` columns (a filtered `saturation.csv` +works as is); it exits if a listed point is outside the grid. + +```sh +cargo build -p aqpbm-cli --release + +# Full grid to 1e7 (the run below): accuracy now, cost later on a quiet machine. +python3 scripts/study_saturation.py --phase accuracy --n-max 1e7 --seeds 3 \ + --jobs 16 --out out_grid_1e7 # add --resume after an interruption +python3 scripts/study_saturation.py --phase cost --n-max 1e7 --seeds 3 \ + --cost-rows 3 --out out_grid_1e7 + +# 1e9 subset: the points listed in out_1e9/points.csv (see Results). +python3 scripts/study_saturation.py --phase accuracy --n-max 1e9 --seeds 3 \ + --jobs 8 --points-from out_1e9/points.csv --out out_1e9 + +python3 scripts/test_study_saturation.py + +# Figures (need matplotlib, pandas, numpy). +python3 scripts/plot_saturation_curves.py out/saturation_curve.csv out/ +python3 scripts/plot_saturation_curves.py out_1e9/saturation_curve.csv out_1e9/ +python3 scripts/plot_saturation_cost.py out/cost_cpu_memory.png out/saturation_cost.jsonl +``` + +Restrict the grid with `--families`, `--thetas`, `--cardinalities`, +`--alphas`; `--one-config` keeps the first config of each sketch. `--jobs` +parallelises the accuracy runs only; the cost runs are always serial. Output +goes to `--out` (default `out/`): raw records in `saturation_accuracy.jsonl`, +`saturation_cost.jsonl` and `exact_cost.jsonl`, the per-N seed-mean errors in +`saturation_curve.csv`, one row per point in `saturation.csv`, and N\* per +point in `crossover.csv`. The plotting scripts draw one config per sketch +(rows=3 cols=1024, lg_k=12, k=200, alpha=0.01, the configs their bounds are +written for). + +## Results (config grid, N up to 1e7) + +`--phase accuracy --n-max 1e7 --seeds 3 --jobs 16 --out out_grid_1e7`: 595 +points (504 Zipf points for the three Vector2D sketches, 63 HLL, 28 +quantile) x 17 sizes x 3 seeds. 36 min wall on the shared 56-core machine +(31 min until it was killed at 498 points, 5 min for the resumed 97). Cost +columns are blank until the cost phase runs. + +### N_sat per config + +| sketch | config | not saturated by 1e7 | median N_sat | max N_sat | +|---|---|---|---|---| +| CMS | rows=3, cols 256 / 1024 / 4096 / 16384 | 2 / 2 / 4 / 4 of 21 | 1.8e4 / 1e5 / 1.8e5 / 3.2e5 | 5.6e5 / 1.8e6 / 1.8e6 / 1.8e6 | +| CMS | rows=5, same cols | 2 / 7 / 4 / 4 | 3.2e4 / 1e5 / 1.8e5 / 1e3 | 5.6e5 / 5.6e5 / 1.8e6 / 1.8e6 | +| CountSketch | rows=3 | 3 / 4 / 7 / 4 | 1.8e5 / 1e5 / 1e6 / 1e5 | 1e6 / 1.8e6 / 1.8e6 / 1.8e6 | +| CountSketch | rows=5 | 3 / 3 / 6 / 6 | 5.6e4 / 1.8e5 / 1e6 / 1e5 | 5.6e5 / 1.8e6 / 1.8e6 / 1.8e6 | +| CMS-heap top-k | rows=3 | 7 / 4 / 2 / 1 | 4.4e3 / 1e3 / 1.8e3 / 1.8e3 | 1.8e6 / 3.2e5 / 5.6e4 / 1.8e6 | +| CMS-heap top-k | rows=5 | 2 / 1 / 0 / 1 | 1.8e3 / 1.4e3 / 1.8e3 / 1.8e3 | 1.8e6 / 1e6 / 1.8e6 / 1e6 | +| HLL | lg_k 12 / 14 / 16 | 2 / 10 / 5 of 21 | 1e5 / 1e4 / 3.3e5 | 1.8e6 each | +| KLL | k 50 / 200 / 800 | 2 / 3 / 3 of 4 | 1e5 / 1.8e5 / 1.8e4 | | +| DDSketch | alpha 0.005 / 0.01 / 0.02 / 0.05 | 0 of 4 each | 1.4e4 / 2.1e3 / 1e3 / 1e3 | 3.2e5 / 3.2e3 / 1e3 / 1e3 | + +108 of 595 points are not saturated by 1e7. For the Vector2D sketches they +sit at K >= 1e5 (CMS 25 of 29, CountSketch 33 of 36, top-k 16 of 18), and +at theta <= 0.5 (low skew: the error ∝ tail mass / w keeps changing while +new keys arrive; 17 CMS and 26 CountSketch points) or, for large w, at +theta >= 1.5, where the plateau is so small (1e-4 to 1e-6) +that the relative band is narrower than the remaining drift. For HLL and KLL +the unsaturated points are seed noise: with 3 seeds their tail wanders by +more than 10% of a plateau of 1e-3 to 1e-2 (the earlier run needed 10 seeds +for KLL). N_sat does not grow systematically with the config: a wider CMS +needs a somewhat longer stream before collisions settle (median 1.8e4 at +cols 256 to 3.2e5 at cols 16384, rows=3), but for the other sketches the +median moves by less than the checkpoint spacing's noise. + +### Plateau vs config + +The plateau is the seed-mean error at N = 1e7. The slope is a least-squares +fit of log(error) on log(knob) (w = cols, m = 2^lg_k, k, alpha), over the +configs with non-zero error; `*` marks a point not saturated by 1e7. For +top-k the fitted quantity is the miss rate 1 − precision_at_k. The rows are +a representative subset of the distributions. + +| sketch | data | plateau at cols 256 / 1024 / 4096 / 16384 (rows=3) | slope rows=3 | slope rows=5 | theory | +|---|---|---|---|---|---| +| CMS | θ=0, K=1e7 | 4.8e+03* / 1.19e+03* / 294* / 71.7* | −1.01 | −1.01 | −1 | +| CMS | θ=0.5, K=1e7 | 162 / 40.2 / 9.89 / 2.38 | −1.01 | −1.02 | −1 | +| CMS | θ=1, K=1e5 | 1.21 / 0.237 / 0.0449 / 0.00619 | −1.26 | −1.31 | −1 | +| CMS | θ=1, K=1e7 | 2.12 / 0.464 / 0.101 / 0.0199 | −1.12 | −1.12 | −1 | +| CMS | θ=1.5, K=1e7 | 0.241 / 0.0288 / 0.00413 / 0.000429 | −1.51 | −1.55 | −1 | +| CMS | θ=2, K=1e7 | 0.0944 / 0.00472 / 0.000359* / 5.39e-06* | −2.30 | −2.33 | −1 | +| CountSketch | θ=0, K=1e7 | 9.46* / 4.9* / 2.64* / 1.6* | −0.43 | −0.42 | −0.5 | +| CountSketch | θ=0.5, K=1e7 | 0.806* / 0.42* / 0.223* / 0.102* | −0.49 | −0.51 | −0.5 | +| CountSketch | θ=1, K=1e5 | 0.265 / 0.0779 / 0.0182 / 0.00739 | −0.88 | −1.02 | −0.5 | +| CountSketch | θ=1, K=1e7 | 0.262 / 0.081 / 0.0183 / 0.00781 | −0.87 | −1.01 | −0.5 | +| CountSketch | θ=1.5, K=1e7 | 0.191 / 0.0257 / 0.00338 / 0.00256 | −1.08 | −1.48 | −0.5 | +| CountSketch | θ=2, K=1e7 | 0.171 / 0.0118 / 0.000819* / 0.00137 | −1.24 | −2.25 | −0.5 | +| top-k miss rate | θ=0, K=1e7 | 1 / 1 / 1 / 1* | 0.00 | 0.00 | — | +| top-k miss rate | θ=0.5, K=1e7 | 0.896 / 0.667* / 0.26* / 0.104 | −0.53 | −0.64 | — | +| top-k miss rate | θ=1, K=1e5 | 0.594* / 0.0312 / 0 / 0 | −2.12 | — | — | +| top-k miss rate | θ=1, K=1e7 | 0.75* / 0.281* / 0.0104 / 0 | −1.54 | — | — | +| top-k miss rate | θ=1.5, K=1e7 | 0.427* / 0.0521 / 0 / 0 | −1.52 | — | — | +| top-k miss rate | θ=2, K=1e7 | 0.125 / 0 / 0 / 0 | — | — | — | + +| sketch | data | plateau, knob small → large | slope | theory | +|---|---|---|---|---| +| HLL, lg_k 12 / 14 / 16 | θ=0, K=1e7 | 0.0131 / 0.00522* / 0.00368 | −0.46 | −0.5 | +| HLL, lg_k 12 / 14 / 16 | θ=1, K=1e7 | 0.0116 / 0.00221* / 0.00213 | −0.61 | −0.5 | +| HLL, lg_k 12 / 14 / 16 | θ=2, K=1e7 | 0.0113* / 0.00415* / 0.00183 | −0.66 | −0.5 | +| KLL, k 50 / 200 / 800 | Pareto a=1.1 | 0.0118 / 0.00313* / 0.000688 | −1.03 | −0.97 | +| KLL, k 50 / 200 / 800 | Pareto a=1.5 | 0.0109 / 0.00336* / 0.00059* | −1.05 | −0.97 | +| KLL, k 50 / 200 / 800 | Pareto a=2 | 0.0143* / 0.00269* / 0.000566* | −1.16 | −0.97 | +| KLL, k 50 / 200 / 800 | Pareto a=3 | 0.011* / 0.0024 / 0.000418* | −1.18 | −0.97 | +| DDSketch, alpha 0.005 / 0.01 / 0.02 / 0.05 | Pareto a=1.1 | 0.00108 / 0.00273 / 0.00537 / 0.0134 | 1.08 | +1 | +| DDSketch, alpha 0.005 / 0.01 / 0.02 / 0.05 | Pareto a=1.5 | 0.00172 / 0.00355 / 0.00726 / 0.0181 | 1.02 | +1 | +| DDSketch, alpha 0.005 / 0.01 / 0.02 / 0.05 | Pareto a=2 | 0.00233 / 0.00492 / 0.00968 / 0.0241 | 1.01 | +1 | +| DDSketch, alpha 0.005 / 0.01 / 0.02 / 0.05 | Pareto a=3 | 0.00376 / 0.00735 / 0.0143 / 0.0361 | 0.98 | +1 | + +- **CMS** follows 1/w exactly (slope −1.0) where collisions with the tail + dominate the error (theta <= 0.5 at any K >= 1e5, including the + unsaturated theta = 0 points). With skew it falls faster than 1/w (−1.1 at + theta 1, −1.5 at 1.5, −2.3 at 2): the 1/w bound is for the mean colliding + mass, but the min over rows discards rows where a heavy key collides, and + that probability falls as (h/w)^rows. At K = 1e3 (not shown) cols >= K + gives zero error, so fits there are steeper (−2 to −4) and not meaningful. + rows=5 matches rows=3 where the tail dominates and is lower otherwise. +- **CountSketch** follows 1/sqrt(w) at low skew (−0.43 to −0.51 at + theta <= 0.5, K = 1e7). With skew it steepens to about −theta (−0.87 at + theta 1, −1.1 at 1.5): its error is governed by the residual L2 norm with + the top ~w keys removed, which for Zipf(theta > 0.5) shrinks as + w^(1/2 − theta), so error ∝ w^(−theta). rows=5 is steeper still (median + of 5 rows). +- **HLL** at K = 1e7 fits −0.46 to −0.66 against −0.5, within the noise of + 3 seeds of |relative error| over three configs. At K <= 1e5 the larger + m reach HLL's linear-counting (small-range) regime, where the error drops + to ~1e-4 or 0, so slopes there (−1 to −1.7) measure the regime change, + not the asymptotic 1/sqrt(m). +- **KLL** fits −1.03 to −1.18 against the −0.97 of its 2.296/k^0.9723 + bound: about 1/k, the noisier values at alpha 2 and 3 coming from 3-seed + plateaus. +- **DDSketch** fits 0.98 to 1.08: rank error ∝ alpha, on every Pareto tail. +- **CMS-heap top-k** has no power law to compare: its miss rate is 1 at + theta = 0 (no heavy hitters to find), falls roughly as w^−0.5 at + theta = 0.5 and as w^−1.5 to w^−2 at theta >= 1 until it hits 0 + (cols >= 4096 finds all top 32 for theta >= 1 at K = 1e5). + +## Results (1e9 subset) + +`--phase accuracy --n-max 1e9 --seeds 3 --jobs 8 --points-from +out_1e9/points.csv --out out_1e9`: 185 points x 25 sizes (1e3 to 1e9, four +per decade) x 3 seeds, 18.4 h wall on the shared machine. The points are all +108 grid points that were not saturated by 1e7, plus every point of the +plotted configs (rows=3 cols=1024, lg_k=12, k=200, alpha=0.01) so their +figures (`out_1e9/error_vs_N__*.png`) cover N up to 1e9. Error values at +N <= 1e7 match the grid run exactly for every sketch except KLL, whose +compaction coin flips are not seeded (seed means differ by up to 2x at small +N and a few percent at the plateau). + +Unsaturated at 1e7 → status at 1e9 (N_sat recomputed on the 25-point curve): + +| sketch | unsaturated at 1e7 | saturated by 1e9 | still not saturated | +|---|---|---|---| +| CMS | 29 | 18 | 11 | +| CountSketch | 36 | 25 | 11 | +| CMS-heap top-k | 18 | 11 | 7 | +| HLL | 17 | 10 | 7 | +| KLL | 8 | 3 | 5 | +| DDSketch | 0 | — | — | +| total | 108 | 67 | 41 | + +What stays unsaturated, and why: + +- **CMS and CountSketch, theta = 0, K = 1e7 (all 8 configs of each; 16 of + the 22).** Error keeps + rising: x5.5 for CMS and x3.1 to x3.9 for CountSketch from 1e7 to 1e9. With + uniform data over 1e7 keys, N = 1e9 is only 100 items per key, so the + sketch's additive error (∝ N/w for CMS, ∝ ||f||2/sqrt(w) for CountSketch) + still grows faster than the true top-100 counts. At K = 1e5 the same + configs saturate by 5.6e7 (about 500 items per key), with the plateau 1.3x + the 1e7 error. Uniform frequency points only saturate at N >> K. The other + 6 are 3 theta = 2 points at cols 16384 with errors near 2e-5, where a few + collisions move the mean, and 3 theta = 0 points at K = 1e3 or 1e5 (1 CMS, + 2 CountSketch). +- **CMS-heap top-k, K = 1e7, theta 0.5 to 1.5, cols 256 and 1024.** The miss rate + halves from 1e7 to 1e9 (0.25 → 0.125 at theta 1) and is still falling: + more items separate the true top 32 from the tail. This is the one sketch + whose error improves with N rather than approaching a plateau from below. +- **HLL at lg_k 14 and 16, theta >= 1.2, K = 1e7; theta = 2 everywhere.** + HLL error depends only on the distinct count seen, which for skewed Zipf + keeps growing until N covers the tail; the 3-seed curves are noisy enough + that the ±10% band rarely holds for three consecutive sizes. At lg_k 12 the + error settles at 0.021 once all 1e7 distinct values are seen (theta <= 1). + The theta = 2 points at lg_k 14 (0.011 to 0.021 at 1e9, above lg_k 12) are + not explained by the bound and need more seeds to trust. +- **KLL.** Errors from 1e7 to 1e9 move by 0.8x to 1.4x with no trend; the + 3-seed plateau is within noise of the band edge, so N_sat flips in both + directions (k=200 alpha 3 was saturated at 1e7 and is not at 1e9). KLL is + effectively flat by 1e6; its N_sat needs 10 seeds (as in the earlier run). + +The longer curve also moves 1e7-saturated verdicts: of the 77 extra points, +55 keep the same N_sat, 13 move it to another size <= 1e7, 7 move it past +1e7 (mostly HLL lg_k 12 at K = 1e7, where the distinct count is still +growing), and 2 lose it (one top-k, one KLL). N_sat from a curve that ends +at 1e7 is therefore a lower bound when K is large or the error metric is +noisy; for CMS/CountSketch with theta >= 0.5 and DDSketch it is stable. + +## Results (earlier reduced run) + +This run predates the config grid and the i64 Pareto data: quantile streams +were Pareto(alpha, scale 1) at f64, and the configs were the then-first ones. +Reduced grid: `--one-config --n-max 1e7`, 3 seeds for the Zipf sketches and +10 seeds for the quantile sketches (`--families quantile --seeds 10`, which +KLL needed to separate its plateau from seed noise). Cost runs were serial on +an otherwise idle 56-core machine. Each cell is `N_sat (error at N = 1e7)`. + +| sketch (config) | metric | theta | K=1e3 | K=1e5 | K=1e7 | +|---|---|---|---|---|---| +| cms-fastpath-vector2d (rows=3 cols=1024) | are_top100 | 0.0 | 3.2e5 (0.254) | not sat. (65.9) | not sat. (1.19e+03) | +| cms-fastpath-vector2d (rows=3 cols=1024) | are_top100 | 0.5 | 5623 (0.0627) | 1e5 (3.63) | 5.6e5 (40.2) | +| cms-fastpath-vector2d (rows=3 cols=1024) | are_top100 | 0.8 | 1e5 (0.031) | 3.2e4 (0.666) | 1e5 (2.3) | +| cms-fastpath-vector2d (rows=3 cols=1024) | are_top100 | 1.0 | 1.8e4 (0.0194) | 5.6e4 (0.237) | 3.2e4 (0.464) | +| cms-fastpath-vector2d (rows=3 cols=1024) | are_top100 | 1.2 | 5623 (0.0121) | 5.6e4 (0.0924) | 5.6e4 (0.126) | +| cms-fastpath-vector2d (rows=3 cols=1024) | are_top100 | 1.5 | 5.6e4 (0.00615) | 1.8e5 (0.0266) | 1.8e5 (0.0288) | +| cms-fastpath-vector2d (rows=3 cols=1024) | are_top100 | 2.0 | 3.2e5 (0.00184) | 1.8e6 (0.00472) | 1.8e6 (0.00472) | +| countsketch-fastpath-vector2d (rows=3 cols=1024) | are_top100 | 0.0 | not sat. (0.342) | not sat. (2.33) | not sat. (4.9) | +| countsketch-fastpath-vector2d (rows=3 cols=1024) | are_top100 | 0.5 | 3.2e4 (0.157) | 1e5 (0.348) | not sat. (0.42) | +| countsketch-fastpath-vector2d (rows=3 cols=1024) | are_top100 | 0.8 | 1778 (0.0855) | 1e5 (0.133) | 1e6 (0.135) | +| countsketch-fastpath-vector2d (rows=3 cols=1024) | are_top100 | 1.0 | 3162 (0.0586) | 5.6e4 (0.0779) | 5.6e4 (0.081) | +| countsketch-fastpath-vector2d (rows=3 cols=1024) | are_top100 | 1.2 | 1000 (0.0404) | 1e5 (0.0495) | 1e5 (0.0492) | +| countsketch-fastpath-vector2d (rows=3 cols=1024) | are_top100 | 1.5 | 1.8e5 (0.0235) | 3.2e5 (0.0265) | 5.6e5 (0.0257) | +| countsketch-fastpath-vector2d (rows=3 cols=1024) | are_top100 | 2.0 | 3162 (0.0106) | 1e5 (0.0115) | 1.8e6 (0.0118) | +| cms-heap-topk-fastpath-vector2d (rows=3 cols=1024) | precision_at_k | 0.0 | not sat. (0.104) | 1.8e4 (0) | 1000 (0) | +| cms-heap-topk-fastpath-vector2d (rows=3 cols=1024) | precision_at_k | 0.5 | 1000 (0.823) | 5.6e4 (0.531) | not sat. (0.333) | +| cms-heap-topk-fastpath-vector2d (rows=3 cols=1024) | precision_at_k | 0.8 | 1000 (1) | 5623 (0.917) | not sat. (0.688) | +| cms-heap-topk-fastpath-vector2d (rows=3 cols=1024) | precision_at_k | 1.0 | 1000 (1) | 1778 (0.969) | not sat. (0.719) | +| cms-heap-topk-fastpath-vector2d (rows=3 cols=1024) | precision_at_k | 1.2 | 1000 (1) | 1000 (0.969) | 3.2e5 (0.865) | +| cms-heap-topk-fastpath-vector2d (rows=3 cols=1024) | precision_at_k | 1.5 | 1000 (1) | 1000 (0.979) | 1000 (0.948) | +| cms-heap-topk-fastpath-vector2d (rows=3 cols=1024) | precision_at_k | 2.0 | 1778 (1) | 3162 (0.99) | 3162 (1) | +| hll (lg_k=12) | relative_error | 0.0 | 1000 (0.011) | 3.2e5 (0.0114) | 3.2e5 (0.0131) | +| hll (lg_k=12) | relative_error | 0.5 | 1000 (0.011) | 1e5 (0.0114) | 1e6 (0.00814) | +| hll (lg_k=12) | relative_error | 0.8 | 1778 (0.011) | 1.8e4 (0.0114) | 1e6 (0.014) | +| hll (lg_k=12) | relative_error | 1.0 | 3.2e4 (0.011) | 1e6 (0.0114) | 3162 (0.0116) | +| hll (lg_k=12) | relative_error | 1.2 | 1778 (0.011) | 1e6 (0.0101) | not sat. (0.0109) | +| hll (lg_k=12) | relative_error | 1.5 | 3.2e4 (0.011) | 1.8e5 (0.0135) | 5.6e5 (0.0234) | +| hll (lg_k=12) | relative_error | 2.0 | 1.8e6 (0.011) | 5.6e4 (0.00604) | not sat. (0.0113) | + +| sketch (config) | metric | alpha=1.1 | alpha=1.5 | alpha=2.0 | alpha=3.0 | +|---|---|---|---|---|---| +| kll-percall (k=200) | mean_rank_err | 1.8e6 (0.00318) | 1.8e5 (0.00345) | 1.8e6 (0.00321) | 1e6 (0.00338) | +| dd (alpha=0.01) | mean_rank_err | 3162 (0.00257) | 1000 (0.00378) | 1000 (0.00485) | 1000 (0.00721) | + +### Figures + +Committed under `docs/figures/saturation/`: `reduced_run/` (error vs N per +sketch with theoretical bounds, the serial cost figure, and a README that +explains every figure) and `grid_1e9/` (the same plots from the 1e9 subset). + +### Reading the curves + +`plot_saturation_curves.py` draws one panel per distribution with the +theoretical bound. None of the bounds depends on N; where the empirical +error still moves with N it is a finite-N effect (noisy ground truth at small +N, or a tail that keeps filling in while N is below K). + +| sketch | bound (this config) | empirical behaviour | +|---|---|---| +| CMS `are_top100` | mean over top 100 of (e/w)/p_i | rises then plateaus; plateau falls steeply with theta (worst case = low theta); theta = 0 with K >= 1e5 never saturates by 1e7 | +| CountSketch `are_top100` | mean over top 100 of sqrt(3/w)·‖p‖₂/p_i | hump then decline to plateau (the 1/N term of ‖f‖₂ fades); far below CMS at low theta; needs N of order 1/Σp² | +| CMS-heap top-k `precision_at_k` | lower bound: share of top 32 with p_i − p_33 > e/w | ≈ 1 for theta >= 0.8 and K <= 1e5; ill-defined at theta = 0; at K = 1e7 precision falls once N approaches K | +| HLL `relative_error` | 2 × 1.04/sqrt(m) = 3.25% | depends on the distinct count seen, not N; constant once every key has been seen | +| KLL `mean_rank_err` | 2.296/k^0.9723 = 1.33% | flat at ≈ 3e-3 from ≈ 5e3 items, independent of alpha; the rule's N_sat is conservative because compaction noise exceeds 2 SE | +| DDSketch `mean_rank_err` | α_dd · a / 2 (derived from the value-error guarantee) | flat by 1e3–1e4; grows with a (lighter tail, larger rank error); memory shrinks with a | + +Serial per-operation cost at N = 1e7: insert is N- and distribution-independent +for every sketch (CMS 27 ns, CountSketch 31 ns, HLL 4 ns, KLL 32 ns, DDSketch +20–27 ns per item) except CMS-heap top-k, which grows from ≈ 100 ns at theta = 0 +to ≈ 2,100 ns at theta = 2. DDSketch memory falls from 6.3 KB (alpha = 1.1) to +2.3 KB (alpha = 3); the others are fixed by their config. + +### Grid cost and crossover N* + +`--phase cost --n-max 1e7 --seeds 3 --cost-rows 3` ran serially on a +separate idle 56-core machine (load 1.0, ≈ 3 h; 343 sketch points plus the +exact baselines). Figures and CSVs: `docs/figures/saturation/grid_cost/`. + +Per-item insert CPU is flat in N and in the distribution for every sketch +(CMS ≈ 22 ns, CountSketch ≈ 26 ns, HLL ≈ 3.4 ns, KLL ≈ 25 ns, DDSketch +≈ 16 ns) except CMS-heap top-k, which rises from ≈ 100 ns at theta 0 to +≈ 1,750 ns at theta 2. DDSketch memory falls with the Pareto index (6.3 KB at +alpha 1.1 to 2.3 KB at alpha 3, alpha_dd = 0.01). + +Crossover at theta = 1, K = 1e5 (frequency, top-k, HLL) and Pareto alpha 1.5 +(quantile): + +| sketch (config) | memory | N\*_mem 10x | N\*_mem 100x | N\*_cpu 10x | N\*_cpu 100x | +|---|---|---|---|---|---| +| CMS / CountSketch 3x256 | 3 KB | 1.8e3 | 1.8e4 | 1e3 | not reached | +| CMS / CountSketch 3x1024 | 12 KB | 5.6e3 | 1e5 | 1e3 | not reached | +| CMS / CountSketch 3x16384 | 192 KB | 1.8e5 | 3.2e6 | 1e3 | not reached | +| CMS-heap top-k (any cols) | 3–192 KB | as CMS | as CMS | not reached | not reached | +| HLL lg_k 12 / 14 / 16 | 4 / 16 / 64 KB | 5.6e3 / 1.8e4 / 1e5 | 5.6e4 / 1.8e5 / 5.6e5 | 3.2e3 / 1e4 / 3.2e4 | not reached | +| KLL k 50 / 200 / 800 | 1.6 / 6.4 / 26 KB | 1.8e3 / 5.6e3 / 1.8e4 | 1.8e4 / 1e5 / 3.2e5 | 1e3 | 1e6 | +| DDSketch alpha 0.01 | 4.6 KB | 5.6e3 | 5.6e4 | 1e3 | 1.8e5 | + +- **Memory**: the exact baseline buffers the stream (~13 bytes/item at 1e7, + independent of K), so a sketch saves 10x after a few thousand items and + 100x after 1e4–1e6, scaling with its own size. +- **CPU**: exact frequency cost (buffer + polars group-by) is ≈ 300 ns/item + and exact cardinality (buffer + n_unique) ≈ 145 ns/item at 1e7, so + CMS/CountSketch (≈ 25 ns) stay ≈ 13x cheaper at every N and HLL (≈ 3.4 ns) + ≈ 40x once its O(m) estimate is amortised, but neither reaches 100x; quantile exact (sort) is superlinear, so + KLL and DDSketch pass 100x at 1e6 and 1.8e5. CMS-heap top-k at theta >= 1 is + slower per item than the exact baseline (≈ 650 ns vs ≈ 300 ns), so its + benefit is memory only. +- The exact baseline here is "store the samples, compute at query time" + (what a TSDB holding raw samples does); a streaming exact counter (hash map, + O(K) memory) would move the frequency memory crossover to N where the + table of K keys outgrows the sketch. + +### Not covered yet + +- **Accuracy after merging.** Every curve is one sketch fed N items. For CMS, + CountSketch, HLL and DDSketch a merge of m shards equals one sketch over + the union, so the curves apply to merged windows. For KLL (merge depth) and + the top-k heap (a globally heavy key can miss every shard's heap) they are + only a lower bound on merged error; measuring that is the next PR. +- **HLL lg_k 10.** asap_sketchlib has register types for lg_k 12, 14, 16 + (and 18 for the bucket list) only, so the HLL grid starts at 12. + diff --git a/scripts/plot_saturation_cost.py b/scripts/plot_saturation_cost.py new file mode 100644 index 00000000..a7bff359 --- /dev/null +++ b/scripts/plot_saturation_cost.py @@ -0,0 +1,100 @@ +"""CPU per operation and memory vs distribution parameter, from serial cost JSONL. + +Usage: plot_cost.py out.png cost.jsonl [override.jsonl ...] +Later files override earlier records for the same (sketch, distribution). +""" +import json +import sys + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 + +INK, INK2, MUTED, SURF = "#0b0b0b", "#52514e", "#898781", "#fcfcfb" +CAT = {1000: "#2a78d6", 100000: "#eb6834", 10000000: "#1baf7a"} +KLAB = {1000: "K=1K", 100000: "K=100K", 10000000: "K=10M"} +SKETCHES = [ + ("cms-fastpath-vector2d", "CMS (rows=3 cols=1024)"), + ("countsketch-fastpath-vector2d", "CountSketch (rows=3 cols=1024)"), + ("cms-heap-topk-fastpath-vector2d", "CMS-heap top-k (rows=3 cols=1024)"), + ("hll", "HLL (lg_k=12)"), + ("kll-percall", "KLL (k=200)"), + ("dd", "DDSketch (alpha=0.01)"), +] +# The config each column shows; the JSONL may hold the whole config grid. +PARAMS = { + "cms-fastpath-vector2d": {"rows": 3, "cols": 1024}, + "countsketch-fastpath-vector2d": {"rows": 3, "cols": 1024}, + "cms-heap-topk-fastpath-vector2d": {"rows": 3, "cols": 1024}, + "hll": {"lg_k": 12}, + "kll-percall": {"k": 200}, + "dd": {"alpha": 0.01}, +} +plt.rcParams.update({ + "font.size": 9, "axes.edgecolor": MUTED, "axes.labelcolor": INK2, + "xtick.color": MUTED, "ytick.color": MUTED, "axes.titlecolor": INK, + "axes.spines.top": False, "axes.spines.right": False, + "axes.grid": True, "grid.color": "#e8e7e3", "grid.linewidth": 0.6, + "figure.facecolor": SURF, "axes.facecolor": SURF, "legend.frameon": False, +}) + + +def per_op(rec, op, rate_key): + """CPU seconds per operation: phase CPU / (ops per second * phase wall seconds).""" + cpu, wall, rate = rec.get(f"{op}_cpu_time_ms"), rec.get(f"{op}_wall_time_ms"), rec.get(rate_key) + if not (cpu and wall and rate): + return None + ops = rate["mean"] * wall["mean"] / 1000.0 + return (cpu["user_ms"]["mean"] + cpu["sys_ms"]["mean"]) / 1000.0 / ops + + +recs = {} +for path in sys.argv[2:]: + for line in open(path): + r = json.loads(line) + if r["sketch_config"]["params"] != PARAMS.get(r["sketch"]): + continue + d = r["workload"]["synthetic"]["description"]["column_spec"][0]["distribution"] + if d["kind"] == "zipf": + key = (r["sketch"], d["skewness"], d["population_size"]) + else: + key = (r["sketch"], d.get("alpha"), None) + recs[key] = r + +ROWS = [ + ("insert", "insert_throughput_items_per_sec", 1e9, "insert CPU (ns / item)"), + ("merge", "merge_folds_per_sec", 1e6, "merge CPU (µs / fold)"), + ("query", "query_throughput_items_per_sec", 1e6, "query CPU (µs / query)"), + ("memory", None, 1.0, "memory_bytes (sketch's own formula)"), +] +fig, axes = plt.subplots(4, 6, figsize=(24, 13), constrained_layout=True) +for col, (sk, title) in enumerate(SKETCHES): + pts = {k: r for k, r in recs.items() if k[0] == sk} + pareto = all(k[2] is None for k in pts) + groups = {None: sorted(pts)} if pareto else { + K: sorted(k for k in pts if k[2] == K) for K in CAT} + for row, (op, rate, scale, ylab) in enumerate(ROWS): + ax = axes[row, col] + for K, keys in groups.items(): + xs, ys = [], [] + for k in keys: + r = pts[k] + v = r.get("memory_bytes") if op == "memory" else per_op(r, op, rate) + if v is not None: + xs.append(k[1]) + ys.append(v * scale) + ax.plot(xs, ys, "-o", color=CAT.get(K, "#2a78d6"), lw=2, ms=7, mec=SURF, mew=2, + label=KLAB.get(K, "Pareto")) + ax.set_ylim(bottom=0) + ax.set_xlabel("Pareto α" if pareto else "Zipf θ") + if row == 0: + ax.set_title(title, loc="left", fontsize=11) + if not pareto: + ax.legend(fontsize=8, labelcolor=INK2) + if col == 0: + ax.set_ylabel(ylab) +fig.suptitle("Serial cost at N = 10⁷ (3 runs + 1 warmup, 16 merge shards, seed 1). " + "Per-op CPU = phase user+sys CPU / operations in the phase.", + x=0.01, ha="left", color=INK, fontsize=12) +fig.savefig(sys.argv[1], dpi=110) +print(sys.argv[1]) diff --git a/scripts/plot_saturation_crossover.py b/scripts/plot_saturation_crossover.py new file mode 100644 index 00000000..4de5b85d --- /dev/null +++ b/scripts/plot_saturation_crossover.py @@ -0,0 +1,105 @@ +"""Exact computation vs sketch: memory and CPU as N grows, with the crossover N*. + +Usage: plot_saturation_crossover.py COST_DIR OUT.png (needs matplotlib) +COST_DIR holds exact_cost.jsonl and saturation_cost.jsonl from `--phase cost`. +""" +import json +import os +import sys + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 + +cost_dir, out = sys.argv[1], sys.argv[2] +INK, INK2, MUTED, SURF = "#0b0b0b", "#52514e", "#898781", "#fcfcfb" +EXACT = {1000: "#2a78d6", 100000: "#eb6834", 10000000: "#1baf7a", None: "#2a78d6"} +SKETCH_GRAYS = ["#c3c2b7", "#898781", "#52514e", "#0b0b0b"] +plt.rcParams.update({ + "font.size": 9, "axes.edgecolor": MUTED, "axes.labelcolor": INK2, + "xtick.color": MUTED, "ytick.color": MUTED, "axes.titlecolor": INK, + "axes.spines.top": False, "axes.spines.right": False, + "axes.grid": True, "grid.color": "#e8e7e3", "grid.linewidth": 0.6, + "figure.facecolor": SURF, "axes.facecolor": SURF, "legend.frameon": False, +}) + + +def dist_of(rec): + d = rec["workload"]["synthetic"]["description"]["column_spec"][0]["distribution"] + n = rec["workload"]["synthetic"]["description"]["row_num"] + if d["kind"] == "zipf": + return ("zipf", d["skewness"], d["population_size"]), n + return ("pareto", d.get("alpha"), None), n + + +def cpu(rec, op): + c = rec.get(f"{op}_cpu_time_ms") + return (c["user_ms"]["mean"] + c["sys_ms"]["mean"]) / 1000.0 if c else 0.0 + + +def one_query(rec): + """CPU of one query (the query phase repeats it over its probes).""" + w, t = rec.get("query_wall_time_ms"), rec.get("query_throughput_items_per_sec") + ops = t["mean"] * w["mean"] / 1000.0 if (w and t) else 0 + return cpu(rec, "query") / ops if ops else 0.0 + + +# Exact: records come in pairs (insert+query run, then prepare run). +exact = {} +lines = [json.loads(x) for x in open(os.path.join(cost_dir, "exact_cost.jsonl"))] +for timed, prepared in zip(lines[0::2], lines[1::2]): + dist, n = dist_of(timed) + exact.setdefault((timed["sketch"], dist), []).append( + (n, prepared["memory_bytes"], cpu(timed, "insert") + one_query(timed) + + cpu(prepared, "prepare"))) + +# Sketch: one cost run at N_final per point; insert CPU scales with N. +sketch = {} +for line in open(os.path.join(cost_dir, "saturation_cost.jsonl")): + r = json.loads(line) + dist, n = dist_of(r) + cfg = " ".join(f"{k}={v}" for k, v in sorted(r["sketch_config"]["params"].items())) + sketch[(r["sketch"], cfg, dist)] = (n, r.get("memory_bytes"), cpu(r, "insert"), one_query(r)) + +PANELS = [ + ("CMS (frequency)", "cms", "cms-fastpath-vector2d", ("zipf", 1.0)), + ("CountSketch (frequency)", "cms", "countsketch-fastpath-vector2d", ("zipf", 1.0)), + ("HLL (cardinality)", "hll", "hll", ("zipf", 1.0)), + ("KLL (quantile)", "kll-cdf", "kll-percall", ("pareto", 1.5)), + ("DDSketch (quantile)", "kll-cdf", "dd", ("pareto", 1.5)), + ("CMS-heap top-k", "cms", "cms-heap-topk-fastpath-vector2d", ("zipf", 1.0)), +] +fig, axes = plt.subplots(2, len(PANELS), figsize=(4.6 * len(PANELS), 8.2), constrained_layout=True) +for col, (title, exact_kind, sk, (kind, param)) in enumerate(PANELS): + for row, what in enumerate(("memory (bytes)", "CPU (s): ingest N items + answer one query")): + ax = axes[row, col] + for (ek, dist), pts in sorted(exact.items(), key=lambda kv: str(kv[0])): + if ek != exact_kind or dist[0] != kind or dist[1] != param: + continue + pts.sort() + ns = [p[0] for p in pts] + ys = [p[1] if row == 0 else p[2] for p in pts] + lab = "exact" + ("" if dist[2] is None else f", K={dist[2]:.0e}".replace("e+0", "e")) + ax.plot(ns, ys, "-o", color=EXACT.get(dist[2], "#2a78d6"), lw=2, ms=4, label=lab) + cfgs = sorted({c for (s, c, d) in sketch if s == sk and d[0] == kind and d[1] == param}, + key=lambda c: [int(x) if x.isdigit() else x for x in + c.replace("=", " ").split()]) + for i, c in enumerate(cfgs): + ks = [d for (s, cc, d) in sketch if s == sk and cc == c and d[0] == kind and d[1] == param] + n_final, mem, ins, qry = sketch[(sk, c, ks[-1])] + xs = [1e3, n_final] + ys = [mem, mem] if row == 0 else [ins * x / n_final + qry for x in xs] + ax.plot(xs, ys, "--", color=SKETCH_GRAYS[i % 4], lw=1.6, label=f"sketch {c}") + ax.set_xscale("log") + ax.set_yscale("log") + ax.set_xlabel("items N") + if col == 0: + ax.set_ylabel(what) + if row == 0: + ax.set_title(f"{title}\n{kind} {param}", loc="left", fontsize=10) + ax.legend(fontsize=7, labelcolor=INK2) +fig.suptitle("Exact computation (polars, solid) vs sketch (dashed): the gap is the saving; " + "N* = first N where exact ≥ 10× / 100× sketch. Serial on an idle machine.", + x=0.01, ha="left", color=INK, fontsize=11) +fig.savefig(out, dpi=110) +print(out) diff --git a/scripts/plot_saturation_curves.py b/scripts/plot_saturation_curves.py new file mode 100644 index 00000000..9dd681e7 --- /dev/null +++ b/scripts/plot_saturation_curves.py @@ -0,0 +1,147 @@ +"""Error vs N per data distribution, one figure per sketch, with theoretical bounds. + +Usage: plot_saturation_curves.py out/saturation_curve.csv OUT_DIR (needs matplotlib, pandas, numpy) +""" +import os +import sys + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 +import pandas as pd # noqa: E402 + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from study_saturation import n_saturation # noqa: E402 + +curve = pd.read_csv(sys.argv[1]) +outdir = sys.argv[2] + +INK, INK2, MUTED, SURF, LINE, BAND = "#0b0b0b", "#52514e", "#898781", "#fcfcfb", "#2a78d6", "#cde2fb" +SAT = "#eb6834" +plt.rcParams.update({ + "font.size": 9, "axes.edgecolor": MUTED, "axes.labelcolor": INK2, + "xtick.color": MUTED, "ytick.color": MUTED, "axes.titlecolor": INK, + "axes.spines.top": False, "axes.spines.right": False, + "axes.grid": True, "grid.color": "#e8e7e3", "grid.linewidth": 0.6, + "figure.facecolor": SURF, "axes.facecolor": SURF, +}) +NAMES = { + "cms-fastpath-vector2d": "CMS (rows=3 cols=1024) · error = are_top100", + "countsketch-fastpath-vector2d": "CountSketch (rows=3 cols=1024) · error = are_top100", + "cms-heap-topk-fastpath-vector2d": "CMS-heap top-k (rows=3 cols=1024) · precision@k (higher is better)", + "hll": "HLL (lg_k=12) · relative error", + "kll-percall": "KLL (k=200) · mean rank error", + "dd": "DDSketch (alpha=0.01) · mean rank error", +} +# The config each figure shows (the bounds below are for these); the curve +# may hold the whole config grid. +CONFIG = { + "cms-fastpath-vector2d": "rows=3 cols=1024", + "countsketch-fastpath-vector2d": "rows=3 cols=1024", + "cms-heap-topk-fastpath-vector2d": "rows=3 cols=1024", + "hll": "lg_k=12", + "kll-percall": "k=200", + "dd": "alpha=0.01", +} +KLAB = {1000: "1K", 100000: "100K", 10000000: "10M"} +BOUND = "#52514e" + +import math # noqa: E402 +import numpy as np # noqa: E402 + +W, M_HLL, K_KLL, ALPHA_DD, TOPK = 1024, 2 ** 12, 200, 0.01, 32 +EPS_CMS = math.e / W # CMS: |est - f| <= eps * N, prob >= 1 - e^-3 +EPS_CS = math.sqrt(3 / W) # CountSketch: |est - f| <= eps * ||f||_2 + + +def zipf_p(theta, k): + w = np.arange(1, int(k) + 1, dtype=float) ** (-theta) + return w / w.sum() + + +def bound(sk, theta, k): + """Theoretical error bound for one panel, in the metric's own units.""" + if sk.startswith("cms-heap-topk"): + p = zipf_p(theta, k) + kk = min(TOPK, len(p) - 1) + return float(np.sum(p[:kk] - p[kk] > EPS_CMS)) / kk + if sk.startswith("cms"): + p = zipf_p(theta, k)[:100] + return float(np.mean(EPS_CMS / p)) + if sk.startswith("countsketch"): + p = zipf_p(theta, k) + l2 = math.sqrt(float(np.sum(p ** 2))) + return float(np.mean(EPS_CS * l2 / p[:100])) + if sk == "hll": + return 2 * 1.04 / math.sqrt(M_HLL) + if sk.startswith("kll"): + return 2.296 / K_KLL ** 0.9723 + if sk == "dd": + return ALPHA_DD * theta / 2 # theta is the Pareto alpha here + return None + + +BOUND_TEXT = { + "cms-fastpath-vector2d": "bound: mean over top-100 of ε/p_i, ε=e/w (|f̂−f| ≤ εN) → N-independent; depends on θ, K", + "countsketch-fastpath-vector2d": "bound: mean over top-100 of ε‖p‖₂/p_i, ε=√(3/w) (|f̂−f| ≤ ε‖f‖₂) → N-independent; depends on θ, K", + "cms-heap-topk-fastpath-vector2d": "lower bound on precision@32: share of top-32 keys with p_i − p_33 > e/w (ignores heap eviction) → N-independent", + "hll": "bound: 2σ = 2·1.04/√m, m=4096 (3.25%) → independent of N; depends only on distinct count (smaller error below 2.5m distinct)", + "kll-percall": "bound: normalized rank error ε ≈ 2.296/k^0.9723 = 1.33% (99% conf., single quantile) → N-independent; exact while N fits the sketch", + "dd": "bound: mean rank error ≤ α_dd·a/2 (from value error α_dd=0.01 and Pareto density) → N-independent; depends on Pareto a", +} + +for sk, name in NAMES.items(): + c = curve[(curve.sketch == sk) & (curve.config == CONFIG[sk])] + if c.empty: + continue + pareto = (c.dist == "pareto").all() + params = sorted(c.param.unique()) + ks = [None] if pareto else sorted(c.cardinality.dropna().unique()) + nr, nc = len(ks), len(params) + fig, axes = plt.subplots(nr, nc, figsize=(2.9 * nc + 0.6, 2.5 * nr + 1.0), + sharex=True, sharey=False, squeeze=False, constrained_layout=True) + for r, k in enumerate(ks): + for j, p in enumerate(params): + ax = axes[r, j] + cc = c[(c.param == p) & ((c.cardinality == k) if k is not None else True)].sort_values("n") + if cc.empty: + ax.set_visible(False) + continue + logy = (cc.seed_mean_error > 0).all() and "topk" not in sk + ns, err, se = cc.n.tolist(), cc.seed_mean_error.tolist(), cc.seed_se.tolist() + floor = min(err) / 3 + lo = [max(e - 2 * s, floor) if logy else e - 2 * s for e, s in zip(err, se)] + hi = [e + 2 * s for e, s in zip(err, se)] + ax.fill_between(ns, lo, hi, color=BAND, lw=0) + ax.plot(ns, err, color=LINE, lw=2, marker="o", ms=3.5) + b = bound(sk, float(p), k) + off = b is not None and not logy and b > 5 * max(max(hi), 1e-12) + if b is not None and not off: + ax.axhline(b, color=BOUND, lw=1.4, ls=":") + nsat = n_saturation(ns, err, 0.10, 3, se) + if nsat is not None: + ax.axvline(nsat, color=SAT, lw=1.2, ls="--") + ax.plot([nsat], [err[ns.index(nsat)]], "o", ms=8, color=SAT, mec=SURF, mew=2) + tag = f"N_sat={nsat:.0e}".replace("e+0", "e") + else: + tag = "not saturated" + head = f"α={p:g}" if pareto else f"θ={p:g}, K={KLAB.get(int(k), k)}" + btxt = "" if b is None else f", bound={b:.3g}" + (" (off scale)" if off else "") + ax.set_title(f"{head}\n{tag}{btxt}", loc="left", fontsize=8.5) + ax.set_xscale("log") + if logy: + ax.set_yscale("log") + if r == nr - 1: + ax.set_xlabel("items inserted N") + if j == 0: + ax.set_ylabel("empirical error") + import textwrap + note = (f"{BOUND_TEXT[sk]} (dotted gray = theoretical bound). Each panel has its own " + f"y-scale (log when all errors > 0); line = seed mean, band = ±2 SE, dashed = " + "N_saturation (within max(10% of plateau, 2 SE) of the last-3 mean)") + fig.suptitle(name + "\n" + "\n".join(textwrap.wrap(note, 34 * nc)), + x=0.01, ha="left", color=INK, fontsize=11) + path = f"{outdir}/error_vs_N__{sk}.png" + fig.savefig(path, dpi=110) + plt.close(fig) + print(path) diff --git a/scripts/recommend_config.py b/scripts/recommend_config.py new file mode 100644 index 00000000..4eb9883c --- /dev/null +++ b/scripts/recommend_config.py @@ -0,0 +1,359 @@ +#!/usr/bin/env python3 +"""Turn worst-case data parameters per query into a recommended sketch config. + +Inputs: + --summary ASAPQuery dataset-analysis results/skew_summary.csv: one row per + (dataset, query_id, kind, weight, range) with the worst case a + sketch sees (worst_theta_cms, worst_K, min_N, max_N, + worst_alpha_rank, worst_alpha_memory), tail_class and target_*. + The older schema (window_len_s, lower/upper, K_win_max, + rows_win_min/max, no targets) is also read; see `read_summary`. + --curves saturation_curve.csv of the config grid (error per N, seed mean). + --curves-big optional larger-N curves (the 1e9 subset), used for a point + when N exceeds the grid's largest N and the point is there. + --saturation / --saturation-big saturation.csv files for n_sat (the big + one wins when it has the point) and, once the cost phase has + run, insert/merge/query CPU and memory_bytes. + --merge-curves optional saturation_merge_curve.csv (branch merged-accuracy). + +Candidate families: key queries -> CMS, CountSketch and the CMS-heap top-k; +value queries -> KLL and DDSketch. No query in the query sets is a +count-distinct, so HLL is never a candidate. + +Worst case per (query, range), rounded to the measured grid toward the harder +side, so the estimate is conservative: + - theta (key rows: the weight a per-key counter sees, count or value) is + rounded DOWN to the grid (flatter is harder for CMS/CountSketch/top-k); + - K (distinct keys per evaluation, K_win_max) is rounded UP; + - alpha for the error is worst_alpha_rank (steepest tail, hardest for rank + error) rounded UP; alpha for DDSketch cost is worst_alpha_memory + (heaviest tail, widest value range) rounded DOWN. + A worst case outside the grid is clamped to the grid edge and flagged. + +The error is read at N = max_N (items in the largest evaluation), linear in +log N between checkpoints; below the first checkpoint the first error is +used, above the last the last one (flagged "extrapolated"). max_N is the right +N for a merged window because a merged CMS/CountSketch/DDSketch equals one +sketch over the union. For top-k and KLL a merge is not exact: with +--merge-curves the error is read from the merged curve at the shard count +m = range_s / step_s (nearest measured m in log scale; the old schema has no +step, so its finest window length is used), otherwise the row is flagged +"single-sketch curve optimistic". + +Flags: "not saturated" when the point never saturated in the grid, "not +saturated at min_N" when min_N < n_sat, "light tail" for value rows whose +tail_class is light (alpha is not meaningful there), "target not met" when no +config reaches the target (the best config is then reported). + +Recommendation: per (query, range, family), the smallest config whose +estimate meets the target (error <= target; precision@k >= target). Smallest +is by memory_bytes when every config of the family has it, else by the +nominal size (rows*cols, k, 1/alpha). Cost columns: insert_ns_per_item = +insert_cpu_secs / N_cost, where N_cost is the point's last curve N (the cost +run's --size); merge_us_per_fold = merge_cpu_secs / (MERGE_SHARDS - 1) +(folding MERGE_SHARDS shard sketches into one); query_phase_us = query_cpu_secs: the +benchmark's whole query phase (every probe: keys seen, 101 quantiles, or a +repeated estimate), not one query. +They are empty until the cost phase has filled saturation.csv. rows=5 configs +without cost (--cost-rows 3) take 5/3 of the rows=3 cost with the same cols +(flagged "cost scaled from rows=3"). +""" + +import argparse +import bisect +import csv +import math +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from study_saturation import MERGE_SHARDS, load_curves, point_key # noqa: E402 + +# family -> (sketch variant, target column, higher is better) +FAMILIES = { + "cms": ("cms-fastpath-vector2d", "target_are_top100", False), + "countsketch": ("countsketch-fastpath-vector2d", "target_are_top100", False), + "topk": ("cms-heap-topk-fastpath-vector2d", "target_precision_at_k", True), + "kll": ("kll-percall", "target_rank_err", False), + "dd": ("dd", "target_rank_err", False), +} +KIND_FAMILIES = {"keys": ["cms", "countsketch", "topk"], "values": ["kll", "dd"]} +# Families whose merged error differs from one sketch over the union. +MERGE_INEXACT = {"topk", "kll"} +# Same defaults as ASAPQuery dataset-analysis fit_skew.py DEFAULT_TARGETS. +DEFAULT_TARGETS = { + "target_are_top100": 0.05, + "target_precision_at_k": 0.95, + "target_hll_rel_err": 0.02, + "target_rank_err": 0.01, +} + +OUTPUT_COLUMNS = [ + "dataset", "query_id", "range", "family", "config", "est_error", "target", + "meets_target", "n_sat", "flags", "memory_bytes", "insert_ns_per_item", + "merge_us_per_fold", "query_phase_us", "grid_param", "grid_K", "N", "shards", +] + + +def num(value): + """float(value), or None for an empty / non-numeric cell.""" + try: + x = float(value) + except (TypeError, ValueError): + return None + return None if math.isnan(x) else x + + +def read_summary(path): + """One dict per (dataset, query_id, range, kind) worst case. + + New schema: key rows without worst_theta_cms (the weight a counter sketch + does not see) are dropped. Old schema: range = window_len_s, theta = + `lower` of the weight a counter sees (value for `sum by`, else count), + K = K_win_max, N = rows_win_min/max, alpha_rank = upper, alpha_memory = + lower, step = the query's finest window length, default targets. + Rows without N or without their worst parameter (BOOM) are dropped. + """ + with open(path, newline="") as f: + rows = list(csv.DictReader(f)) + new = rows and "worst_theta_cms" in rows[0] + finest = {} + if not new: + for r in rows: + key = (r["dataset"], r["query_id"]) + s = num(r["window_len_s"]) + if s is not None: + finest[key] = min(finest.get(key, s), s) + out = [] + for r in rows: + kind = r["kind"] + if new: + q = {"range": r["range"], "range_s": num(r["range_s"]), "step_s": num(r["step_s"]), + "theta": num(r["worst_theta_cms"]), "K": num(r["worst_K"]), + "min_N": num(r["min_N"]), "max_N": num(r["max_N"]), + "alpha_rank": num(r["worst_alpha_rank"]), + "alpha_memory": num(r["worst_alpha_memory"])} + for t, default in DEFAULT_TARGETS.items(): + q[t] = default if num(r.get(t)) is None else num(r.get(t)) + else: + counter_weight = "value" if r["promql"].startswith("sum by") else "count" + if kind == "keys" and r["weight"] != counter_weight: + continue + q = {"range": r["window_len_s"], "range_s": num(r["window_len_s"]), + "step_s": finest.get((r["dataset"], r["query_id"])), + "theta": num(r["lower"]), "K": num(r["K_win_max"]), + "min_N": num(r["rows_win_min"]), "max_N": num(r["rows_win_max"]), + "alpha_rank": num(r["upper"]), "alpha_memory": num(r["lower"]), + **DEFAULT_TARGETS} + q.update(dataset=r["dataset"], query_id=r["query_id"], kind=kind, + tail_class=r.get("tail_class", "")) + needed = ["theta", "K"] if kind == "keys" else ["alpha_rank"] + if q["max_N"] is None or any(q[k] is None for k in needed): + continue + out.append(q) + return out + + +def round_down(x, grid): + """(largest grid value <= x, flag); clamps to the smallest value.""" + grid = sorted(grid) + i = bisect.bisect_right(grid, x + 1e-12) - 1 + if i < 0: + return grid[0], f"{x:g} below grid, used {grid[0]:g}" + return grid[i], "" + + +def round_up(x, grid): + """(smallest grid value >= x, flag); clamps to the largest value.""" + grid = sorted(grid) + i = bisect.bisect_left(grid, x - 1e-12) + if i == len(grid): + return grid[-1], f"{x:g} beyond grid, used {grid[-1]:g}" + return grid[i], "" + + +def error_at(curve, n): + """(error at n, flag) from a sorted [(n, error, se)] curve, linear in log n.""" + ns = [c[0] for c in curve] + if n <= ns[0]: + return curve[0][1], ("N below grid" if n < ns[0] else "") + if n >= ns[-1]: + return curve[-1][1], (f"extrapolated beyond N={ns[-1]:g}" if n > ns[-1] else "") + i = bisect.bisect_right(ns, n) + (n0, e0, _), (n1, e1, _) = curve[i - 1], curve[i] + w = (math.log(n) - math.log(n0)) / (math.log(n1) - math.log(n0)) + return e0 + w * (e1 - e0), "" + + +def nominal_size(config): + """Size order when memory_bytes is missing: rows*cols, k, 1/alpha.""" + kv = dict(part.split("=") for part in config.split()) + if "cols" in kv: + return int(kv["rows"]) * int(kv["cols"]) + if "k" in kv: + return int(kv["k"]) + return 1.0 / float(kv["alpha"]) + + +def load_saturation(path): + """{point key: saturation.csv row}; {} when path is None or missing.""" + if not path or not os.path.exists(path): + return {} + with open(path, newline="") as f: + return {point_key(r["sketch"], r["config"], r["dist"], r["param"], r["cardinality"]): r + for r in csv.DictReader(f)} + + +def load_merge_curves(path): + """{point key: {shards: sorted curve}} from a saturation_merge_curve.csv.""" + curves = {} + if not path: + return curves + with open(path, newline="") as f: + for r in csv.DictReader(f): + key = point_key(r["sketch"], r["config"], r["dist"], r["param"], r["cardinality"]) + curves.setdefault(key, {}).setdefault(int(r["shards"]), []).append( + (int(r["n"]), float(r["seed_mean_error"]), float(r["seed_se"]))) + return {k: {m: sorted(c) for m, c in per_m.items()} for k, per_m in curves.items()} + + +def cost_of(sat, key, n_cost): + """(memory_bytes, insert_ns_per_item, merge_us_per_fold, query_phase_us, flag), + '' where the cost phase has not filled saturation.csv; a rows=5 config + without cost takes 5/3 of the rows=3 one with the same cols.""" + row, scale, flag = sat.get(key), 1.0, "" + sketch, config = key[0], key[1] + if (row is None or row["memory_bytes"] == "") and config.startswith("rows=5 "): + row3 = sat.get((sketch, config.replace("rows=5 ", "rows=3 "), *key[2:])) + if row3 is not None and row3["memory_bytes"] != "": + row, scale, flag = row3, 5.0 / 3.0, "cost scaled from rows=3" + if row is None or row["memory_bytes"] == "": + return "", "", "", "", "" + + def scaled(col, factor): + x = num(row[col]) + return "" if x is None else x * scale * factor + + return (scaled("memory_bytes", 1.0), scaled("insert_cpu_secs", 1e9 / n_cost), + scaled("merge_cpu_secs", 1e6 / (MERGE_SHARDS - 1)), + scaled("query_cpu_secs", 1e6), flag) + + +def recommend(q, curves, curves_big, sat, sat_big, merge): + """One output row per candidate family of query-range q.""" + out = [] + for family in KIND_FAMILIES[q["kind"]]: + sketch, target_col, higher_better = FAMILIES[family] + target = q[target_col] + points = [k for k in curves if k[0] == sketch] + if not points: + continue + configs = sorted({k[1] for k in points}, key=nominal_size) + common = [] + if q["kind"] == "keys": + param, f1 = round_down(q["theta"], {k[3] for k in points}) + card, f2 = round_up(q["K"], {int(k[4]) for k in points}) + cost_param, card = param, str(card) + common += [f"theta {f1}" if f1 else "", f"K {f2}" if f2 else ""] + else: + param, f1 = round_up(q["alpha_rank"], {k[3] for k in points}) + cost_param = q["alpha_memory"] if q["alpha_memory"] is not None else q["alpha_rank"] + cost_param, _ = round_down(cost_param, {k[3] for k in points}) + card = "" + common += [f"alpha {f1}" if f1 else ""] + if q["tail_class"] == "light": + common.append("light tail") + n = q["max_N"] + shards = 1 + if q["range_s"] and q["step_s"]: + shards = max(1, round(q["range_s"] / q["step_s"])) + candidates = [] + for config in configs: + key = (sketch, config, points[0][2], float(param), card) + curve = curves.get(key) + if not curve: + continue + flags = list(common) + if n > curve[-1][0] and key in curves_big: + curve = curves_big[key] + if family in MERGE_INEXACT and shards > 1: + per_m = merge.get(key) + if per_m: + m = min(per_m, key=lambda m: abs(math.log(m / shards))) + curve = per_m[m] + flags.append(f"merged curve at m={m}") + elif merge: + flags.append("no merge curve for point; single-sketch curve optimistic") + else: + flags.append("single-sketch curve optimistic") + est, f = error_at(curve, n) + flags.append(f) + srow = sat_big.get(key) or sat.get(key) + n_sat = srow["n_sat"] if srow else "" + if n_sat == "not_saturated": + flags.append("not saturated") + elif num(n_sat) is not None and q["min_N"] is not None and q["min_N"] < num(n_sat): + flags.append("not saturated at min_N") + n_cost = curves[key][-1][0] + cost_key = (sketch, config, key[2], float(cost_param), card) + *cost, cflag = cost_of(sat, cost_key, n_cost) + flags.append(cflag) + meets = est >= target if higher_better else est <= target + candidates.append({ + "dataset": q["dataset"], "query_id": q["query_id"], "range": q["range"], + "family": family, "config": config, "est_error": est, "target": target, + "meets_target": meets, "n_sat": n_sat, + "flags": "; ".join(x for x in flags if x), + "memory_bytes": cost[0], "insert_ns_per_item": cost[1], + "merge_us_per_fold": cost[2], "query_phase_us": cost[3], + "grid_param": param, "grid_K": card, "N": int(n), "shards": shards, + }) + if not candidates: + continue + if all(c["memory_bytes"] != "" for c in candidates): + candidates.sort(key=lambda c: c["memory_bytes"]) + meeting = [c for c in candidates if c["meets_target"]] + if meeting: + best = meeting[0] + else: + sign = -1 if higher_better else 1 + best = min(candidates, key=lambda c: sign * c["est_error"]) + best["flags"] = "; ".join(x for x in ["target not met", best["flags"]] if x) + out.append(best) + return out + + +def main(argv=None): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + "--summary", + default="/mydata/ASAPQuery/asap-tools/dataset-analysis/results/skew_summary.csv") + parser.add_argument("--curves", default="out_grid_1e7/saturation_curve.csv") + parser.add_argument("--curves-big", default="out_1e9/saturation_curve.csv") + parser.add_argument("--saturation", default="out_grid_1e7/saturation.csv") + parser.add_argument("--saturation-big", default="out_1e9/saturation.csv") + parser.add_argument("--merge-curves", help="saturation_merge_curve.csv (optional)") + parser.add_argument("--out", default="out/recommendations.csv") + args = parser.parse_args(argv) + + curves = load_curves(args.curves) + curves_big = load_curves(args.curves_big) if os.path.exists(args.curves_big) else {} + sat = load_saturation(args.saturation) + sat_big = load_saturation(args.saturation_big) + merge = load_merge_curves(args.merge_curves) + rows = [] + for q in read_summary(args.summary): + rows.extend(recommend(q, curves, curves_big, sat, sat_big, merge)) + os.makedirs(os.path.dirname(args.out) or ".", exist_ok=True) + with open(args.out, "w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=OUTPUT_COLUMNS) + writer.writeheader() + writer.writerows(rows) + print(f"Done. {len(rows)} rows in {args.out}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/study_saturation.py b/scripts/study_saturation.py new file mode 100644 index 00000000..195d79d0 --- /dev/null +++ b/scripts/study_saturation.py @@ -0,0 +1,496 @@ +#!/usr/bin/env python3 +"""Find the stream length N at which each sketch's accuracy stops changing. + +For every (sketch, config, distribution point) the study runs approxbench once +per checkpoint N and seed (a fresh `--size N` stream each time, since accuracy +is only scored after the full stream), averages the error over seeds, and +reports N_sat (see `n_saturation`). At the final N it adds one cost run for +insert/merge/query CPU and memory_bytes, and runs the family's exact polars +baseline at every checkpoint N to find the crossover N* (see `n_star`). + +Frequency, top-k and cardinality sketches run on Zipf(theta) over K keys; +quantile sketches run on Pareto(alpha, scale 1000) floored to i64 (the exact +quantile baseline is i64 only), which is unbounded, so K is blank. +`BENCH_WARMUP_SECS` defaults to 0 here: the cost numbers are indicative, and +scripts/export_rqe_optimizer_costs.sh stays the source of optimizer costs. + +--phase accuracy runs only the (parallel) accuracy runs; --phase cost reads +saturation_curve.csv from --out and runs only the serial cost runs, so CPU +can be timed later on a quiet machine. Both take the same grid arguments. +--resume keeps the complete curves of an interrupted accuracy run; +--cost-rows 3 times only the rows=3 Vector2D configs. + +Writes, under --out: + saturation_accuracy.jsonl, saturation_cost.jsonl, + exact_cost.jsonl raw approxbench records + (cost ones keyed by point) + saturation_curve.csv seed-mean error per N + saturation.csv one row per point (cost + columns blank after + --phase accuracy) + crossover.csv N* per point +""" + +import argparse +import csv +import json +import math +import os +import subprocess +import sys +from concurrent.futures import ThreadPoolExecutor + +FREQ_CONFIGS = [f"rows={r} cols={c}" for r in (3, 5) for c in (256, 1024, 4096, 16384)] + +# (family, variant, configs, comparator, error metric). Variants follow +# scripts/export_rqe_optimizer_costs.sh; configs step 4x in the size knob +# (asap HLL is compiled at lg_k 12, 14, 16 only, so it has no lg_k 10). +SKETCHES = [ + ("frequency", "cms-fastpath-vector2d", FREQ_CONFIGS, "frequency", "are_top100"), + ("frequency", "countsketch-fastpath-vector2d", FREQ_CONFIGS, "frequency", "are_top100"), + ("topk", "cms-heap-topk-fastpath-vector2d", FREQ_CONFIGS, "topk", "precision_at_k"), + ("cardinality", "hll", ["lg_k=12", "lg_k=14", "lg_k=16"], "cardinality", "relative_error"), + ("quantile", "kll-percall", ["k=50", "k=200", "k=800"], "rank-error", "mean_rank_err"), + ("quantile", "dd", ["alpha=0.005", "alpha=0.01", "alpha=0.02", "alpha=0.05"], + "rank-error", "mean_rank_err"), +] + +# The exact polars baseline (variant, config) each family is compared with. +# The config is parsed and ignored; top-k shares the frequency group_by. +EXACT = { + "frequency": ("cms", "rows=3 cols=256"), + "topk": ("cms", "rows=3 cols=256"), + "cardinality": ("hll", "lg_k=12"), + "quantile": ("kll-cdf", "k=200"), +} +CROSSOVER_FACTORS = (10, 100) + +MERGE_SHARDS = 16 + +SUMMARY_COLUMNS = [ + "family", "sketch", "config", "dist", "param", "cardinality", "n_sat", + "final_error", "error_metric", "insert_cpu_secs", "merge_cpu_secs", + "query_cpu_secs", "memory_bytes", +] +CROSSOVER_COLUMNS = [ + "family", "sketch", "config", "dist", "param", "cardinality", "n_sat", + "sketch_memory_bytes", "sketch_cpu_secs", +] + [f"{m}_n_star_{f}x" for m in ("memory", "cpu") for f in CROSSOVER_FACTORS] +CURVE_COLUMNS = [ + "family", "sketch", "config", "dist", "param", "cardinality", "n", + "seed_mean_error", "seed_se", +] + + +def checkpoints(n_min, n_max, per_decade): + """Log-spaced sizes from n_min to n_max inclusive, per_decade per decade.""" + lo, hi = math.log10(n_min), math.log10(n_max) + steps = round((hi - lo) * per_decade) + ns = [round(10 ** (lo + i / per_decade)) for i in range(steps + 1)] + return sorted(set(ns)) + + +def n_saturation(ns, errors, tolerance, tail, ses=None, abs_tol=1e-9): + """Smallest checkpoint from which the error stays on its plateau. + + The plateau is the mean of the last `tail` seed-mean errors. A checkpoint + is in band when |error - plateau| <= max(tolerance * |plateau|, 2 * se, + abs_tol), where se is that checkpoint's standard error across seeds + (`ses`, 0 when omitted), so seed noise alone never breaks the plateau; + abs_tol only matters when the plateau is (near) zero. N_sat is ns[i] for + the smallest i such that every checkpoint i..end is in band. Comparing + magnitudes makes direction irrelevant (precision_at_k rises, errors fall). + + Returns None ("not_saturated") when there are fewer than `tail` + checkpoints, or when no checkpoint before the tail qualifies (including a + tail that is not itself in band): the tail agreeing with its own mean says + nothing about saturation. + """ + if len(errors) < tail: + return None + plateau = sum(errors[-tail:]) / tail + ses = ses or [0.0] * len(errors) + first = len(errors) + for i in range(len(errors) - 1, -1, -1): + band = max(tolerance * abs(plateau), 2 * ses[i], abs_tol) + if abs(errors[i] - plateau) > band: + break + first = i + if first > len(errors) - tail - 1: + return None + return ns[first] + + +def n_star(ns, exact, sketch, factor): + """Smallest ns[i] from which exact >= factor * sketch holds at every later + checkpoint, or None. "From which it stays" rather than "first time": + fixed per-run overheads (polars start-up) can make the exact side look + expensive at the smallest N and cheap again further up.""" + star = None + for n, e, s in zip(reversed(ns), reversed(exact), reversed(sketch)): + if e < factor * s: + break + star = n + return star + + +def point_key(sketch, config, dist, param, cardinality): + """How a point is matched across CSVs: param and cardinality compared as + numbers, so a CSV round-tripped through pandas ("1000.0") still matches.""" + card = str(int(float(cardinality))) if str(cardinality) != "" else "" + return (sketch, config, dist, float(param), card) + + +def load_curves(path): + """{point key: sorted [(n, seed-mean error, se)]} from a saturation_curve.csv.""" + curves = {} + with open(path, newline="") as f: + for r in csv.DictReader(f): + key = point_key(r["sketch"], r["config"], r["dist"], r["param"], r["cardinality"]) + curves.setdefault(key, []).append( + (int(r["n"]), float(r["seed_mean_error"]), float(r["seed_se"]))) + return {key: sorted(curve) for key, curve in curves.items()} + + +def load_saved(path, key): + """{key(*record["key"]): record} from a cost JSONL of keyed records.""" + with open(path) as f: + records = [json.loads(line) for line in f] + return {key(*r.pop("key")): r for r in records} + + +def saved_record(saved, key, path): + """The saved record for `key`, or exit when the cost phase never ran it.""" + if key not in saved: + sys.exit(f"{path} has no record for {key}; rerun --phase cost with this " + "grid and --cost-rows") + return saved[key] + + +def read_curve(path, points, ns, tolerance, tail): + """(point, n_sat, final error) per point, from an earlier accuracy phase. + + Exits when a point's curve is missing or was measured over other sizes + than `ns`, i.e. the grid arguments differ from the accuracy run's. + """ + curves = load_curves(path) + results = [] + for p in points: + curve = curves.get(point_key(p[1], p[2], *p[6:]), []) + if [n for n, _, _ in curve] != ns: + sys.exit(f"{path} has no curve over these sizes for {p[1]} ({p[2]}) " + f"{p[6]}={p[7]} K={p[8]}; pass the accuracy run's grid arguments") + means = [e for _, e, _ in curve] + ses = [se for _, _, se in curve] + results.append((p, n_saturation(ns, means, tolerance, tail, ses), means[-1])) + return results + + +def mean_and_se(values): + """Mean and standard error of the mean (0 for a single value).""" + mean = sum(values) / len(values) + if len(values) < 2: + return mean, 0.0 + var = sum((v - mean) ** 2 for v in values) / (len(values) - 1) + return mean, math.sqrt(var / len(values)) + + +def run(binary, args): + """Run approxbench sketchbench and return the last JSONL record it prints.""" + out = subprocess.run( + [binary, "sketchbench"] + args, + check=True, capture_output=True, text=True, stdin=subprocess.DEVNULL, + ).stdout + return json.loads(out.strip().splitlines()[-1]) + + +def error(record, metric): + acc = record["bench"]["accuracy"] + if metric == "are_top100" and metric not in acc: + # Fewer than 100 distinct keys were seen, so the top 100 is all of them. + return acc["are_all"] + return acc[metric] + + +def query_secs(record): + """CPU seconds of one query: the query phase repeats the query over its + probes (all keys seen, 101 quantiles, or a fixed repeat count), and the + exact baseline uses a different probe count, so both sides are compared + per query.""" + cpu, wall = record.get("query_cpu_time_ms"), record.get("query_wall_time_ms") + rate = record.get("query_throughput_items_per_sec") + if not (cpu and wall and rate): + return 0.0 + ops = rate["mean"] * wall["mean"] / 1000.0 + return cpu_secs(record, "query") / ops if ops else 0.0 + + +def cpu_secs(record, op): + cpu = record.get(f"{op}_cpu_time_ms") + if not cpu: + return "" + return (cpu["user_ms"]["mean"] + cpu["sys_ms"]["mean"]) / 1000.0 + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--binary", default="./target/release/approxbench") + parser.add_argument("--out", default="out") + parser.add_argument("--families", default="frequency,topk,cardinality,quantile") + parser.add_argument("--one-config", action="store_true", + help="only the first config of each sketch") + parser.add_argument("--thetas", default="0,0.5,0.8,1.0,1.2,1.5,2.0") + parser.add_argument("--cardinalities", default="1000,100000,10000000") + parser.add_argument("--alphas", default="1.1,1.5,2,3") + parser.add_argument("--n-min", type=float, default=1e3) + parser.add_argument("--n-max", type=float, default=1e8) + parser.add_argument("--per-decade", type=int, default=4) + parser.add_argument("--seeds", type=int, default=5) + parser.add_argument("--tolerance", type=float, default=0.10) + parser.add_argument("--plateau-tail", type=int, default=3) + parser.add_argument("--jobs", type=int, default=1, + help="parallel accuracy runs (cost runs are always serial)") + parser.add_argument("--phase", choices=["accuracy", "cost", "crossover", "all"], + default="all", + help="cost reads saturation_curve.csv from --out; crossover " + "recomputes crossover.csv from the cost phase's JSONL in --out") + parser.add_argument("--points-from", + help="CSV with sketch,config,dist,param,cardinality columns " + "(e.g. a filtered saturation.csv): run only those points") + parser.add_argument("--resume", action="store_true", + help="accuracy: keep the points whose curve over these sizes is " + "already in --out's saturation_curve.csv, run the rest") + parser.add_argument("--cost-rows", type=int, + help="cost: measure only this rows= value of the Vector2D sketches " + "(other rows get blank cost columns and no crossover row)") + args = parser.parse_args() + + families = args.families.split(",") + thetas = [float(t) for t in args.thetas.split(",")] + cardinalities = [int(k) for k in args.cardinalities.split(",")] + alphas = [float(a) for a in args.alphas.split(",")] + ns = checkpoints(args.n_min, args.n_max, args.per_decade) + seeds = list(range(1, args.seeds + 1)) + os.environ.setdefault("BENCH_WARMUP_SECS", "0") + os.makedirs(args.out, exist_ok=True) + + # A point is (family, variant, config, comparator, metric, dataset args, + # dist, param, cardinality). + points = [] + for family, variant, configs, comparator, metric in SKETCHES: + if family not in families: + continue + for config in configs[:1] if args.one_config else configs: + if family == "quantile": + for alpha in alphas: + # --cardinality is required by the CLI but ignored for pareto. + dataset = ["--dataset", "pareto", "--pareto-alpha", str(alpha), + "--pareto-scale", "1000", "--cardinality", "1", + "--dtype", "i64"] + points.append((family, variant, config, comparator, metric, + dataset, "pareto", alpha, "")) + continue + for theta in thetas: + for k in cardinalities: + dataset = ["--dataset", "zipf", "--zipf-s", str(theta), + "--cardinality", str(k), "--dtype", "i64"] + points.append((family, variant, config, comparator, metric, + dataset, "zipf", theta, k)) + if args.points_from: + with open(args.points_from, newline="") as f: + wanted = {point_key(r["sketch"], r["config"], r["dist"], r["param"], + r["cardinality"]) for r in csv.DictReader(f)} + points = [p for p in points if point_key(p[1], p[2], *p[6:]) in wanted] + if len(points) != len(wanted): + sys.exit(f"--points-from names {len(wanted)} points but only {len(points)} " + "are in the grid; widen --families/--thetas/--cardinalities/--alphas") + print(f"{len(points)} points x {len(ns)} sizes x {len(seeds)} seeds", file=sys.stderr) + + def accuracy(point, n, seed): + _, variant, config, comparator, metric, dataset, *_ = point + return run(args.binary, [ + "--variant", variant, "--library", "lib", "--config", config, + "--operations", "query", "--metrics", "accuracy", + "--comparator", comparator, "--runs", "1", "--warmup-runs", "0", + "--size", str(n), "--seed", str(seed), + ] + dataset) + + if args.phase in ("cost", "crossover"): + results = read_curve(os.path.join(args.out, "saturation_curve.csv"), points, ns, + args.tolerance, args.plateau_tail) + else: + curve_path = os.path.join(args.out, "saturation_curve.csv") + # --resume keeps complete curves and writes them back first, so a + # point cut off mid-write is dropped and rerun, and a second + # interruption loses nothing that was kept. Rows of points outside + # this grid are written back as they are, so a narrower resume + # deletes nothing. + done, kept_rows = {}, [] + if args.resume and os.path.exists(curve_path): + grid = {point_key(p[1], p[2], *p[6:]) for p in points} + done = {key: c for key, c in load_curves(curve_path).items() + if key in grid and [n for n, _, _ in c] == ns} + with open(curve_path, newline="") as f: + for r in csv.DictReader(f): + key = point_key(r["sketch"], r["config"], r["dist"], r["param"], + r["cardinality"]) + if key not in grid or key in done: + kept_rows.append([r[c] for c in CURVE_COLUMNS]) + raw = open(os.path.join(args.out, "saturation_accuracy.jsonl"), + "a" if args.resume else "w") + curve_file = open(curve_path, "w", newline="") + curve = csv.writer(curve_file) + curve.writerow(CURVE_COLUMNS) + curve.writerows(kept_rows) + curve_file.flush() + # Submit everything up front so the pool stays full; results are consumed + # in point order, so the curve CSV fills point by point. + with ThreadPoolExecutor(max_workers=args.jobs) as pool: + futures = [ + None if point_key(p[1], p[2], *p[6:]) in done else + [[pool.submit(accuracy, p, n, s) for s in seeds] for n in ns] for p in points + ] + print(f"{len(done)} points kept, {sum(f is not None for f in futures)} to run", + file=sys.stderr) + results = [] + try: + for point, per_n in zip(points, futures): + family, variant, config, _, metric, _, dist, param, k = point + if per_n is None: + kept = done[point_key(variant, config, dist, param, k)] + means = [mean for _, mean, _ in kept] + n_sat = n_saturation(ns, means, args.tolerance, args.plateau_tail, + [se for _, _, se in kept]) + results.append((point, n_sat, means[-1])) + continue + means, ses = [], [] + for n, per_seed in zip(ns, per_n): + records = [f.result() for f in per_seed] + for r in records: + raw.write(json.dumps(r) + "\n") + mean, se = mean_and_se([error(r, metric) for r in records]) + means.append(mean) + ses.append(se) + curve.writerow([family, variant, config, dist, param, k, n, mean, se]) + curve_file.flush() + n_sat = n_saturation(ns, means, args.tolerance, args.plateau_tail, ses) + results.append((point, n_sat, means[-1])) + print(f" {variant} ({config}) {dist}={param} K={k}: n_sat={n_sat}", + file=sys.stderr) + except BaseException: + # A failed run (or Ctrl-C) should surface now, not after the queue drains. + for f in (f for per_n in futures if per_n for fs in per_n for f in fs): + f.cancel() + raise + raw.close() + curve_file.close() + + # Cost at the final N, one serial run per point with the merge setup of + # export_rqe_optimizer_costs.sh. --phase accuracy leaves these blank. + rows, records = [], [] + cost = args.phase != "accuracy" + rebuild = args.phase == "crossover" + cost_path = os.path.join(args.out, "saturation_cost.jsonl") + saved = load_saved(cost_path, point_key) if rebuild else None + cost_raw = open(cost_path, "w") if cost and not rebuild else None + for point, n_sat, final_error in results: + family, variant, config, _, metric, dataset, dist, param, k = point + record = {} + # rows=5 costs ~5/3 of rows=3 (insert, query and memory are per row). + skip = args.cost_rows is not None and config.startswith("rows=") \ + and not config.startswith(f"rows={args.cost_rows} ") + if rebuild and not skip: + record = saved_record(saved, point_key(variant, config, dist, param, k), + cost_path) + elif cost and not skip: + record = run(args.binary, [ + "--variant", variant, "--library", "lib", "--config", config, + "--operations", "insert,query,merge", + "--metrics", "throughput,cpu,memory", + "--merge-shards", str(MERGE_SHARDS), "--runs", "3", "--warmup-runs", "1", + "--size", str(ns[-1]), "--seed", str(seeds[0]), "--flat", + ] + dataset) + cost_raw.write(json.dumps( + {"key": [variant, config, dist, param, k], **record}) + "\n") + records.append(record) + rows.append([ + family, variant, config, dist, param, k, + "not_saturated" if n_sat is None else n_sat, + final_error, metric, cpu_secs(record, "insert"), + cpu_secs(record, "merge"), cpu_secs(record, "query"), + record.get("memory_bytes", ""), + ]) + if cost_raw: + cost_raw.close() + + if not rebuild: + with open(os.path.join(args.out, "saturation.csv"), "w", newline="") as f: + writer = csv.writer(f) + writer.writerow(SUMMARY_COLUMNS) + writer.writerows(rows) + print(f"Done. {os.path.join(args.out, 'saturation.csv')}", file=sys.stderr) + if not cost: + return 0 + + # The family's exact baseline at every checkpoint, once per (baseline, + # dataset). Memory is its memory_bytes after prepare (the buffered stream + # plus any count table), the same self-reported formula as the sketch's; + # CPU is insert + prepare (the polars pass) + one query. + exact = {} + measured = [(r, record) for r, record in zip(results, records) if record] + exact_path = os.path.join(args.out, "exact_cost.jsonl") + saved_exact = load_saved(exact_path, lambda v, ds, n, op: (v, tuple(ds), n, op)) \ + if rebuild else None + with open(exact_path, "a" if rebuild else "w") as exact_raw: + for (point, _, _), _ in measured: + variant, config = EXACT[point[0]] + for n in ns: + key = (variant, tuple(point[5]), n) + if key in exact: + continue + base = [ + "--variant", variant, "--library", "polars", "--config", config, + "--runs", "3", "--warmup-runs", "1", "--size", str(n), + "--seed", str(seeds[0]), "--flat", + ] + point[5] + if rebuild: + timed, prepared = (saved_record(saved_exact, key + (op,), exact_path) + for op in ("timed", "prepared")) + else: + timed = run(args.binary, base + [ + "--operations", "insert,query", "--metrics", "throughput,cpu,memory"]) + prepared = run(args.binary, base + [ + "--operations", "prepare", "--metrics", "latency,cpu,memory"]) + for op, r in (("timed", timed), ("prepared", prepared)): + exact_raw.write(json.dumps( + {"key": [variant, point[5], n, op], **r}) + "\n") + exact[key] = (prepared["memory_bytes"], cpu_secs(timed, "insert") + + query_secs(timed) + cpu_secs(prepared, "prepare")) + + with open(os.path.join(args.out, "crossover.csv"), "w", newline="") as f: + writer = csv.writer(f) + writer.writerow(CROSSOVER_COLUMNS) + for (point, n_sat, _), record in measured: + family, variant, config, _, _, dataset, dist, param, k = point + per_n = [exact[(EXACT[family][0], tuple(dataset), n)] for n in ns] + insert, query = cpu_secs(record, "insert"), query_secs(record) + memory = record["memory_bytes"] + # Measured at the final N only: insert CPU is a per-item cost, so it + # scales with N; one query's CPU and memory are taken as constant. + sketch_cpu = [insert * n / ns[-1] + query for n in ns] + stars = [n_star(ns, [m for m, _ in per_n], [memory] * len(ns), x) + for x in CROSSOVER_FACTORS] + stars += [n_star(ns, [c for _, c in per_n], sketch_cpu, x) + for x in CROSSOVER_FACTORS] + writer.writerow([ + family, variant, config, dist, param, k, + "not_saturated" if n_sat is None else n_sat, memory, insert + query, + ] + ["not_reached" if n is None else n for n in stars]) + print(f"Done. {os.path.join(args.out, 'crossover.csv')}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_recommend_config.py b/scripts/test_recommend_config.py new file mode 100644 index 00000000..b9f534a9 --- /dev/null +++ b/scripts/test_recommend_config.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +"""Unit tests for recommend_config on tiny synthetic curve / summary CSVs. + +Run: python3 scripts/test_recommend_config.py +""" + +import csv +import os +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from recommend_config import ( # noqa: E402 + OUTPUT_COLUMNS, error_at, main, round_down, round_up, +) +from study_saturation import CURVE_COLUMNS, SUMMARY_COLUMNS # noqa: E402 + +SUMMARY_HEADER = [ + "dataset", "query_id", "promql", "kind", "weight", "range", "range_s", "step_s", + "worst_theta_cms", "worst_K", "min_N", "max_N", "worst_alpha_rank", + "worst_alpha_memory", "target_are_top100", "target_precision_at_k", + "target_hll_rel_err", "target_rank_err", "tail_class", +] +CMS = "cms-fastpath-vector2d" +SMALL, BIG = "rows=3 cols=256", "rows=3 cols=1024" + + +def write_csv(path, header, rows): + with open(path, "w", newline="") as f: + writer = csv.writer(f) + writer.writerow(header) + writer.writerows(rows) + + +def key_query(theta, K, min_N, max_N, target=""): + return ["d", "q", "sum by (k) (x)", "keys", "value", "instant", "", "60", + theta, K, min_N, max_N, "", "", target, "", "", "", ""] + + +class RoundingTest(unittest.TestCase): + def test_theta_rounds_down_and_K_rounds_up(self): + self.assertEqual(round_down(0.97, [0.5, 0.8, 1.0]), (0.8, "")) + self.assertEqual(round_down(1.0, [0.5, 0.8, 1.0]), (1.0, "")) + self.assertEqual(round_up(538, [1000, 100000]), (1000, "")) + self.assertEqual(round_up(1000, [1000, 100000]), (1000, "")) + + def test_outside_the_grid_clamps_and_flags(self): + value, flag = round_up(2e7, [1000, 10000000]) + self.assertEqual(value, 10000000) + self.assertIn("beyond grid", flag) + value, flag = round_down(0.3, [0.5, 1.0]) + self.assertEqual(value, 0.5) + self.assertIn("below grid", flag) + + def test_error_interpolates_in_log_n_and_flags_extrapolation(self): + curve = [(100, 0.1, 0), (10000, 0.3, 0)] + self.assertAlmostEqual(error_at(curve, 1000)[0], 0.2) + self.assertEqual(error_at(curve, 10 ** 6), (0.3, "extrapolated beyond N=10000")) + + +class RecommendTest(unittest.TestCase): + def setUp(self): + self.dir = tempfile.mkdtemp() + self.curves = os.path.join(self.dir, "curve.csv") + # theta 0.5 is harder than 1.0; the small config meets 0.05 only at 1.0. + rows = [] + for theta, (small, big) in {0.5: (0.2, 0.04), 1.0: (0.03, 0.01)}.items(): + for config, err in ((SMALL, small), (BIG, big)): + for n in (1000, 100000): + rows.append(["frequency", CMS, config, "zipf", theta, 1000, n, err, 0]) + write_csv(self.curves, CURVE_COLUMNS, rows) + self.saturation = os.path.join(self.dir, "saturation.csv") + self.write_saturation(n_sat=1000, memory="") + + def write_saturation(self, n_sat, memory): + rows = [] + for theta in (0.5, 1.0): + for config in (SMALL, BIG): + mem = "" if memory == "" else memory[config] + cost = ["", "", ""] if memory == "" else [0.5, 0.15, 0.001] + rows.append(["frequency", CMS, config, "zipf", theta, 1000, n_sat, 0, + "are_top100", *cost, mem]) + write_csv(self.saturation, SUMMARY_COLUMNS, rows) + + def run_cms(self, query): + summary = os.path.join(self.dir, "summary.csv") + write_csv(summary, SUMMARY_HEADER, [query]) + out = os.path.join(self.dir, "rec.csv") + main(["--summary", summary, "--curves", self.curves, + "--curves-big", os.path.join(self.dir, "none.csv"), + "--saturation", self.saturation, + "--saturation-big", os.path.join(self.dir, "none.csv"), "--out", out]) + with open(out, newline="") as f: + rows = list(csv.DictReader(f)) + self.assertEqual(list(rows[0]), OUTPUT_COLUMNS) + return next(r for r in rows if r["family"] == "cms") + + def test_theta_between_grid_points_uses_the_lower_one(self): + row = self.run_cms(key_query(0.97, 538, 5000, 5000)) + self.assertEqual(row["grid_param"], "0.5") + self.assertEqual(row["grid_K"], "1000") + self.assertEqual(row["config"], BIG) + + def test_smallest_config_meeting_the_target_wins(self): + row = self.run_cms(key_query(1.0, 1000, 5000, 5000)) + self.assertEqual(row["config"], SMALL) + self.assertEqual(row["meets_target"], "True") + + def test_no_config_meets_the_target_reports_the_best(self): + row = self.run_cms(key_query(0.5, 1000, 5000, 5000, target="0.001")) + self.assertEqual(row["config"], BIG) + self.assertEqual(row["meets_target"], "False") + self.assertIn("target not met", row["flags"]) + + def test_missing_cost_leaves_columns_empty_and_orders_by_nominal_size(self): + row = self.run_cms(key_query(0.5, 1000, 5000, 5000, target="0.5")) + self.assertEqual(row["config"], SMALL) + for col in ("memory_bytes", "insert_ns_per_item", "merge_us_per_fold", "query_phase_us"): + self.assertEqual(row[col], "") + + def test_present_cost_orders_by_memory_and_converts_units(self): + # The nominally bigger config is the smaller one in memory. + self.write_saturation(n_sat=1000, memory={SMALL: 9000, BIG: 5000}) + row = self.run_cms(key_query(0.5, 1000, 5000, 5000, target="0.5")) + self.assertEqual(row["config"], BIG) + self.assertEqual(float(row["memory_bytes"]), 5000) + self.assertAlmostEqual(float(row["insert_ns_per_item"]), 0.5 * 1e9 / 100000) + self.assertAlmostEqual(float(row["merge_us_per_fold"]), 0.15 * 1e6 / 15) + self.assertAlmostEqual(float(row["query_phase_us"]), 1000) + + def test_min_N_below_n_sat_is_flagged(self): + self.write_saturation(n_sat=100000, memory="") + row = self.run_cms(key_query(1.0, 1000, 5000, 5000)) + self.assertIn("not saturated at min_N", row["flags"]) + row = self.run_cms(key_query(1.0, 1000, 200000, 200000)) + self.assertNotIn("not saturated", row["flags"]) + self.assertIn("extrapolated", row["flags"]) + + def test_never_saturated_is_flagged(self): + self.write_saturation(n_sat="not_saturated", memory="") + row = self.run_cms(key_query(1.0, 1000, 5000, 5000)) + self.assertIn("not saturated", row["flags"]) + + +class MergeCurveTest(unittest.TestCase): + def test_top_k_uses_the_nearest_shard_count(self): + d = tempfile.mkdtemp() + topk = "cms-heap-topk-fastpath-vector2d" + curves = os.path.join(d, "curve.csv") + write_csv(curves, CURVE_COLUMNS, [ + ["topk", topk, SMALL, "zipf", 1.0, 1000, n, 0.99, 0] for n in (1000, 100000)]) + merge = os.path.join(d, "merge.csv") + write_csv(merge, CURVE_COLUMNS[:-2] + ["shards", "seed_mean_error", "seed_se"], [ + ["topk", topk, SMALL, "zipf", 1.0, 1000, n, m, err, 0] + for m, err in ((4, 0.9), (64, 0.5)) for n in (1000, 100000)]) + summary = os.path.join(d, "summary.csv") + query = key_query(1.0, 1000, 5000, 5000) + query[5:7] = ["1h", "3600"] # range 1h at step 60: m = 60, nearest 64 + write_csv(summary, SUMMARY_HEADER, [query]) + out = os.path.join(d, "rec.csv") + base = ["--summary", summary, "--curves", curves, "--curves-big", "none", + "--saturation", "none", "--saturation-big", "none", "--out", out] + main(base + ["--merge-curves", merge]) + with open(out, newline="") as f: + row = next(csv.DictReader(f)) + self.assertAlmostEqual(float(row["est_error"]), 0.5) + self.assertIn("merged curve at m=64", row["flags"]) + self.assertEqual(row["shards"], "60") + main(base) + with open(out, newline="") as f: + row = next(csv.DictReader(f)) + self.assertAlmostEqual(float(row["est_error"]), 0.99) + self.assertIn("single-sketch curve optimistic", row["flags"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_study_saturation.py b/scripts/test_study_saturation.py new file mode 100644 index 00000000..5e07677f --- /dev/null +++ b/scripts/test_study_saturation.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python3 +"""Unit tests for study_saturation's pure logic, and its cost phase against a +fake approxbench (so no real cost run happens). + +Run: python3 scripts/test_study_saturation.py +""" + +import csv +import json +import os +import subprocess +import sys +import tempfile +import textwrap +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from study_saturation import checkpoints, n_saturation, n_star, read_curve # noqa: E402 + +SCRIPT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "study_saturation.py") + +NS = [10, 20, 30, 40, 50, 60] + + +class NSaturationTest(unittest.TestCase): + def test_seed_noise_widens_the_band(self): + # A small plateau with seed noise wider than 10% of it: without the + # standard-error term this never saturates. + errors = [0.05, 0.019, 0.013, 0.018, 0.013, 0.017] + self.assertIsNone(n_saturation(NS, errors, 0.10, 3)) + ses = [0.004, 0.002, 0.002, 0.002, 0.002, 0.002] + self.assertEqual(n_saturation(NS, errors, 0.10, 3, ses), 20) + + def test_falling_error_saturates_where_it_enters_the_band(self): + errors = [1.0, 0.5, 0.21, 0.2, 0.2, 0.19] + self.assertEqual(n_saturation(NS, errors, 0.10, 3), 30) + + def test_a_flat_curve_saturates_at_the_first_checkpoint(self): + self.assertEqual(n_saturation(NS, [0.3] * 6, 0.10, 3), 10) + + def test_rising_metric_works_the_same(self): + # precision_at_k: higher is better; only stability matters. + errors = [0.2, 0.5, 0.9, 0.95, 0.96, 0.97] + self.assertEqual(n_saturation(NS, errors, 0.10, 3), 30) + + def test_an_excursion_after_the_band_moves_n_sat_past_it(self): + errors = [0.2, 0.5, 0.2, 0.2, 0.2, 0.2] + self.assertEqual(n_saturation(NS, errors, 0.10, 3), 30) + + def test_only_the_tail_in_band_is_not_saturated(self): + errors = [1.0, 0.8, 0.6, 0.2, 0.2, 0.2] + self.assertIsNone(n_saturation(NS, errors, 0.10, 3)) + + def test_a_still_falling_tail_is_not_saturated(self): + errors = [1.0, 0.5, 0.25, 0.12, 0.06, 0.03] + self.assertIsNone(n_saturation(NS, errors, 0.10, 3)) + + def test_too_few_checkpoints_is_not_saturated(self): + self.assertIsNone(n_saturation(NS[:2], [0.1, 0.1], 0.10, 3)) + self.assertIsNone(n_saturation(NS[:3], [0.1, 0.1, 0.1], 0.10, 3)) + + def test_zero_plateau_uses_the_absolute_fallback(self): + errors = [0.5, 0.0, 0.0, 0.0, 0.0, 0.0] + self.assertEqual(n_saturation(NS, errors, 0.10, 3), 20) + + +class CheckpointsTest(unittest.TestCase): + def test_four_per_decade_spans_both_ends(self): + ns = checkpoints(1e3, 1e5, 4) + self.assertEqual(ns[0], 1000) + self.assertEqual(ns[-1], 100000) + self.assertEqual(len(ns), 9) + self.assertEqual(ns[1], 1778) + + +class NStarTest(unittest.TestCase): + def test_first_size_where_exact_reaches_the_factor(self): + ns = [10, 100, 1000] + self.assertEqual(n_star(ns, [5, 50, 500], [5, 5, 5], 10), 100) + self.assertEqual(n_star(ns, [5, 50, 500], [5, 5, 5], 100), 1000) + + def test_equal_counts_as_reached(self): + self.assertEqual(n_star([10], [50], [5], 10), 10) + + def test_must_stay_reached_to_the_largest_size(self): + # A fixed exact-side overhead makes the ratio high at the smallest N, + # dip below the factor, then grow again: N* is where it stays. + ns = [10, 100, 1000, 10000] + self.assertEqual(n_star(ns, [50, 20, 80, 900], [1, 10, 10, 10], 10), 10000) + + def test_never_reached_is_none(self): + # Exact and sketch both linear in N: the ratio never grows. + self.assertIsNone(n_star([10, 100], [20, 200], [10, 100], 10)) + + +def write_csv(path, header, rows): + with open(path, "w", newline="") as f: + writer = csv.writer(f) + writer.writerow(header) + writer.writerows(rows) + + +CURVE_HEADER = ["family", "sketch", "config", "dist", "param", "cardinality", "n", + "seed_mean_error", "seed_se"] + + +class ReadCurveTest(unittest.TestCase): + def test_recomputes_n_sat_and_matches_param_as_a_number(self): + point = ("cardinality", "hll", "lg_k=12", "cardinality", "relative_error", [], + "zipf", 1.0, 1000) + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "curve.csv") + errors = [1.0, 0.5, 0.21, 0.2, 0.2, 0.19] + write_csv(path, CURVE_HEADER, + [["cardinality", "hll", "lg_k=12", "zipf", "1", 1000, n, e, 0] + for n, e in zip(NS, errors)]) + [(p, n_sat, final)] = read_curve(path, [point], NS, 0.10, 3) + self.assertIs(p, point) + self.assertEqual(n_sat, 30) + self.assertEqual(final, 0.19) + + def test_other_sizes_exit(self): + point = ("cardinality", "hll", "lg_k=12", "cardinality", "relative_error", [], + "zipf", 1.0, 1000) + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "curve.csv") + write_csv(path, CURVE_HEADER, + [["cardinality", "hll", "lg_k=12", "zipf", 1.0, 1000, 10, 0.1, 0]]) + with self.assertRaises(SystemExit): + read_curve(path, [point], NS, 0.10, 3) + + +# Stands in for approxbench: the sketch is 1000 bytes at 1 s insert + 0.5 s +# query at the final N; the exact baseline holds 8 bytes per item and spends +# 1e-6 s per item. +FAKE_APPROXBENCH = textwrap.dedent("""\ + import json, sys + a = sys.argv + n = int(a[a.index("--size") + 1]) + lib = a[a.index("--library") + 1] + ops = a[a.index("--operations") + 1] + cpu = lambda s: {"user_ms": {"mean": s * 1000}, "sys_ms": {"mean": 0}} + if "accuracy" in a: + r = {"bench": {"accuracy": {"relative_error": 0.01, "are_top100": 0.01}}} + elif lib == "lib": + r = {"memory_bytes": 1000, "insert_cpu_time_ms": cpu(1.0), + "query_cpu_time_ms": cpu(0.5), "merge_cpu_time_ms": cpu(0.1)} + elif ops == "prepare": + r = {"memory_bytes": 8 * n, "prepare_cpu_time_ms": cpu(1e-6 * n)} + else: + r = {"memory_bytes": 8 * n, "insert_cpu_time_ms": cpu(0), + "query_cpu_time_ms": cpu(0)} + print(json.dumps(r)) +""") + + +def fake_binary(d): + binary = os.path.join(d, "approxbench") + with open(binary, "w") as f: + f.write("#!" + sys.executable + "\n" + FAKE_APPROXBENCH) + os.chmod(binary, 0o755) + return binary + + +class AccuracyPhaseTest(unittest.TestCase): + def run_study(self, d, points_csv): + return subprocess.run([ + sys.executable, SCRIPT, "--binary", fake_binary(d), "--out", d, + "--phase", "accuracy", "--families", "cardinality", "--one-config", + "--thetas", "0,1.0", "--cardinalities", "1000", "--n-max", "1e6", + "--per-decade", "1", "--seeds", "2", "--points-from", points_csv, + ], capture_output=True) + + def test_points_from_selects_points_and_cost_columns_stay_blank(self): + with tempfile.TemporaryDirectory() as d: + points = os.path.join(d, "points.csv") + write_csv(points, ["sketch", "config", "dist", "param", "cardinality"], + [["hll", "lg_k=12", "zipf", "1", "1000"]]) + self.assertEqual(self.run_study(d, points).returncode, 0) + with open(os.path.join(d, "saturation.csv"), newline="") as f: + [summary] = list(csv.DictReader(f)) + self.assertFalse(os.path.exists(os.path.join(d, "crossover.csv"))) + self.assertFalse(os.path.exists(os.path.join(d, "saturation_cost.jsonl"))) + self.assertEqual((summary["param"], summary["n_sat"]), ("1.0", "1000")) + self.assertEqual(summary["memory_bytes"], "") + + def test_points_outside_the_grid_exit(self): + with tempfile.TemporaryDirectory() as d: + points = os.path.join(d, "points.csv") + write_csv(points, ["sketch", "config", "dist", "param", "cardinality"], + [["hll", "lg_k=12", "zipf", "2.0", "1000"]]) + self.assertNotEqual(self.run_study(d, points).returncode, 0) + + +class ResumeTest(unittest.TestCase): + def test_keeps_complete_curves_and_reruns_the_rest(self): + with tempfile.TemporaryDirectory() as d: + ns = checkpoints(1e3, 1e6, 1) + curve = os.path.join(d, "saturation_curve.csv") + # theta 0 is complete (error 0.5, which the fake would not give); + # theta 1 was cut off after two sizes. + write_csv(curve, CURVE_HEADER, + [["cardinality", "hll", "lg_k=12", "zipf", 0.0, 1000, n, 0.5, 0] + for n in ns] + + [["cardinality", "hll", "lg_k=12", "zipf", 1.0, 1000, n, 0.9, 0] + for n in ns[:2]] + # theta 2 is outside this resume's grid and must survive. + + [["cardinality", "hll", "lg_k=12", "zipf", 2.0, 1000, n, 0.7, 0] + for n in ns]) + subprocess.run([ + sys.executable, SCRIPT, "--binary", fake_binary(d), "--out", d, + "--phase", "accuracy", "--families", "cardinality", "--one-config", + "--thetas", "0,1.0", "--cardinalities", "1000", "--n-max", "1e6", + "--per-decade", "1", "--seeds", "2", "--resume", + ], check=True, capture_output=True) + with open(curve, newline="") as f: + rows = list(csv.DictReader(f)) + with open(os.path.join(d, "saturation.csv"), newline="") as f: + summary = list(csv.DictReader(f)) + with open(os.path.join(d, "saturation_accuracy.jsonl")) as f: + raw = f.readlines() + errors = {} + for r in rows: + errors.setdefault(r["param"], []).append(float(r["seed_mean_error"])) + self.assertEqual(errors, {"0.0": [0.5] * 4, "1.0": [0.01] * 4, "2.0": [0.7] * 4}) + self.assertEqual([r["final_error"] for r in summary], ["0.5", "0.01"]) + # Only the rerun point was measured: 4 sizes x 2 seeds. + self.assertEqual(len(raw), 8) + + +class CostPhaseTest(unittest.TestCase): + def test_cost_rows_measures_only_that_rows_value(self): + with tempfile.TemporaryDirectory() as d: + ns = checkpoints(1e3, 1e6, 1) + configs = ["rows=3 cols=256", "rows=5 cols=256"] + write_csv(os.path.join(d, "saturation_curve.csv"), CURVE_HEADER, + [["frequency", "cms-fastpath-vector2d", c, "zipf", 1.0, 1000, n, 0.01, 0] + for c in configs for n in ns]) + points = os.path.join(d, "points.csv") + write_csv(points, ["sketch", "config", "dist", "param", "cardinality"], + [["cms-fastpath-vector2d", c, "zipf", 1.0, 1000] for c in configs]) + subprocess.run([ + sys.executable, SCRIPT, "--binary", fake_binary(d), "--out", d, + "--phase", "cost", "--families", "frequency", "--thetas", "1.0", + "--cardinalities", "1000", "--n-max", "1e6", "--per-decade", "1", + "--points-from", points, "--cost-rows", "3", + ], check=True, capture_output=True) + with open(os.path.join(d, "saturation.csv"), newline="") as f: + summary = list(csv.DictReader(f)) + with open(os.path.join(d, "crossover.csv"), newline="") as f: + crossover = list(csv.DictReader(f)) + self.assertEqual([(r["config"], r["memory_bytes"]) for r in summary], + [("rows=3 cols=256", "1000"), ("rows=5 cols=256", "")]) + self.assertEqual([r["config"] for r in crossover], ["rows=3 cols=256"]) + + def test_cost_phase_writes_crossover_from_an_existing_curve(self): + with tempfile.TemporaryDirectory() as d: + binary = fake_binary(d) + ns = checkpoints(1e3, 1e6, 1) + write_csv(os.path.join(d, "saturation_curve.csv"), CURVE_HEADER, + [["cardinality", "hll", "lg_k=12", "zipf", 1.0, 1000, n, 0.01, 0] + for n in ns]) + subprocess.run([ + sys.executable, SCRIPT, "--binary", binary, "--out", d, "--phase", "cost", + "--families", "cardinality", "--one-config", "--thetas", "1.0", + "--cardinalities", "1000", "--n-max", "1e6", "--per-decade", "1", + ], check=True, capture_output=True) + with open(os.path.join(d, "crossover.csv"), newline="") as f: + [row] = list(csv.DictReader(f)) + with open(os.path.join(d, "saturation.csv"), newline="") as f: + [summary] = list(csv.DictReader(f)) + self.assertEqual(summary["n_sat"], "1000") + self.assertEqual(summary["insert_cpu_secs"], "1.0") + # 8N >= 10 * 1000 bytes first at N = 1e4; >= 100 * 1000 at N = 1e5. + self.assertEqual(row["memory_n_star_10x"], "10000") + self.assertEqual(row["memory_n_star_100x"], "100000") + # Sketch CPU at N is 1 * N / 1e6 plus one query; the fake reports no + # query throughput, so one query costs 0. Exact is 1e-6 * N, so it + # never reaches 10x. + self.assertEqual(row["cpu_n_star_10x"], "not_reached") + self.assertEqual(row["sketch_cpu_secs"], "1.0") + + def test_crossover_phase_rebuilds_from_saved_records(self): + with tempfile.TemporaryDirectory() as d: + binary = fake_binary(d) + ns = checkpoints(1e3, 1e6, 1) + write_csv(os.path.join(d, "saturation_curve.csv"), CURVE_HEADER, + [["cardinality", "hll", "lg_k=12", "zipf", 1.0, 1000, n, 0.01, 0] + for n in ns]) + args = [sys.executable, SCRIPT, "--binary", binary, "--out", d, + "--families", "cardinality", "--one-config", "--thetas", "1.0", + "--cardinalities", "1000", "--n-max", "1e6", "--per-decade", "1"] + subprocess.run(args + ["--phase", "cost"], check=True, capture_output=True) + path = os.path.join(d, "crossover.csv") + with open(path) as f: + measured = f.read() + os.remove(path) + # No binary needed: the rebuild reads only the saved JSONL. + subprocess.run(args[:3] + ["/nonexistent"] + args[4:] + ["--phase", "crossover"], + check=True, capture_output=True) + with open(path) as f: + rebuilt = f.read() + self.assertEqual(rebuilt, measured) + + def test_crossover_phase_matches_saved_records_by_key(self): + with tempfile.TemporaryDirectory() as d: + ns = checkpoints(1e3, 1e6, 1) + configs = ["rows=3 cols=256", "rows=5 cols=256"] + write_csv(os.path.join(d, "saturation_curve.csv"), CURVE_HEADER, + [["frequency", "cms-fastpath-vector2d", c, "zipf", 1.0, 1000, n, 0.01, 0] + for c in configs for n in ns]) + points = os.path.join(d, "points.csv") + write_csv(points, ["sketch", "config", "dist", "param", "cardinality"], + [["cms-fastpath-vector2d", c, "zipf", 1.0, 1000] for c in configs]) + args = [sys.executable, SCRIPT, "--binary", fake_binary(d), "--out", d, + "--families", "frequency", "--thetas", "1.0", "--cardinalities", + "1000", "--n-max", "1e6", "--per-decade", "1", "--points-from", points] + subprocess.run(args + ["--phase", "cost"], check=True, capture_output=True) + # Tell the rows=5 record apart, and store it first. + cost = os.path.join(d, "saturation_cost.jsonl") + with open(cost) as f: + records = [json.loads(line) for line in f] + records[1]["memory_bytes"] = 5000 + with open(cost, "w") as f: + f.writelines(json.dumps(r) + "\n" for r in reversed(records)) + subprocess.run(args + ["--phase", "crossover", "--cost-rows", "3"], + check=True, capture_output=True) + with open(os.path.join(d, "crossover.csv"), newline="") as f: + [row] = list(csv.DictReader(f)) + # A point the cost phase never measured exits. + with open(cost, "w") as f: + f.write(json.dumps(records[0]) + "\n") + missing = subprocess.run(args + ["--phase", "crossover"], capture_output=True) + self.assertEqual((row["config"], row["sketch_memory_bytes"]), ("rows=3 cols=256", "1000")) + self.assertNotEqual(missing.returncode, 0) + self.assertIn(b"has no record", missing.stderr) + + +if __name__ == "__main__": + unittest.main()