From a702b9d8efc4f74176889cdec56ce6a3e45b5000 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 10 Oct 2026 03:46:15 +0000 Subject: [PATCH 01/18] polars: tag Hydra baseline subset keys by column (#74) The polars exact baselines keyed a label subset by its `;`-joined values, asap_sketchlib 0.2.2's subkey. 0.3.0's Hydra tags each value with its column (`label0:a;label1:b`), so on label columns that share a value domain the baseline merged a key1 group with the same key2 group and scored nonzero against itself. subset_key now builds the 0.3.0 key over hydra_shared's `label{i}` schema (escaping `\`, `:` and `;`), empty labels keep their column, and every lookup goes through it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FhcWJExZcmVqS6r6rEtjis --- configs/datagen/hydra_columns.yaml | 11 ++-- .../datagen/hydra_kll_disjoint_labels.yaml | 7 +-- sketch-bench/src/wrappers/hydra_kll/polars.rs | 2 +- sketch-bench/src/wrappers/polars_shared.rs | 58 +++++++++++++++---- 4 files changed, 56 insertions(+), 22 deletions(-) diff --git a/configs/datagen/hydra_columns.yaml b/configs/datagen/hydra_columns.yaml index edfd1250..50cb4b5c 100644 --- a/configs/datagen/hydra_columns.yaml +++ b/configs/datagen/hydra_columns.yaml @@ -29,12 +29,11 @@ # Reported throughput is records per second, which is the denominator that stays # comparable as `d` changes. # -# The columns here share the default alphabet, so their value domains overlap: a -# query for label `"abc"` cannot distinguish the key1 group from the key2 group, -# and the estimate absorbs both. This is #74, and it is the reason the `polars` -# rows do not score zero on this file. Give each column its own -# `string.alphabet` to make the domains disjoint, and the baselines drop to -# exactly zero. +# The columns here share the default alphabet, so their value domains overlap. +# Neither side aliases them: asap_sketchlib 0.3.0 keys each subset by column +# (`label0:abc`), and the `polars` exact baselines build the same tagged keys +# (`polars_shared::subset_key`), so a key1 value never matches the same key2 +# value and the baselines score exactly zero (#74). column_num: 3 column_label: [key1, key2, value] diff --git a/configs/datagen/hydra_kll_disjoint_labels.yaml b/configs/datagen/hydra_kll_disjoint_labels.yaml index 23392aee..a936152b 100644 --- a/configs/datagen/hydra_kll_disjoint_labels.yaml +++ b/configs/datagen/hydra_kll_disjoint_labels.yaml @@ -1,8 +1,7 @@ # `hydra_columns_f64.yaml` with a separate alphabet per label column, so a key1 -# value can never render the same as a key2 value (#74). On the shared alphabet -# a key1 query also picks up the matching key2 group, and that collision shows -# up as rank error; here the `polars` baseline scores zero and any rank error -# is the sketch's. `export_rqe_optimizer_costs.sh` priced `hydra-kll` on it +# value can never render the same as a key2 value. Since #74 the sketch and the +# `polars` baseline both key subsets by column, so the shared alphabet no longer +# aliases either; this file stays as the disjoint-domain case. `export_rqe_optimizer_costs.sh` priced `hydra-kll` on it # until #174; HydraKLL has no saturation curve, so nothing measures it now. column_num: 3 diff --git a/sketch-bench/src/wrappers/hydra_kll/polars.rs b/sketch-bench/src/wrappers/hydra_kll/polars.rs index 777cfcee..44ecb2f4 100644 --- a/sketch-bench/src/wrappers/hydra_kll/polars.rs +++ b/sketch-bench/src/wrappers/hydra_kll/polars.rs @@ -47,7 +47,7 @@ impl PolarsSubpopQuantile { /// rank interval contains `phi * n`, which is what the rank-error /// comparator scores against, so an exact answer scores zero. pub fn estimate_subpop_quantile(&self, labels: &[&str], phi: f64) -> f64 { - let Some(values) = self.sorted.get(&labels.join(";")) else { + let Some(values) = self.sorted.get(&prefix_key(labels)) else { return f64::NAN; }; if values.is_empty() { diff --git a/sketch-bench/src/wrappers/polars_shared.rs b/sketch-bench/src/wrappers/polars_shared.rs index dcf6344e..a6e49846 100644 --- a/sketch-bench/src/wrappers/polars_shared.rs +++ b/sketch-bench/src/wrappers/polars_shared.rs @@ -262,10 +262,11 @@ impl PolarsQuantileCore { } } -/// The `;`-joined key of one label subset, in column order: the subset key -/// asap_sketchlib 0.2.2's `Hydra::update` built, so the baseline matches a -/// value in any label column. 0.3.0's Hydra keys a subset by column, so the -/// two differ when label columns share a value domain. +/// The key of the label subset `mask` picks out of `parts`, in column order, +/// each value tagged with its column (`label0:a;label2:c`, with `\`, `:` and `;` +/// escaped): asap_sketchlib 0.3.0's subkey over the `label{i}` schema +/// `hydra_shared` declares. So a value in one column never matches the same +/// value in another (#74), in the baseline as in the sketch. pub fn subset_key(parts: &[&str], mask: usize) -> String { let mut out = String::new(); for (j, part) in parts.iter().enumerate() { @@ -273,17 +274,27 @@ pub fn subset_key(parts: &[&str], mask: usize) -> String { if !out.is_empty() { out.push(';'); } - out.push_str(part); + out.push_str(&format!("label{j}:")); + for ch in part.chars() { + if matches!(ch, '\\' | ':' | ';') { + out.push('\\'); + } + out.push(ch); + } } } out } +/// The key a probe of the leading label columns `labels` reads. +pub fn prefix_key(labels: &[&str]) -> String { + subset_key(labels, (1 << labels.len()) - 1) +} + /// Expand one record into `(subset_key, value)` rows, one per non-empty subset -/// of its labels. Empty label parts are dropped, as asap_sketchlib 0.2.2's -/// Hydra did (0.3.0's keeps them in their column). +/// of its labels. An empty label keeps its column, as 0.3.0's Hydra does. pub fn fan_out(key: &str, value: &V, keys: &mut Vec, values: &mut Vec) { - let parts: Vec<&str> = key.split(';').filter(|s| !s.is_empty()).collect(); + let parts: Vec<&str> = key.split(';').collect(); for mask in 1..(1usize << parts.len()) { keys.push(subset_key(&parts, mask)); values.push(value.clone()); @@ -350,7 +361,7 @@ impl PolarsSubpopFrequencyCore { pub fn query(&self, labels: &[&str], value: &T) -> f64 { self.counts - .get(&(labels.join(";"), value.count_key())) + .get(&(prefix_key(labels), value.count_key())) .copied() .unwrap_or(0) as f64 } @@ -415,7 +426,7 @@ impl PolarsSubpopCardinalityCore { } pub fn query(&self, labels: &[&str]) -> f64 { - self.distinct.get(&labels.join(";")).copied().unwrap_or(0) as f64 + self.distinct.get(&prefix_key(labels)).copied().unwrap_or(0) as f64 } pub fn memory_bytes(&self) -> usize { @@ -484,7 +495,7 @@ impl PolarsSubpopVectorCore { } pub fn query(&self, labels: &[&str]) -> f64 { - self.exact.get(&labels.join(";")).copied().unwrap_or(0.0) + self.exact.get(&prefix_key(labels)).copied().unwrap_or(0.0) } pub fn memory_bytes(&self) -> usize { @@ -521,3 +532,28 @@ fn grouped_values( .expect("polars group_by collect"), ) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn subset_keys_tag_each_value_with_its_column() { + assert_eq!(subset_key(&["a", "b", "c"], 0b101), "label0:a;label2:c"); + assert_eq!(subset_key(&["x:y;z", ""], 0b11), r"label0:x\:y\;z;label1:"); + assert_eq!(prefix_key(&["a", "b"]), "label0:a;label1:b"); + } + + /// #74: with both columns holding `a`, the column-0 group `a` saw one + /// record. The old untagged keys counted it twice, once per column. + #[test] + fn a_value_shared_by_two_columns_is_counted_once() { + let mut core = PolarsSubpopFrequencyCore::::default(); + core.update(&("a;a".to_string(), 7)); + core.update(&("a;b".to_string(), 7)); + core.finalize(); + assert_eq!(core.query(&["a"], &7), 2.0); + assert_eq!(core.query(&["a", "a"], &7), 1.0); + assert_eq!(core.query(&["b"], &7), 0.0); + } +} From 912c3abd978831ef5e2316724f5d73a1ee7ba3aa Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 10 Oct 2026 04:02:49 +0000 Subject: [PATCH 02/18] polars: write subset key tags without a temporary String; fix test comment Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FhcWJExZcmVqS6r6rEtjis --- configs/datagen/hydra_kll_disjoint_labels.yaml | 5 +++-- sketch-bench/src/wrappers/polars_shared.rs | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/configs/datagen/hydra_kll_disjoint_labels.yaml b/configs/datagen/hydra_kll_disjoint_labels.yaml index a936152b..cff3f56d 100644 --- a/configs/datagen/hydra_kll_disjoint_labels.yaml +++ b/configs/datagen/hydra_kll_disjoint_labels.yaml @@ -1,8 +1,9 @@ # `hydra_columns_f64.yaml` with a separate alphabet per label column, so a key1 # value can never render the same as a key2 value. Since #74 the sketch and the # `polars` baseline both key subsets by column, so the shared alphabet no longer -# aliases either; this file stays as the disjoint-domain case. `export_rqe_optimizer_costs.sh` priced `hydra-kll` on it -# until #174; HydraKLL has no saturation curve, so nothing measures it now. +# aliases either; this file stays as the disjoint-domain case. +# `export_rqe_optimizer_costs.sh` priced `hydra-kll` on it until #174; HydraKLL +# has no saturation curve, so nothing measures it now. column_num: 3 column_label: [key1, key2, value] diff --git a/sketch-bench/src/wrappers/polars_shared.rs b/sketch-bench/src/wrappers/polars_shared.rs index a6e49846..d049a1b9 100644 --- a/sketch-bench/src/wrappers/polars_shared.rs +++ b/sketch-bench/src/wrappers/polars_shared.rs @@ -5,6 +5,7 @@ use crate::wrappers::hll::CardinalityValue; use ::polars::prelude::*; use std::collections::HashMap; +use std::fmt::Write as _; /// the resulting key→count table for O(1) per-key queries. /// Shared by `cms/polars` and `countsketch/polars`. @@ -274,7 +275,7 @@ pub fn subset_key(parts: &[&str], mask: usize) -> String { if !out.is_empty() { out.push(';'); } - out.push_str(&format!("label{j}:")); + let _ = write!(out, "label{j}:"); for ch in part.chars() { if matches!(ch, '\\' | ':' | ';') { out.push('\\'); @@ -544,8 +545,8 @@ mod tests { assert_eq!(prefix_key(&["a", "b"]), "label0:a;label1:b"); } - /// #74: with both columns holding `a`, the column-0 group `a` saw one - /// record. The old untagged keys counted it twice, once per column. + /// #74: the column-0 group `a` holds 2 records (`a;a`, `a;b`). The old + /// untagged key counted 3: `a;a`'s `a` once per column. #[test] fn a_value_shared_by_two_columns_is_counted_once() { let mut core = PolarsSubpopFrequencyCore::::default(); From 653d6a8dbe4d6f9d34c5eafe24c44d9c000746da Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 10 Oct 2026 04:09:25 +0000 Subject: [PATCH 03/18] hydra: interleaved merge split; merge exactness and bound tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `--merge-split contiguous|interleaved` (default contiguous, unchanged) picks how merge runs split the stream into shards. Interleaved reorders the stream so the existing contiguous `partition` deals it round-robin by record, at the contiguous split's shard sizes, so every row's folds take it unchanged. Tests: hydra-cms, hydra-cs and hydra-hll grids folded from 4 and 7 shards, under both splits, answer every probed subpopulation exactly as the single stream; and every scored group on a fixed 20k-record stream lands inside its hydra doc §2.2 bound (CMS also never under). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FhcWJExZcmVqS6r6rEtjis --- README.md | 11 + aqpbm-cli/src/cli.rs | 11 + aqpbm-cli/src/main.rs | 5 + aqpbm-cli/src/rows/mod.rs | 4 +- aqpbm-cli/src/rows/statistic.rs | 13 +- docs/saturation_study.md | 3 +- sketch-bench/src/request.rs | 2 + sketch-bench/src/wrappers/hydra_shared.rs | 295 ++++++++++++++++++++++ sketch-bench/src/wrappers/mod.rs | 60 +++++ 9 files changed, 398 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f48e88d6..5d2925cd 100644 --- a/README.md +++ b/README.md @@ -447,6 +447,17 @@ approxbench: hydra-cms/lib config={"cell_cols":1024,"cell_rows":3,"cols":1024,"r `--spec configs/datagen/hydra_columns.yaml` is how user can define the synthetic table for insertion. The file can be found [under configs directory](./configs/datagen/hydra_columns.yaml). +**`--merge-shards`**: how many shards `merge` folds: the stream is split that +many ways, one sketch is fed per shard, and the sketches are folded into one. + +**`--merge-split`**: how the stream is split into those shards. `contiguous` +(the default) cuts it into consecutive stretches; `interleaved` deals it +round-robin by record, so each shard is a sample of the whole stream (records +`0, k, 2k, ...` go to the first of `k` shards). The shard sizes are the same +either way. CMS, CountSketch and HLL cells (and their Hydra grids) merge +exactly, so they answer identically under both; KLL and UnivMon merges are +lossy and can differ. + **`--flat`**: flatten record such that output data of multiple experiments is flatten into one json file. #### General Explanation diff --git a/aqpbm-cli/src/cli.rs b/aqpbm-cli/src/cli.rs index bd14e42a..6b911613 100644 --- a/aqpbm-cli/src/cli.rs +++ b/aqpbm-cli/src/cli.rs @@ -234,6 +234,17 @@ pub struct SketchbenchArgs { /// merge exactly, so a gap is a defect; for KLL it is the result. #[arg(long, default_value_t = 2, help_heading = "Measurement content")] pub merge_shards: usize, + /// How the merge operation splits the stream into its shards: `contiguous` + /// cuts it into stretches, `interleaved` deals it round-robin by record, so + /// each shard is a sample of the whole stream. The shard sizes are the same + /// either way. + #[arg( + long, + default_value = "contiguous", + value_parser = ["contiguous", "interleaved"], + help_heading = "Measurement content" + )] + pub merge_split: String, /// Path to append JSONL records to. `-` or omitted sends them to stdout. #[arg(long, help_heading = "Output")] diff --git a/aqpbm-cli/src/main.rs b/aqpbm-cli/src/main.rs index 8334d4dd..6ccef32e 100644 --- a/aqpbm-cli/src/main.rs +++ b/aqpbm-cli/src/main.rs @@ -50,6 +50,7 @@ use report_sink::ReportSink; // not know the set; it asks. use sketch_bench::registry; use sketch_bench::request::Requirement; +use sketch_bench::wrappers::MergeSplit; /// What is measured. No default and no `all`: a request names the measurements /// it wants, and a shorthand that swept every operation against every metric @@ -357,6 +358,10 @@ fn run_sketchbench(args: SketchbenchArgs) -> Result<()> { width, workers: args.workers.max(1), merge_shards: args.merge_shards, + merge_split: match args.merge_split.as_str() { + "interleaved" => MergeSplit::Interleaved, + _ => MergeSplit::Contiguous, + }, comparator: args.comparator.clone(), runs: args.runs, warmup_runs: args.warmup_runs, diff --git a/aqpbm-cli/src/rows/mod.rs b/aqpbm-cli/src/rows/mod.rs index e1f1c59c..5f146a55 100644 --- a/aqpbm-cli/src/rows/mod.rs +++ b/aqpbm-cli/src/rows/mod.rs @@ -7,8 +7,8 @@ use std::rc::Rc; use sketch_bench::request::{Dtype, Requirement}; use sketch_bench::wrappers::{ - BuildError, Folds, InsertBody, InsertStepBody, ParallelInsertBody, PrepareBody, QueryBody, - QueryPass, StepPass, + interleave, BuildError, Folds, InsertBody, InsertStepBody, MergeSplit, ParallelInsertBody, + PrepareBody, QueryBody, QueryPass, StepPass, }; /// A row that cannot be built at the requested config says so in its own words; diff --git a/aqpbm-cli/src/rows/statistic.rs b/aqpbm-cli/src/rows/statistic.rs index 3e6c0678..cea3cbf4 100644 --- a/aqpbm-cli/src/rows/statistic.rs +++ b/aqpbm-cli/src/rows/statistic.rs @@ -343,6 +343,7 @@ pub(super) fn scored_row( prepare: Option>, ) -> Result where + I: Clone, G: GroundTruth + 'static, G::Truth: 'static, G::Probe: 'static, @@ -350,6 +351,12 @@ where { let (probes, score) = questions(ground_truth, &table)?; let items = materialise(description, table)?; + // What the folds cut into shards: the stream itself, or reordered so the + // same cut deals it round-robin. + let merged = match req.merge_split { + MergeSplit::Contiguous => items.clone(), + MergeSplit::Interleaved => Rc::new(interleave(&items, shards(req))), + }; let mut bodies = Vec::with_capacity(want.len()); for &(operation, metric) in want { let n = passes(req, metric); @@ -371,7 +378,7 @@ where (Operation::Merge, Metric::Accuracy) => answered( merge.expect(SUPPORTED).2( &req.params, - items.clone(), + merged.clone(), probes.clone(), shards(req), n, @@ -381,11 +388,11 @@ where metric, ), (Operation::Merge, Metric::Latency) => stepped( - merge.expect(SUPPORTED).1(&req.params, items.clone(), shards(req), n) + merge.expect(SUPPORTED).1(&req.params, merged.clone(), shards(req), n) .map_err(cannot_build)?, ), (Operation::Merge, _) => timed( - merge.expect(SUPPORTED).0(&req.params, items.clone(), shards(req), n) + merge.expect(SUPPORTED).0(&req.params, merged.clone(), shards(req), n) .map_err(cannot_build)?, (shards(req) - 1) as u64, ), diff --git a/docs/saturation_study.md b/docs/saturation_study.md index 834f119f..a2ad6f7d 100644 --- a/docs/saturation_study.md +++ b/docs/saturation_study.md @@ -548,7 +548,8 @@ Crossover at theta = 1, K = 1e5 (frequency, top-k, HLL) and Pareto alpha 1.5 - **Accuracy after merging** is now measured (`--operations merge --metrics accuracy --merge-shards m`, or `--merge-shards-list 1,4,16,64` here, into `saturation_merge_curve.csv`): the N-item stream is split into m contiguous - shards, one sketch per shard is folded into one, and it is scored with the + shards (approxbench `--merge-split interleaved` deals it round-robin), one + sketch per shard is folded into one, and it is scored with the query's comparator against the whole-stream truth. A bounded run (first config per sketch, N up to 1e7, 3 seeds, theta 0.5/1.0/1.5, K 1e3/1e5/1e7, alpha 1.1/2) found: diff --git a/sketch-bench/src/request.rs b/sketch-bench/src/request.rs index 1a13d182..ef6f611c 100644 --- a/sketch-bench/src/request.rs +++ b/sketch-bench/src/request.rs @@ -118,6 +118,8 @@ pub struct Requirement { pub workers: usize, /// How many shards a merge measurement folds pub merge_shards: usize, + /// How a merge measurement splits the stream into those shards + pub merge_split: crate::wrappers::MergeSplit, /// A named comparator, or `None` for the row's default. pub comparator: Option, /// Measured runs, and the warm-ups run and discarded before them diff --git a/sketch-bench/src/wrappers/hydra_shared.rs b/sketch-bench/src/wrappers/hydra_shared.rs index 0521e338..2ccd7250 100644 --- a/sketch-bench/src/wrappers/hydra_shared.rs +++ b/sketch-bench/src/wrappers/hydra_shared.rs @@ -86,3 +86,298 @@ pub(crate) fn check_grid(rows: usize, cols: usize, algorithm: &str) -> Result<() pub(crate) fn grid_overhead_bytes(rows: usize, cols: usize) -> usize { (rows * cols + 1) * std::mem::size_of::() } + +#[cfg(test)] +mod tests { + use crate::params::{HydraCmsParams, HydraCsParams, HydraHllParams, ParamSet}; + use crate::wrappers::hydra_cms::sketchlib::merge_query_hydra_cms; + use crate::wrappers::hydra_cs::sketchlib::merge_query_hydra_cs; + use crate::wrappers::hydra_hll::sketchlib::merge_query_hydra_hll; + use crate::wrappers::{interleave, QueryPass}; + use std::collections::{HashMap, HashSet}; + use std::rc::Rc; + + /// Records over two labels, deterministic: `r0..r5` with weights halving + /// from 32, `s0..s4` uniform, and a value skewed towards 0 over about + /// 2000 distinct values. + fn records(n: usize) -> Vec<(String, i64)> { + let mut state = 0x9e37_79b9_7f4a_7c15u64; + let mut next = move || { + state = state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1); + state >> 11 + }; + (0..n) + .map(|_| { + let pick = (next() % 63) as u32; + let region = (0..6).find(|&r| pick < 64 - (64 >> (r + 1))).unwrap_or(5); + let service = next() % 5; + let u = (next() % 10_000) as f64 / 10_000.0; + (format!("r{region};s{service}"), (u * u * 2000.0) as i64) + }) + .collect() + } + + /// Every subpopulation the grid answers for: each first label alone and + /// each pair, as the query names them (leading columns). + fn groups(items: &[(String, i64)]) -> Vec> { + let mut out: Vec> = Vec::new(); + for (key, _) in items { + let parts: Vec = key.split(';').map(str::to_string).collect(); + for g in [parts[..1].to_vec(), parts] { + if !out.contains(&g) { + out.push(g); + } + } + } + out + } + + fn in_group(key: &str, group: &[String]) -> bool { + key.split(';').zip(group).all(|(a, b)| a == b) + } + + /// Every (group, value) pair that occurs, the frequency probes. + fn pairs(items: &[(String, i64)]) -> Vec<(Vec, i64)> { + let mut seen = HashSet::new(); + for g in groups(items) { + for (key, v) in items { + if in_group(key, &g) { + seen.insert((g.clone(), *v)); + } + } + } + let mut out: Vec<_> = seen.into_iter().collect(); + out.sort(); + out + } + + fn answers(mut passes: Vec>) -> Vec { + (passes.pop().expect("one pass"))().0 + } + + // ---------- merge exactness (hydra doc §2.3) ---------- + + const SHARDS: [usize; 2] = [4, 7]; + + /// Asks `run` of the stream folded from every shard count, under both + /// splits, and of the single stream: every answer must be equal, because + /// these cells merge exactly and every cell clones one seeded template. + fn merged_equals_single( + items: Vec<(String, i64)>, + run: impl Fn(Rc>, usize) -> Vec, + ) { + let single = run(Rc::new(items.clone()), 1); + for k in SHARDS { + for (split, stream) in [ + ("contiguous", items.clone()), + ("interleaved", interleave(&items, k)), + ] { + assert_eq!( + run(Rc::new(stream), k), + single, + "{k} {split} shards answer differently from one stream" + ); + } + } + } + + /// A small grid, so groups collide and the merge has contamination to + /// carry too. + const SMALL_W: usize = 16; + + #[test] + fn hydra_cms_merged_from_shards_answers_as_one_stream() { + let items = records(3_000); + let probes = Rc::new(pairs(&items)); + let params = ParamSet::of(&HydraCmsParams { + rows: 3, + cols: SMALL_W, + cell_rows: 3, + cell_cols: 64, + }); + merged_equals_single(items, |stream, k| { + answers(merge_query_hydra_cms(¶ms, stream, probes.clone(), k, 1).unwrap()) + }); + } + + #[test] + fn hydra_cs_merged_from_shards_answers_as_one_stream() { + let items = records(3_000); + let probes = Rc::new(pairs(&items)); + let params = ParamSet::of(&HydraCsParams { + rows: 3, + cols: SMALL_W, + cell_rows: 3, + cell_cols: 64, + }); + merged_equals_single(items, |stream, k| { + answers(merge_query_hydra_cs(¶ms, stream, probes.clone(), k, 1).unwrap()) + }); + } + + #[test] + fn hydra_hll_merged_from_shards_answers_as_one_stream() { + let items = records(3_000); + let probes = Rc::new(groups(&items)); + let params = ParamSet::of(&HydraHllParams { + rows: 3, + cols: SMALL_W, + }); + merged_equals_single(items, |stream, k| { + answers(merge_query_hydra_hll(¶ms, stream, probes.clone(), k, 1).unwrap()) + }); + } + + // ---------- the §2.2 bounds, on a fixed stream ---------- + // + // Every scored group must land inside its bound in a good row. `BETA`, the + // Markov factor on the colliders' mass, is generous: the bounds are + // sanity checks on the grid, not a measurement of its confidence. + + const N: usize = 20_000; + const BETA: f64 = 8.0; + /// Subsets every two-label record is inserted under. + const FAN_OUT: f64 = 3.0; + const CELL_COLS: usize = 512; + + /// Each value's fanned-out frequency `F_v`: its count over all subkeys. + fn fanned(items: &[(String, i64)]) -> HashMap { + let mut f = HashMap::new(); + for (_, v) in items { + *f.entry(*v).or_insert(0.0) += FAN_OUT; + } + f + } + + /// Group `g`'s value frequencies `f_q(v)`. + fn frequencies(items: &[(String, i64)], g: &[String]) -> HashMap { + let mut f = HashMap::new(); + for (_, v) in items.iter().filter(|(key, _)| in_group(key, g)) { + *f.entry(*v).or_insert(0.0) += 1.0; + } + f + } + + #[test] + fn hydra_cms_never_under_and_within_its_bound() { + let items = records(N); + let probes = pairs(&items); + let params = ParamSet::of(&HydraCmsParams { + rows: 3, + cols: SMALL_W, + cell_rows: 3, + cell_cols: CELL_COLS, + }); + let got = answers( + merge_query_hydra_cms( + ¶ms, + Rc::new(items.clone()), + Rc::new(probes.clone()), + 1, + 1, + ) + .unwrap(), + ); + let (fv, m, w) = (fanned(&items), FAN_OUT * N as f64, SMALL_W as f64); + let eps = std::f64::consts::E / CELL_COLS as f64; + let mut by_group = HashMap::new(); + for ((g, v), est) in probes.iter().zip(got) { + let f = by_group + .entry(g.clone()) + .or_insert_with(|| frequencies(&items, g)); + let (truth, n_q) = (f[v], f.values().sum::()); + assert!(est >= truth, "{g:?} {v}: {est} under {truth}"); + let bound = truth + eps * n_q + BETA * (fv[v] + eps * m) / w; + assert!(est <= bound, "{g:?} {v}: {est} over its bound {bound}"); + } + } + + #[test] + fn hydra_cs_within_its_bound() { + let items = records(N); + let probes = pairs(&items); + let params = ParamSet::of(&HydraCsParams { + rows: 3, + cols: SMALL_W, + cell_rows: 3, + cell_cols: CELL_COLS, + }); + let got = answers( + merge_query_hydra_cs( + ¶ms, + Rc::new(items.clone()), + Rc::new(probes.clone()), + 1, + 1, + ) + .unwrap(), + ); + let (fv, w) = (fanned(&items), SMALL_W as f64); + let f2: f64 = fv.values().map(|f| f * f).sum(); + let eps = 1.0 / (CELL_COLS as f64).sqrt(); + // The colliders' L2 under the one Markov event `F2_C <= BETA * F2 / W`. + let l2_c = (BETA * f2 / w).sqrt(); + let mut by_group = HashMap::new(); + for ((g, v), est) in probes.iter().zip(got) { + let f = by_group + .entry(g.clone()) + .or_insert_with(|| frequencies(&items, g)); + let truth = f[v]; + let l2_q = f.values().map(|x| x * x).sum::().sqrt(); + let (low, high) = ( + truth - eps * (l2_q + l2_c), + truth + eps * l2_q + BETA * fv[v] / w + eps * l2_c, + ); + assert!( + (low..=high).contains(&est), + "{g:?} {v}: {est} outside [{low}, {high}]" + ); + } + } + + #[test] + fn hydra_hll_within_its_bound() { + let items = records(N); + let probes = groups(&items); + let params = ParamSet::of(&HydraHllParams { + rows: 3, + cols: SMALL_W, + }); + let got = answers( + merge_query_hydra_hll( + ¶ms, + Rc::new(items.clone()), + Rc::new(probes.clone()), + 1, + 1, + ) + .unwrap(), + ); + let distinct = |g: &[String]| frequencies(&items, g).len() as f64; + // `ΣD`: every subkey's distinct count, the first label, the second and + // the pair. + let mut by_subkey: HashMap> = HashMap::new(); + for (key, v) in &items { + let (region, service) = key.split_once(';').unwrap(); + for subkey in [format!("{region};"), format!(";{service}"), key.clone()] { + by_subkey.entry(subkey).or_default().insert(*v); + } + } + let sum_d: f64 = by_subkey.values().map(|s| s.len() as f64).sum(); + // Three standard errors of a 2^14-register HLL. + let eps = 3.0 * 1.04 / ((1u64 << 14) as f64).sqrt(); + for (g, est) in probes.iter().zip(got) { + let d_q = distinct(g); + let (low, high) = ( + (1.0 - eps) * d_q, + (1.0 + eps) * (d_q + BETA * sum_d / SMALL_W as f64), + ); + assert!( + (low..=high).contains(&est), + "{g:?}: {est} outside [{low}, {high}]" + ); + } + } +} diff --git a/sketch-bench/src/wrappers/mod.rs b/sketch-bench/src/wrappers/mod.rs index 39006635..343730ec 100644 --- a/sketch-bench/src/wrappers/mod.rs +++ b/sketch-bench/src/wrappers/mod.rs @@ -148,6 +148,36 @@ pub fn partition(items: &[T], n: usize) -> Vec<&[T]> { items.chunks(chunk).collect() } +/// How a merge run splits the stream into its shards. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum MergeSplit { + /// Each shard a stretch of the stream, as [`partition`] cuts it. + #[default] + Contiguous, + /// Round-robin by record, so each shard is a sample of the whole stream: + /// the stream is first reordered by [`interleave`]. + Interleaved, +} + +/// The stream reordered so that [`partition`] into `n` hands out a round-robin +/// split: record `j` goes to the next shard with room, so each shard holds +/// every `n`-th record, in stream order, and the shard sizes are exactly the +/// contiguous split's. +pub fn interleave(items: &[T], n: usize) -> Vec { + let sizes: Vec = partition(items, n).iter().map(|p| p.len()).collect(); + let mut shards: Vec> = sizes.iter().map(|&len| Vec::with_capacity(len)).collect(); + let mut next = 0; + for item in items { + // Only the last shard can be short, so this skips at most once a round. + while shards[next].len() == sizes[next] { + next = (next + 1) % sizes.len(); + } + shards[next].push(item.clone()); + next = (next + 1) % sizes.len(); + } + shards.concat() +} + // All four helpers below exist for one rule: a row that cannot build at the // requested parameters says so, and never builds at different ones under the // requested label — so `sketch_config` is always the config that ran. @@ -232,3 +262,33 @@ pub fn hh_heap_footprint(capacity: usize) -> usize { std::mem::size_of::(), ) } + +#[cfg(test)] +mod tests { + use super::*; + + /// Cut by [`partition`], the interleaved stream deals record `j` to shard + /// `j mod n` while every shard has room, and keeps the contiguous split's + /// sizes, including its short last shard. + #[test] + fn interleave_deals_round_robin_at_the_contiguous_sizes() { + let items: Vec = (0..10).collect(); + let shards: Vec> = partition(&interleave(&items, 4), 4) + .iter() + .map(|s| s.to_vec()) + .collect(); + // Contiguous: chunks of 3, so sizes 3, 3, 3, 1; record 7 skips the + // full last shard. + assert_eq!( + shards, + vec![vec![0, 4, 7], vec![1, 5, 8], vec![2, 6, 9], vec![3]] + ); + // Divisible: every shard is one residue class. + let shards = partition(&interleave(&items, 5), 5) + .iter() + .map(|s| s.to_vec()) + .collect::>(); + assert_eq!(shards, (0..5).map(|r| vec![r, r + 5]).collect::>()); + assert!(interleave::(&[], 4).is_empty()); + } +} From 9a6a5a520032bdc2fcebc22375facb74d7814dec Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 10 Oct 2026 04:11:33 +0000 Subject: [PATCH 04/18] datagen: child_of columns for hierarchical labels A `child_of: ` column with `fan_out: f` nests under an earlier string column: its value is the parent row's value, a `.`, and a child index in [0, f) drawn from the column's own distribution. Bad indices, a fan_out that disagrees with the domain, and fields that would not mean what they say on a child are refused before drawing. Adds configs/datagen/hydra_hier.yaml (region 4 -> service 25 -> endpoint 25, status 4, user_id value) for the hydra rows, docs, and tests. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FhcWJExZcmVqS6r6rEtjis --- aqpbm-cli/src/external.rs | 2 + aqpbm-cli/src/flatten_record.rs | 2 + aqpbm-cli/src/main.rs | 2 + aqpbm-cli/src/repeat.rs | 2 + aqpbm-core/schema/merged_record.schema.json | 18 +++ aqpbm-core/src/atomic_costs.rs | 2 + aqpbm-core/src/benchmark_result/schema.rs | 2 + aqpbm-core/src/erp.rs | 2 + aqpbm-datagen/src/column.rs | 91 +++++++++++++ aqpbm-datagen/src/lib.rs | 144 ++++++++++++++++++++ aqpbm-datagen/src/table.rs | 29 +++- configs/datagen/hydra_hier.yaml | 74 ++++++++++ docs/aqpbm-datagen.md | 27 ++++ 13 files changed, 395 insertions(+), 2 deletions(-) create mode 100644 configs/datagen/hydra_hier.yaml diff --git a/aqpbm-cli/src/external.rs b/aqpbm-cli/src/external.rs index 50eb22b1..3f65ca8d 100644 --- a/aqpbm-cli/src/external.rs +++ b/aqpbm-cli/src/external.rs @@ -269,6 +269,8 @@ fn shape_column(data_type: &str) -> ColumnSpec { special_rule: RULE_NONE, data_type: data_type.to_string(), string: None, + child_of: None, + fan_out: None, } } diff --git a/aqpbm-cli/src/flatten_record.rs b/aqpbm-cli/src/flatten_record.rs index c69b1004..00076d05 100644 --- a/aqpbm-cli/src/flatten_record.rs +++ b/aqpbm-cli/src/flatten_record.rs @@ -249,6 +249,8 @@ mod tests { special_rule: aqpbm_datagen::RULE_NONE, data_type: "i64".into(), string: None, + child_of: None, + fan_out: None, }, 1000, ); diff --git a/aqpbm-cli/src/main.rs b/aqpbm-cli/src/main.rs index 8334d4dd..c6315d19 100644 --- a/aqpbm-cli/src/main.rs +++ b/aqpbm-cli/src/main.rs @@ -258,6 +258,8 @@ fn dataset_spec(args: &SketchbenchArgs) -> Result { }; (opts != StringOpts::default()).then_some(opts) }, + child_of: None, + fan_out: None, }, args.size.expect("clap requires --size") as u64, ))) diff --git a/aqpbm-cli/src/repeat.rs b/aqpbm-cli/src/repeat.rs index 19ae79e9..4ebcec1f 100644 --- a/aqpbm-cli/src/repeat.rs +++ b/aqpbm-cli/src/repeat.rs @@ -191,6 +191,8 @@ mod tests { special_rule: aqpbm_datagen::RULE_NONE, data_type: "i64".into(), string: None, + child_of: None, + fan_out: None, }, 1000, ); diff --git a/aqpbm-core/schema/merged_record.schema.json b/aqpbm-core/schema/merged_record.schema.json index 4b858a32..dd09e6e3 100644 --- a/aqpbm-core/schema/merged_record.schema.json +++ b/aqpbm-core/schema/merged_record.schema.json @@ -393,6 +393,15 @@ "format": "uint64", "minimum": 0 }, + "child_of": { + "description": "Index of an earlier `string` column this one nests under, for\nhierarchical labels. A value is the parent row's value, a `.`, and a\nchild index in `[0, fan_out)` drawn from this column's distribution.", + "type": [ + "integer", + "null" + ], + "format": "uint", + "minimum": 0 + }, "data_type": { "description": "Which [`ColumnData`] variant this column renders into.", "type": "string" @@ -400,6 +409,15 @@ "distribution": { "$ref": "#/$defs/DataDistribution" }, + "fan_out": { + "description": "How many children each parent value has. Required with `child_of`, and\nchecked against the distribution's domain like `cardinality` is.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, "shift": { "description": "Added to every value, for a column that should not start where its\ndistribution naturally does. Under the monotonic rule it is the value the\nseries starts from.", "type": [ diff --git a/aqpbm-core/src/atomic_costs.rs b/aqpbm-core/src/atomic_costs.rs index fa5c8ee7..449b0d2f 100644 --- a/aqpbm-core/src/atomic_costs.rs +++ b/aqpbm-core/src/atomic_costs.rs @@ -418,6 +418,8 @@ mod tests { special_rule: aqpbm_datagen::RULE_NONE, data_type: "i64".into(), string: None, + child_of: None, + fan_out: None, }, 1_000_000, ) diff --git a/aqpbm-core/src/benchmark_result/schema.rs b/aqpbm-core/src/benchmark_result/schema.rs index 6b644160..84472aab 100644 --- a/aqpbm-core/src/benchmark_result/schema.rs +++ b/aqpbm-core/src/benchmark_result/schema.rs @@ -490,6 +490,8 @@ mod tests { special_rule: aqpbm_datagen::RULE_NONE, data_type: "i64".into(), string: None, + child_of: None, + fan_out: None, }, 1_000_000, ) diff --git a/aqpbm-core/src/erp.rs b/aqpbm-core/src/erp.rs index 6a4e42c8..9bddc906 100644 --- a/aqpbm-core/src/erp.rs +++ b/aqpbm-core/src/erp.rs @@ -181,6 +181,8 @@ mod tests { special_rule: 0, data_type: "u64".into(), string: None, + child_of: None, + fan_out: None, }, 100_000, ), diff --git a/aqpbm-datagen/src/column.rs b/aqpbm-datagen/src/column.rs index a68168c9..07e8899d 100644 --- a/aqpbm-datagen/src/column.rs +++ b/aqpbm-datagen/src/column.rs @@ -36,6 +36,15 @@ pub struct ColumnSpec { /// Rendering options for `data_type: string`. Absent means the defaults. #[serde(default, skip_serializing_if = "Option::is_none")] pub string: Option, + /// Index of an earlier `string` column this one nests under, for + /// hierarchical labels. A value is the parent row's value, a `.`, and a + /// child index in `[0, fan_out)` drawn from this column's distribution. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub child_of: Option, + /// How many children each parent value has. Required with `child_of`, and + /// checked against the distribution's domain like `cardinality` is. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fan_out: Option, } impl ColumnSpec { @@ -53,6 +62,21 @@ impl ColumnSpec { ))); } + match (self.child_of, self.fan_out) { + (None, None) => {} + (Some(_), None) => { + return Err(DataGenError::BadParam( + "child_of: needs fan_out, the number of children per parent value".into(), + )) + } + (None, Some(_)) => { + return Err(DataGenError::BadParam( + "fan_out: applies only to a child_of column".into(), + )) + } + (Some(_), Some(fan_out)) => self.validate_child(fan_out)?, + } + // `cardinality` restates the domain. Silently preferring one over the // other would let a report cite a key space the column never had. if let Some(claimed) = self.cardinality { @@ -98,6 +122,54 @@ impl ColumnSpec { Ok(()) } + /// The checks a `child_of` column adds. Its index of the parent is checked + /// by the table, which knows the other columns. + fn validate_child(&self, fan_out: u64) -> Result<(), DataGenError> { + if self.data_type != "string" { + return Err(DataGenError::BadParam(format!( + "child_of: a child value is text, so data_type has to be `string`, not '{}'", + self.data_type + ))); + } + if fan_out == 0 { + return Err(DataGenError::BadParam("fan_out must be > 0".into())); + } + match self.distribution.domain() { + Some(domain) if domain.size == fan_out => {} + Some(domain) => { + return Err(DataGenError::BadParam(format!( + "fan_out {fan_out} disagrees with the {} domain, which holds {}; the \ + distribution draws the child index", + self.distribution.tag(), + domain.size, + ))) + } + None => { + return Err(DataGenError::BadParam(format!( + "child_of: the child index needs a bounded domain, and {} has none", + self.distribution.tag(), + ))) + } + } + // A child column holds parent cardinality × fan_out values and is not + // rendered from a rank, so neither field would mean what it says. + if self.cardinality.is_some() { + return Err(DataGenError::BadParam( + "child_of: cardinality would restate fan_out, not the column's distinct \ + count (parent cardinality × fan_out); drop it" + .into(), + )); + } + if self.string.is_some() { + return Err(DataGenError::BadParam( + "child_of: a child value is its parent's plus an index, so a `string:` \ + block has nothing to render; drop it" + .into(), + )); + } + Ok(()) + } + /// The raw draw stream: `row_num` values straight from the distribution, /// before any rule, shift or rendering. Split out because a /// `column_connected` group draws this once and every member column renders @@ -163,4 +235,23 @@ impl ColumnSpec { ))), } } + + /// Render a `child_of` column: each row's parent value, a `.`, and the + /// draw's 0-based rank in the domain as the child index. + pub(crate) fn render_child( + &self, + raw: &[f64], + parents: &[String], + ) -> Result { + let domain = self + .distribution + .domain() + .expect("child domains were validated above"); + Ok(ColumnData::String( + raw.iter() + .zip(parents) + .map(|(draw, parent)| format!("{parent}.{}", (draw - domain.lower) as u64)) + .collect(), + )) + } } diff --git a/aqpbm-datagen/src/lib.rs b/aqpbm-datagen/src/lib.rs index 867131a3..f5e2679d 100644 --- a/aqpbm-datagen/src/lib.rs +++ b/aqpbm-datagen/src/lib.rs @@ -61,6 +61,8 @@ mod tests { special_rule: RULE_NONE, data_type: data_type.into(), string: None, + child_of: None, + fan_out: None, } } @@ -502,6 +504,148 @@ mod tests { assert!(err.contains("used twice"), "{err}"); } + // ---------- child_of ---------- + + fn child(parent: usize, fan_out: u64, distribution: DataDistribution) -> ColumnSpec { + ColumnSpec { + child_of: Some(parent), + fan_out: Some(fan_out), + ..column(distribution, "string") + } + } + + /// region (4) → service (fan-out 25, Zipf) → endpoint (fan-out 5), then a value. + fn hier_table() -> TableDescription { + TableDescription { + column_num: 4, + column_label: vec![ + "region".into(), + "service".into(), + "endpoint".into(), + "value".into(), + ], + column_spec: vec![ + column(uniform(0.0, 4.0, 1), "string"), + child(0, 25, zipf(25, 1.2, 2)), + child(1, 5, uniform(0.0, 5.0, 3)), + column(zipf(1000, 1.1, 4), "i64"), + ], + column_connected: Vec::new(), + row_num: 20_000, + } + } + + /// Each child splits into its parent, a `.`, and an index below `fan_out`; + /// returns the distinct children seen under each parent. + fn children_by_parent( + parents: &[String], + children: &[String], + fan_out: u64, + ) -> std::collections::HashMap> { + let mut by_parent: std::collections::HashMap<_, std::collections::HashSet<_>> = + Default::default(); + for (parent, child) in parents.iter().zip(children) { + let (head, index) = child.rsplit_once('.').expect("a child has a `.`"); + assert_eq!(head, parent, "'{child}' does not start with its parent"); + assert!(index.parse::().unwrap() < fan_out, "'{child}'"); + by_parent + .entry(parent.clone()) + .or_default() + .insert(child.clone()); + } + by_parent + } + + #[test] + fn a_child_table_is_deterministic() { + let d = hier_table(); + assert_eq!(d.generate().unwrap(), d.generate().unwrap()); + } + + #[test] + fn every_child_extends_its_parent_within_its_fan_out() { + let t = hier_table().generate().unwrap(); + let region = t.data[0].as_string().unwrap(); + let service = t.data[1].as_string().unwrap(); + let endpoint = t.data[2].as_string().unwrap(); + + let services = children_by_parent(region, service, 25); + assert_eq!(services.len(), 4); + // 20k draws over 25 Zipf(1.2) ranks reach every service under every region. + assert!(services.values().all(|s| s.len() == 25), "{services:?}"); + + let endpoints = children_by_parent(service, endpoint, 5); + assert_eq!(endpoints.len(), 100); + assert!(endpoints.values().all(|e| e.len() <= 5)); + } + + /// The child index is the column's own draw, ranked: the distribution is + /// what picks it, and the parent plays no part. + #[test] + fn the_child_index_follows_the_columns_distribution() { + let d = hier_table(); + let draws = d.column_spec[1].draw(d.row_num as usize).unwrap(); + let t = d.generate().unwrap(); + let service = t.data[1].as_string().unwrap(); + let mut counts = [0u64; 25]; + for (draw, s) in draws.iter().zip(service) { + let index: u64 = s.rsplit_once('.').unwrap().1.parse().unwrap(); + assert_eq!(index, (draw - 1.0) as u64); + counts[index as usize] += 1; + } + // Zipf(1.2) over 25 ranks gives rank 1 about 30% of the mass. + let head = counts[0] as f64 / d.row_num as f64; + assert!((0.25..0.35).contains(&head), "{head}"); + assert!(counts[0] > counts[1] && counts[1] > counts[24]); + } + + #[test] + fn hydra_hier_yaml_generates_its_hierarchy() { + let d = + TableDescription::from_path(std::path::Path::new("../configs/datagen/hydra_hier.yaml")) + .unwrap(); + let t = d.generate().unwrap(); + let col = |i: usize| t.data[i].as_string().unwrap(); + assert_eq!(children_by_parent(col(0), col(1), 25).len(), 4); + assert_eq!(children_by_parent(col(1), col(2), 25).len(), 100); + } + + #[test] + fn bad_child_columns_are_refused_by_name() { + type Edit = fn(&mut TableDescription); + let cases: Vec<(&str, Edit)> = vec![ + ("earlier column", |d| d.column_spec[1].child_of = Some(1)), + ("earlier column", |d| d.column_spec[1].child_of = Some(9)), + ("not a string column", |d| { + d.column_spec[0].data_type = "i64".into() + }), + ("needs fan_out", |d| d.column_spec[1].fan_out = None), + ("only to a child_of", |d| d.column_spec[1].child_of = None), + ("fan_out must be > 0", |d| { + d.column_spec[1].fan_out = Some(0) + }), + ("fan_out 24 disagrees", |d| { + d.column_spec[1].fan_out = Some(24) + }), + ("bounded domain", |d| { + d.column_spec[1].distribution = normal(0.0, 1.0, 2) + }), + ("has to be `string`", |d| { + d.column_spec[1].data_type = "u64".into() + }), + ("cardinality", |d| d.column_spec[1].cardinality = Some(25)), + ("`string:` block", |d| { + d.column_spec[1].string = Some(StringOpts::default()) + }), + ]; + for (want, edit) in cases { + let mut d = hier_table(); + edit(&mut d); + let err = d.generate().unwrap_err().to_string(); + assert!(err.contains(want), "wanted '{want}' in: {err}"); + } + } + // ---------- serde ---------- #[test] diff --git a/aqpbm-datagen/src/table.rs b/aqpbm-datagen/src/table.rs index 9b336c41..cb64c73d 100644 --- a/aqpbm-datagen/src/table.rs +++ b/aqpbm-datagen/src/table.rs @@ -119,6 +119,27 @@ impl TableDescription { })?; } + // A child is generated after its parent, which an earlier index + // guarantees and which also rules out a column nesting under itself. + for (i, spec) in self.column_spec.iter().enumerate() { + let Some(parent) = spec.child_of else { + continue; + }; + let label = &self.column_label[i]; + if parent >= i { + return Err(DataGenError::BadParam(format!( + "column '{label}': child_of {parent} has to name an earlier column, \ + one of 0..{i}" + ))); + } + if self.column_spec[parent].data_type != "string" { + return Err(DataGenError::BadParam(format!( + "column '{label}': child_of names '{}', which is not a string column", + self.column_label[parent] + ))); + } + } + let mut claimed: HashMap<&str, usize> = HashMap::new(); for (g, group) in self.column_connected.iter().enumerate() { if group.len() < 2 { @@ -217,7 +238,7 @@ impl TableDescription { group_bits.insert(g, bits); } - let mut data = Vec::with_capacity(self.column_spec.len()); + let mut data: Vec = Vec::with_capacity(self.column_spec.len()); for (i, spec) in self.column_spec.iter().enumerate() { // Only one of these buffers is initialized. Either one lives just // long enough for this column's render pass. @@ -244,7 +265,11 @@ impl TableDescription { &own_draw } }; - data.push(spec.render(raw).map_err(|e| { + let rendered = match spec.child_of { + Some(parent) => spec.render_child(raw, data[parent].as_string()?), + None => spec.render(raw), + }; + data.push(rendered.map_err(|e| { DataGenError::BadParam(format!("column '{}': {e}", self.column_label[i])) })?); } diff --git a/configs/datagen/hydra_hier.yaml b/configs/datagen/hydra_hier.yaml new file mode 100644 index 00000000..968e6ecd --- /dev/null +++ b/configs/datagen/hydra_hier.yaml @@ -0,0 +1,74 @@ +# Hierarchical labels for the record-ingesting rows (the `hydra-*` algorithms): +# region → service → endpoint, plus an independent status label, then the value +# column. As in `hydra_columns.yaml`, the last column is the value and the ones +# before it are labels. +# +# approxbench sketchbench --variant hydra-cms --library lib \ +# --spec configs/datagen/hydra_hier.yaml \ +# --config "rows=3 cols=1024 cell_rows=3 cell_cols=1024" \ +# --operations query --metrics accuracy +# +# A `child_of` column nests under an earlier label: its value is the parent row's +# value, a `.`, and a child index in `[0, fan_out)` drawn from its own +# distribution. So a service reads `.7` and an endpoint `.7.3`, +# and each parent value has at most `fan_out` children. The index is drawn +# independently of the parent, so every region shares the same service skew. +# +# Four labels cost `2^4 - 1 = 15` cell insertions per record. Distinct label +# values: region 4, service 4 × 25 = 100, endpoint 100 × 25 = 2500, status 4. + +column_num: 5 +column_label: [region, service, endpoint, status, user_id] +row_num: 200000 + +column_spec: + # region: 4 values, equally likely. + - data_type: string + string: + min_len: 4 + max_len: 4 + distribution: + kind: uniform + lower_bound: 0.0 + upper_bound: 4.0 + seed: 1 + + # service: 25 per region, skewed, so a few services carry most traffic. + - data_type: string + child_of: 0 + fan_out: 25 + distribution: + kind: zipf + skewness: 1.1 + population_size: 25 + seed: 2 + + # endpoint: 25 per service, skewed. + - data_type: string + child_of: 1 + fan_out: 25 + distribution: + kind: zipf + skewness: 1.2 + population_size: 25 + seed: 3 + + # status: 4 values, heavily skewed toward the first (the "200"). + - data_type: string + string: + min_len: 3 + max_len: 3 + distribution: + kind: zipf + skewness: 2.0 + population_size: 4 + seed: 4 + + # user_id: the value. Count-Min counts its occurrences per group, + # HyperLogLog counts how many are distinct. + - data_type: i64 + distribution: + kind: zipf + skewness: 0.8 + population_size: 1000000 + seed: 5 diff --git a/docs/aqpbm-datagen.md b/docs/aqpbm-datagen.md index fcccfa78..3ff1f839 100644 --- a/docs/aqpbm-datagen.md +++ b/docs/aqpbm-datagen.md @@ -35,6 +35,8 @@ pub struct ColumnSpec { pub special_rule: u32, // bit mask about special_rule; 0 means no special rules pub data_type: String, // defines which case of enum ColumnData should be chosen from pub string: Option, // if this column is about string data, description about how string looks like + pub child_of: Option, // index of an earlier string column this one nests under; see child_of below + pub fan_out: Option, // children per parent value; required with child_of } enum DataDistribution { @@ -83,6 +85,31 @@ Thus, the field `column_connected` is used to describe such relation. The requirement is: columns related to each other needs to have identical distribution and distribution parameter. They will be generated once, and processed to meet the need. +### child_of + +Labels are often hierarchical: a service lives in one region, an endpoint in one service. +A column with `child_of: ` and `fan_out: f` nests under the earlier column at that index: +its value is the parent row's value, a `.`, and a child index in `[0, f)`. +The child index is drawn from the column's own `distribution` (its 0-based rank in the domain), so the usual skew and seed options pick it, +independently of the parent: every parent value shares one child distribution and has at most `f` distinct children. + +```yaml +- data_type: string + child_of: 0 # nests under column 0, e.g. "abcd" -> "abcd.7" + fan_out: 25 + distribution: {kind: zipf, skewness: 1.1, population_size: 25, seed: 2} +``` + +Each of the following is a description error: + +- `child_of` without `fan_out`, or `fan_out` without `child_of` +- `child_of` naming the column itself, a later column, or no column (a parent is generated first) +- a parent that is not `data_type: string`, or a child that is not +- `fan_out` of 0, or one that disagrees with the distribution's domain size (an unbounded distribution has none) +- `cardinality` or a `string:` block on a child column (its distinct count is the parent's times `fan_out`, and it is not rendered from a rank) + +`configs/datagen/hydra_hier.yaml` uses it for a region → service → endpoint hierarchy. + ## Output The output is a struct containing data: From 62ae3f02efb743a986b1df700f768aeb7b5efc33 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 10 Oct 2026 04:13:38 +0000 Subject: [PATCH 05/18] hydra: score any target grouping; per-group error output Every hydra-* row (lib and polars, all five variants and hydra-univmon's four statistics) takes `--group-columns J,K,...` (default `0`, today's behavior) in place of the hardcoded SCORED_LABEL_COLUMN. A subpopulation probe is now positioned (`Vec>`, Some at the grouped columns), so `hydra_shared::query` asks any non-empty subset of the key columns and the polars baselines look up `subset_key` with that subset's mask (`group_key`, generalizing `prefix_key`). Each grouped comparator reports its per-group error in its own metric (relative error; mean rank error for hydra-kll) and summarises it as err_mean, err_p50, err_p90, err_max, groups_scored, plus schema_width, records and fanned_mass. `--per-group-out PATH` writes the rows as `group_key,n_q,error`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FhcWJExZcmVqS6r6rEtjis --- README.md | 6 + aqpbm-cli/src/cli.rs | 17 +++ aqpbm-cli/src/main.rs | 15 ++ aqpbm-cli/src/rows/hydra_univmon.rs | 12 +- aqpbm-cli/src/rows/mod.rs | 9 +- aqpbm-cli/src/rows/statistic.rs | 85 +++++++++-- aqpbm-cli/tests/hydra_grouping.rs | 137 ++++++++++++++++++ aqpbm-core/src/accuracy/mod.rs | 37 ++++- .../src/accuracy/subpopulation/cardinality.rs | 91 ++++++++---- .../src/accuracy/subpopulation/entropy.rs | 27 +++- .../src/accuracy/subpopulation/frequency.rs | 113 ++++++++++----- .../subpopulation/frequency_vector.rs | 77 +++++++--- .../src/accuracy/subpopulation/l1_norm.rs | 86 ++++++++++- .../src/accuracy/subpopulation/l2_norm.rs | 27 +++- aqpbm-core/src/accuracy/subpopulation/mod.rs | 82 ++++++++++- .../src/accuracy/subpopulation/rank_error.rs | 112 +++++++++----- configs/datagen/hydra_columns.yaml | 3 +- configs/datagen/hydra_columns_f64.yaml | 3 +- configs/datagen/hydra_columns_string.yaml | 3 +- configs/datagen/hydra_columns_u64.yaml | 3 +- .../datagen/hydra_kll_disjoint_labels.yaml | 3 +- configs/datagen/hydra_univmon_columns.yaml | 3 +- sketch-bench/src/request.rs | 4 + sketch-bench/src/wrappers/hydra_cms/mod.rs | 19 ++- sketch-bench/src/wrappers/hydra_cms/polars.rs | 4 +- .../src/wrappers/hydra_cms/sketchlib.rs | 10 +- sketch-bench/src/wrappers/hydra_cs/mod.rs | 15 +- sketch-bench/src/wrappers/hydra_cs/polars.rs | 4 +- .../src/wrappers/hydra_cs/sketchlib.rs | 10 +- sketch-bench/src/wrappers/hydra_hll/mod.rs | 6 +- sketch-bench/src/wrappers/hydra_hll/polars.rs | 4 +- .../src/wrappers/hydra_hll/sketchlib.rs | 6 +- sketch-bench/src/wrappers/hydra_kll/mod.rs | 6 +- sketch-bench/src/wrappers/hydra_kll/polars.rs | 6 +- .../src/wrappers/hydra_kll/sketchlib.rs | 6 +- sketch-bench/src/wrappers/hydra_shared.rs | 17 ++- .../src/wrappers/hydra_univmon/mod.rs | 19 ++- .../src/wrappers/hydra_univmon/polars.rs | 14 +- .../src/wrappers/hydra_univmon/sketchlib.rs | 28 ++-- sketch-bench/src/wrappers/polars_shared.rs | 38 +++-- 40 files changed, 885 insertions(+), 282 deletions(-) create mode 100644 aqpbm-cli/tests/hydra_grouping.rs diff --git a/README.md b/README.md index f48e88d6..06118710 100644 --- a/README.md +++ b/README.md @@ -449,6 +449,12 @@ The file can be found [under configs directory](./configs/datagen/hydra_columns. **`--flat`**: flatten record such that output data of multiple experiments is flatten into one json file. +**`--group-columns`** (Hydra rows only): which label columns the subpopulation is taken over, as comma-separated indices into the columns before the value column, e.g. `1` or `0,1`; default `0`. +Any non-empty subset works, since the grid stores every subset of a record's labels; it picks the one the `hydra-*` row (`lib` and `polars`) asks and scores. +With `--metrics accuracy`, the record also summarises each scored group's error in the row's own metric (relative error; mean rank error for `hydra-kll`): `err_mean`, `err_p50`, `err_p90`, `err_max`, `groups_scored`, plus `schema_width` (`d`), `records` (`N`) and `fanned_mass` (`N·(2^d − 1)`). + +**`--per-group-out PATH`** (Hydra rows only): write those per-group errors as a CSV, `group_key,n_q,error`, one row per scored group (`n_q` is its record count, `group_key` its tagged labels such as `label0:a;label1:x`). + #### General Explanation Most of the input and output is shared between this hydra example and the previous hll example. diff --git a/aqpbm-cli/src/cli.rs b/aqpbm-cli/src/cli.rs index bd14e42a..1667a45c 100644 --- a/aqpbm-cli/src/cli.rs +++ b/aqpbm-cli/src/cli.rs @@ -234,10 +234,27 @@ pub struct SketchbenchArgs { /// merge exactly, so a gap is a defect; for KLL it is the result. #[arg(long, default_value_t = 2, help_heading = "Measurement content")] pub merge_shards: usize, + /// Which label columns the `hydra-*` rows group by, comma-separated + /// indices into the columns before the value column: any non-empty subset + /// (`0`, `1`, `0,1`). The grid stores every subset, so this picks the one + /// asked and scored. Every other target ignores it. + #[arg( + long, + value_delimiter = ',', + default_value = "0", + help_heading = "Measurement content" + )] + pub group_columns: Vec, /// Path to append JSONL records to. `-` or omitted sends them to stdout. #[arg(long, help_heading = "Output")] pub report: Option, + /// Write a `hydra-*` row's per-group errors to this CSV, one row per scored + /// group: `group_key,n_q,error` (`n_q` is the group's record count, `error` + /// the row's own metric). The record's `err_*` fields summarise the same + /// rows. Takes one accuracy measurement: query or merge, not both. + #[arg(long, help_heading = "Output")] + pub per_group_out: Option, /// Write one line for the whole invocation instead of one per measurement, /// folding its records into a single flattened row: one slot per operation, /// one field per metric. One record per measurement is the default; this is the diff --git a/aqpbm-cli/src/main.rs b/aqpbm-cli/src/main.rs index 8334d4dd..105035df 100644 --- a/aqpbm-cli/src/main.rs +++ b/aqpbm-cli/src/main.rs @@ -350,6 +350,19 @@ fn run_sketchbench(args: SketchbenchArgs) -> Result<()> { None => ParamSet::empty(&variant), }; + let mut group_columns = args.group_columns.clone(); + group_columns.sort_unstable(); + group_columns.dedup(); + if args.per_group_out.is_some() { + if !variant.starts_with("hydra-") { + bail!("--per-group-out: only the hydra-* rows score groups, and {variant} is not one"); + } + if want.iter().filter(|(_, m)| *m == Metric::Accuracy).count() > 1 { + bail!( + "--per-group-out writes one accuracy measurement; ask for query or merge, not both" + ); + } + } let req = Requirement { variant: variant.clone(), library: library.clone(), @@ -357,6 +370,8 @@ fn run_sketchbench(args: SketchbenchArgs) -> Result<()> { width, workers: args.workers.max(1), merge_shards: args.merge_shards, + group_columns, + per_group_out: args.per_group_out.as_ref().map(std::path::PathBuf::from), comparator: args.comparator.clone(), runs: args.runs, warmup_runs: args.warmup_runs, diff --git a/aqpbm-cli/src/rows/hydra_univmon.rs b/aqpbm-cli/src/rows/hydra_univmon.rs index eca3acd0..34320aa9 100644 --- a/aqpbm-cli/src/rows/hydra_univmon.rs +++ b/aqpbm-cli/src/rows/hydra_univmon.rs @@ -77,7 +77,7 @@ pub(crate) fn row_hydra_univmon_l1_norm_lib( table, want, SubpopL1NormGT { - group_columns: vec![SCORED_LABEL_COLUMN], + group_columns: group_columns(req, description)?, value_column: value_column(description), }, hus::insert_hydra_univmon::<$t>, @@ -114,7 +114,7 @@ pub(crate) fn row_hydra_univmon_l1_norm_polars( table, want, SubpopL1NormGT { - group_columns: vec![SCORED_LABEL_COLUMN], + group_columns: group_columns(req, description)?, value_column: value_column(description), }, hup::insert_polars_subpop_l1_norm::<$t>, @@ -147,7 +147,7 @@ pub(crate) fn row_hydra_univmon_l2_norm_lib( table, want, SubpopL2NormGT { - group_columns: vec![SCORED_LABEL_COLUMN], + group_columns: group_columns(req, description)?, value_column: value_column(description), }, hus::insert_hydra_univmon::<$t>, @@ -184,7 +184,7 @@ pub(crate) fn row_hydra_univmon_l2_norm_polars( table, want, SubpopL2NormGT { - group_columns: vec![SCORED_LABEL_COLUMN], + group_columns: group_columns(req, description)?, value_column: value_column(description), }, hup::insert_polars_subpop_l2_norm::<$t>, @@ -217,7 +217,7 @@ pub(crate) fn row_hydra_univmon_entropy_lib( table, want, SubpopEntropyGT { - group_columns: vec![SCORED_LABEL_COLUMN], + group_columns: group_columns(req, description)?, value_column: value_column(description), }, hus::insert_hydra_univmon::<$t>, @@ -254,7 +254,7 @@ pub(crate) fn row_hydra_univmon_entropy_polars( table, want, SubpopEntropyGT { - group_columns: vec![SCORED_LABEL_COLUMN], + group_columns: group_columns(req, description)?, value_column: value_column(description), }, hup::insert_polars_subpop_entropy::<$t>, diff --git a/aqpbm-cli/src/rows/mod.rs b/aqpbm-cli/src/rows/mod.rs index e1f1c59c..66524a42 100644 --- a/aqpbm-cli/src/rows/mod.rs +++ b/aqpbm-cli/src/rows/mod.rs @@ -23,12 +23,12 @@ use aqpbm_core::accuracy::keyed::{ }; use aqpbm_core::accuracy::quantile::{RankErrorGT, RelativeValueErrorGT}; use aqpbm_core::accuracy::subpopulation::{ - SubpopCardinalityGT, SubpopEntropyGT, SubpopFrequencyGT, SubpopL1NormGT, SubpopL2NormGT, - SubpopRankErrorGT, + GroupKey, SubpopCardinalityGT, SubpopEntropyGT, SubpopFrequencyGT, SubpopL1NormGT, + SubpopL2NormGT, SubpopRankErrorGT, }; use aqpbm_core::accuracy::topk::TopkGT; use aqpbm_core::accuracy::Score; -use aqpbm_core::accuracy::{questions, CountedValue, GroundTruth}; +use aqpbm_core::accuracy::{questions, CountedValue, GroundTruth, PerGroup}; use aqpbm_core::error::RunError; use aqpbm_core::measure::{ record_calls, runs_for, Measurement, Pass as CorePass, Report, RunOutcome, MIN_MERGE_SHARDS, @@ -37,9 +37,6 @@ use aqpbm_core::metrics::{Metric, Operation}; use aqpbm_core::{ColumnItem, GeneratedTable, TableDescription}; use sketch_bench::wrappers::frequency_value::FrequencyValue; -/// The label column every subpopulation comparator scores over. -const SCORED_LABEL_COLUMN: usize = 0; - const KEYED_KEY_COLUMN: usize = 0; fn value_column(description: &TableDescription) -> usize { diff --git a/aqpbm-cli/src/rows/statistic.rs b/aqpbm-cli/src/rows/statistic.rs index 3e6c0678..87a227aa 100644 --- a/aqpbm-cli/src/rows/statistic.rs +++ b/aqpbm-cli/src/rows/statistic.rs @@ -109,6 +109,28 @@ pub(super) fn topk_row( ) } +/// The label columns a grouped row asks and scores: `--group-columns`, which +/// must name at least one column, and only columns before the value column, +/// since those are the grid's key columns. +pub(super) fn group_columns( + req: &Requirement, + description: &TableDescription, +) -> Result, RunError> { + let labels = value_column(description); + if req.group_columns.is_empty() { + return Err(RunError::Sketch( + "--group-columns names no column; a group is taken over at least one".into(), + )); + } + if let Some(&column) = req.group_columns.iter().find(|&&c| c >= labels) { + return Err(RunError::Sketch(format!( + "--group-columns {column}: the label columns are 0..{labels}, the ones \ + before the value column" + ))); + } + Ok(req.group_columns.clone()) +} + /// A row answering **subpopulation frequency**: how often a value occurs inside /// a group. #[allow(clippy::too_many_arguments, clippy::type_complexity)] @@ -119,8 +141,8 @@ pub(super) fn subpop_frequency_row( want: &[(Operation, Metric)], insert: InsertBody<(String, V)>, insert_step: InsertStepBody<(String, V)>, - query: QueryBody<(String, V), (Vec, V), f64>, - merge: Option, V), f64>>, + query: QueryBody<(String, V), (GroupKey, V), f64>, + merge: Option>, prepare: Option>, ) -> Result { scored_row( @@ -128,7 +150,10 @@ pub(super) fn subpop_frequency_row( description, table, want, - SubpopFrequencyGT::::over_columns(vec![SCORED_LABEL_COLUMN], value_column(description)), + SubpopFrequencyGT::::over_columns( + group_columns(req, description)?, + value_column(description), + ), peel_labeled::, insert, insert_step, @@ -148,8 +173,8 @@ pub(super) fn subpop_cardinality_row( want: &[(Operation, Metric)], insert: InsertBody<(String, V)>, insert_step: InsertStepBody<(String, V)>, - query: QueryBody<(String, V), Vec, f64>, - merge: Option, f64>>, + query: QueryBody<(String, V), GroupKey, f64>, + merge: Option>, prepare: Option>, ) -> Result { scored_row( @@ -158,7 +183,7 @@ pub(super) fn subpop_cardinality_row( table, want, SubpopCardinalityGT { - group_columns: vec![SCORED_LABEL_COLUMN], + group_columns: group_columns(req, description)?, value_column: value_column(description), }, peel_labeled::, @@ -181,12 +206,12 @@ pub(super) fn subpop_vector_row( ground_truth: G, insert: InsertBody<(String, V)>, insert_step: InsertStepBody<(String, V)>, - query: QueryBody<(String, V), Vec, f64>, - merge: Option, f64>>, + query: QueryBody<(String, V), GroupKey, f64>, + merge: Option>, prepare: Option>, ) -> Result where - G: GroundTruth, Answer = f64> + 'static, + G: GroundTruth + 'static, G::Truth: 'static, { scored_row( @@ -212,8 +237,8 @@ pub(super) fn subpop_quantile_row( want: &[(Operation, Metric)], insert: InsertBody<(String, V)>, insert_step: InsertStepBody<(String, V)>, - query: QueryBody<(String, V), (Vec, f64), f64>, - merge: Option, f64), f64>>, + query: QueryBody<(String, V), (GroupKey, f64), f64>, + merge: Option>, prepare: Option>, ) -> Result { scored_row( @@ -222,7 +247,7 @@ pub(super) fn subpop_quantile_row( table, want, SubpopRankErrorGT { - group_columns: vec![SCORED_LABEL_COLUMN], + group_columns: group_columns(req, description)?, value_column: value_column(description), }, peel_labeled::, @@ -348,7 +373,11 @@ where G::Probe: 'static, G::Answer: 'static, { - let (probes, score) = questions(ground_truth, &table)?; + let (probes, score, per_group) = questions(ground_truth, &table)?; + let score = match &req.per_group_out { + Some(path) => per_group_written(score, per_group, path)?, + None => score, + }; let items = materialise(description, table)?; let mut bodies = Vec::with_capacity(want.len()); for &(operation, metric) in want { @@ -399,6 +428,36 @@ where Ok(bodies) } +/// The scorer, also writing each scored group's error to `path` as +/// `group_key,n_q,error` whenever it scores. Every pass answers the same, so +/// each rewrites the same rows. The file is created here, so a path that +/// cannot be written fails before anything is measured. +fn per_group_written( + score: Score, + per_group: PerGroup, + path: &std::path::Path, +) -> Result, RunError> { + let fail = + |e: std::io::Error| RunError::Sketch(format!("--per-group-out {}: {e}", path.display())); + std::fs::File::create(path).map_err(fail)?; + let path = path.to_path_buf(); + Ok(Rc::new(move |answers: &[A]| { + let mut csv = String::from("group_key,n_q,error\n"); + for g in per_group(answers) { + // A label may hold a comma or a quote; CSV quotes the field then. + let key = if g.group.contains([',', '"', '\n']) { + format!("\"{}\"", g.group.replace('"', "\"\"")) + } else { + g.group + }; + csv.push_str(&format!("{key},{},{}\n", g.n_q, g.error)); + } + std::fs::write(&path, csv) + .unwrap_or_else(|e| panic!("--per-group-out {}: {e}", path.display())); + score(answers) + })) +} + /// A row that answers **nothing**: measured but not scored. The parallel-insert /// rows, whose worker sketches are dropped rather than asked, and whose ingest /// is one call over the whole stream rather than a loop over it. diff --git a/aqpbm-cli/tests/hydra_grouping.rs b/aqpbm-cli/tests/hydra_grouping.rs new file mode 100644 index 00000000..50464482 --- /dev/null +++ b/aqpbm-cli/tests/hydra_grouping.rs @@ -0,0 +1,137 @@ +//! `--group-columns`: a Hydra row asks and scores the label subset it names, +//! any non-empty one, not only column 0. On a grid large enough that no two of +//! the few subkeys share a column in most rows, the sketch answers each group +//! as the exact baseline does, so a group asked at the wrong columns shows up +//! as error. + +use std::process::Command; + +/// Two overlapping label columns (both render ranks over one alphabet, so +/// `b` is a value of each) over few values, and a value column at `dtype`. +fn spec(dtype: &str) -> std::path::PathBuf { + let path = std::env::temp_dir().join(format!( + "hydra_grouping_{}_{dtype}.yaml", + std::process::id() + )); + let yaml = format!( + "column_num: 3 +column_label: [key1, key2, value] +row_num: 5000 +column_spec: + - data_type: string + distribution: {{kind: uniform, lower_bound: 0.0, upper_bound: 4.0, seed: 1}} + - data_type: string + distribution: {{kind: zipf, skewness: 1.1, population_size: 3, seed: 2}} + - data_type: {dtype} + distribution: {{kind: zipf, skewness: 1.2, population_size: 40, seed: 3}} +" + ); + std::fs::write(&path, yaml).expect("the temp dir is writable"); + path +} + +/// The query accuracy record of one run, plus the per-group CSV it wrote. +fn scored( + variant: &str, + library: &str, + config: &str, + dtype: &str, + group_columns: &str, +) -> (serde_json::Value, String) { + let spec = spec(dtype); + let csv = spec.with_extension(format!("{variant}.{library}.{group_columns}.csv")); + let out = Command::new(env!("CARGO_BIN_EXE_approxbench")) + .args(["sketchbench", "--variant", variant, "--library", library]) + .args(["--config", config, "--dtype", dtype]) + .args(["--spec", spec.to_str().expect("utf-8 path")]) + .args(["--group-columns", group_columns]) + .args(["--per-group-out", csv.to_str().expect("utf-8 path")]) + .args(["--operations", "query", "--metrics", "accuracy"]) + .args(["--runs", "1", "--warmup-runs", "0"]) + .env("BENCH_WARMUP_SECS", "0") + .output() + .expect("approxbench runs"); + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + let stdout = String::from_utf8(out.stdout).expect("utf-8"); + let record: serde_json::Value = + serde_json::from_str(stdout.lines().last().expect("one record")).expect("JSON"); + let csv = std::fs::read_to_string(&csv).expect("--per-group-out was written"); + (record["bench"]["accuracy"].clone(), csv) +} + +/// Every Hydra family, lib at a grid far larger than its 19 subkeys and the +/// polars baseline, grouped by column 1 alone and by both columns: both score +/// (about) zero, and the per-group file has one row per scored group, keyed by +/// the columns asked. +#[test] +fn any_label_subset_is_asked_where_it_was_inserted() { + let rows = [ + ("hydra-cms", "rows=3 cols=1024 cell_rows=3 cell_cols=256", "i64"), + ("hydra-cs", "rows=3 cols=1024 cell_rows=3 cell_cols=256", "i64"), + ("hydra-hll", "rows=3 cols=1024", "i64"), + ("hydra-kll", "rows=3 cols=1024 cell_k=4096", "f64"), + ( + "hydra-univmon-l1-norm", + "rows=3 cols=1024 cell_heap_size=64 cell_sketch_row=3 cell_sketch_col=256 cell_layer_size=4", + "i64", + ), + ]; + for (variant, config, dtype) in rows { + for group_columns in ["1", "0,1"] { + for (library, tolerance) in [("polars", 0.0), ("lib", 0.01)] { + let (acc, csv) = scored(variant, library, config, dtype, group_columns); + let at = format!("{variant}/{library} --group-columns {group_columns}: {acc}"); + assert!(acc["err_max"].as_f64().expect(&at) <= tolerance, "{at}"); + assert_eq!(acc["schema_width"], 2.0, "{at}"); + assert_eq!(acc["records"], 5000.0, "{at}"); + assert_eq!(acc["fanned_mass"], 15000.0, "{at}"); + + let mut lines = csv.lines(); + assert_eq!(lines.next(), Some("group_key,n_q,error"), "{at}"); + let rows: Vec<&str> = lines.collect(); + assert_eq!(rows.len() as f64, acc["groups_scored"].as_f64().expect(&at)); + let n_q: u64 = rows + .iter() + .map(|r| r.split(',').nth(1).and_then(|n| n.parse::().ok())) + .map(|n| n.expect("n_q is a count")) + .sum(); + assert_eq!(n_q, 5000, "every record is in exactly one group: {at}"); + let key = rows[0].split(',').next().expect("a key"); + match group_columns { + "1" => assert!(key.starts_with("label1:") && !key.contains(';'), "{key}"), + _ => assert!( + key.starts_with("label0:") && key.contains(";label1:"), + "{key}" + ), + } + } + } + } +} + +/// A grouping column has to be a label column: the value column is not a key +/// of the grid. +#[test] +fn a_group_column_past_the_labels_is_refused() { + let spec = spec("i64"); + let out = Command::new(env!("CARGO_BIN_EXE_approxbench")) + .args(["sketchbench", "--variant", "hydra-cms", "--library", "lib"]) + .args([ + "--spec", + spec.to_str().expect("utf-8 path"), + "--dtype", + "i64", + ]) + .args(["--group-columns", "2"]) + .args(["--operations", "query", "--metrics", "accuracy"]) + .args(["--runs", "1", "--warmup-runs", "0"]) + .output() + .expect("approxbench runs"); + assert!(!out.status.success()); + let err = String::from_utf8_lossy(&out.stderr); + assert!(err.contains("--group-columns 2"), "{err}"); +} diff --git a/aqpbm-core/src/accuracy/mod.rs b/aqpbm-core/src/accuracy/mod.rs index 1310d39d..698b38d0 100644 --- a/aqpbm-core/src/accuracy/mod.rs +++ b/aqpbm-core/src/accuracy/mod.rs @@ -102,6 +102,27 @@ pub trait GroundTruth { probes: &[Self::Probe], answers: &[Self::Answer], ) -> BTreeMap; + + /// ④ Each scored group's error, for a comparator whose population is + /// subpopulations; empty for every other. [`score`](Self::score) summarises + /// the same rows, and `--per-group-out` writes them. + fn per_group( + &self, + _truth: &Self::Truth, + _probes: &[Self::Probe], + _answers: &[Self::Answer], + ) -> Vec { + Vec::new() + } +} + +/// One scored subpopulation: its key, how many records it carried (`n_q`), and +/// its error in the comparator's own metric, lower is better. +#[derive(Clone, Debug, PartialEq)] +pub struct GroupError { + pub group: String, + pub n_q: u64, + pub error: f64, } /// How a row's answers turn into the named error metrics: the scorer @@ -109,14 +130,18 @@ pub trait GroundTruth { /// comparator type. pub type Score = Rc BTreeMap>; +/// The same answers, read back group by group: [`GroundTruth::per_group`]. +pub type PerGroup = Rc Vec>; + /// Draw the questions: the exact answer over the generated table, the probes -/// off it, and a scorer holding both. Run once per row, outside every clock, -/// so no comparator type reaches a measurement. +/// off it, and a scorer holding both (and the per-group reader beside it). Run +/// once per row, outside every clock, so no comparator type reaches a +/// measurement. #[allow(clippy::type_complexity)] pub fn questions( gt: G, table: &GeneratedTable, -) -> Result<(Rc>, Score), RunError> +) -> Result<(Rc>, Score, PerGroup), RunError> where G: GroundTruth + 'static, G::Truth: 'static, @@ -133,7 +158,11 @@ where let (gt, truth, probes) = (gt.clone(), truth.clone(), probes.clone()); Rc::new(move |answers: &[G::Answer]| gt.score(&truth, &probes, answers)) }; - Ok((probes, score)) + let per_group: PerGroup = { + let (gt, truth, probes) = (gt.clone(), truth.clone(), probes.clone()); + Rc::new(move |answers: &[G::Answer]| gt.per_group(&truth, &probes, answers)) + }; + Ok((probes, score, per_group)) } pub(crate) fn scalar_error(truth: f64, answers: &[f64]) -> BTreeMap { diff --git a/aqpbm-core/src/accuracy/subpopulation/cardinality.rs b/aqpbm-core/src/accuracy/subpopulation/cardinality.rs index 69e990b5..c3128b81 100644 --- a/aqpbm-core/src/accuracy/subpopulation/cardinality.rs +++ b/aqpbm-core/src/accuracy/subpopulation/cardinality.rs @@ -7,8 +7,8 @@ use std::hash::Hash; use aqpbm_datagen::{ColumnData, DataGenError, GeneratedTable}; -use super::{group_labels, owned, Group}; -use crate::accuracy::{curve, GroundTruth}; +use super::{group_errors, group_labels, owned, write_groups, GroupKey}; +use crate::accuracy::{curve, GroundTruth, GroupError}; /// Ground truth for a grouped cardinality sketch: how many distinct values one /// subpopulation held. The population is the **subpopulation**, one entry per @@ -19,75 +19,91 @@ pub struct SubpopCardinalityGT { pub value_column: usize, } -/// Exact distinct-value counts per group, plus the ranking and the unfiltered -/// population. +/// Exact distinct-value counts per group, plus the ranking, the unfiltered +/// population, each group's record count and the stream's shape. pub struct SubpopCardTruth { - exact: HashMap, - ranked: Vec, - all: Vec, + exact: HashMap, + ranked: Vec, + all: Vec, + sizes: HashMap, + records: u64, + schema_width: usize, } /// One pass over the rows at whatever type the value column holds: a row /// contributes its value to exactly one group, and the group's truth is the -/// size of that set. +/// size of that set; the second number is how many records it carried. fn distinct_per_group( labels: &[&str], - width: usize, + columns: &[usize], values: &[V], -) -> HashMap { - let mut per_group: HashMap, HashSet<&V>> = HashMap::new(); - for (group, value) in labels.chunks(width).zip(values) { +) -> HashMap { + let mut per_group: HashMap, (HashSet<&V>, u64)> = HashMap::new(); + for (group, value) in labels.chunks(columns.len()).zip(values) { match per_group.get_mut(group) { - Some(seen) => { + Some((seen, records)) => { seen.insert(value); + *records += 1; } None => { - per_group.insert(group.to_vec(), HashSet::from([value])); + per_group.insert(group.to_vec(), (HashSet::from([value]), 1)); } } } per_group .into_iter() - .map(|(group, seen)| (owned(&group), seen.len() as u64)) + .map(|(group, (seen, records))| (owned(&group, columns), (seen.len() as u64, records))) .collect() } impl GroundTruth for SubpopCardinalityGT { type Truth = SubpopCardTruth; - type Probe = Group; + type Probe = GroupKey; type Answer = f64; fn truth(&self, table: &GeneratedTable) -> Result { let labels = group_labels(table, &self.group_columns)?; - let width = self.group_columns.len(); - let exact = match table.column(self.value_column)? { - ColumnData::Int64(v) => distinct_per_group(&labels, width, v), - ColumnData::Unsigned64(v) => distinct_per_group(&labels, width, v), - ColumnData::String(v) => distinct_per_group(&labels, width, v), + let columns = &self.group_columns; + let counted = match table.column(self.value_column)? { + ColumnData::Int64(v) => distinct_per_group(&labels, columns, v), + ColumnData::Unsigned64(v) => distinct_per_group(&labels, columns, v), + ColumnData::String(v) => distinct_per_group(&labels, columns, v), ColumnData::Float64(v) => { let bits: Vec = v.iter().map(|x| x.to_bits()).collect(); - distinct_per_group(&labels, width, &bits) + distinct_per_group(&labels, columns, &bits) } }; + let exact: HashMap = + counted.iter().map(|(g, (d, _))| (g.clone(), *d)).collect(); + let sizes = counted.into_iter().map(|(g, (_, n))| (g, n)).collect(); - let mut by_count: Vec<(Group, u64)> = exact.iter().map(|(g, c)| (g.clone(), *c)).collect(); + let mut by_count: Vec<(GroupKey, u64)> = + exact.iter().map(|(g, c)| (g.clone(), *c)).collect(); by_count.sort_unstable_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); let all = curve::shuffled(&by_count); let ranked = by_count.into_iter().map(|(g, _)| g).collect(); - Ok(SubpopCardTruth { exact, ranked, all }) + Ok(SubpopCardTruth { + exact, + ranked, + all, + sizes, + records: table.row_num, + schema_width: self.value_column, + }) } - fn probes(&self, truth: &SubpopCardTruth) -> Vec { + fn probes(&self, truth: &SubpopCardTruth) -> Vec { curve::union_of(&truth.all, &truth.ranked, Clone::clone) } fn score( &self, truth: &SubpopCardTruth, - probes: &[Group], + probes: &[GroupKey], answers: &[f64], ) -> BTreeMap { - let est: HashMap<&Group, f64> = probes.iter().zip(answers).map(|(p, a)| (p, *a)).collect(); + let est: HashMap<&GroupKey, f64> = + probes.iter().zip(answers).map(|(p, a)| (p, *a)).collect(); let mut metrics: BTreeMap = BTreeMap::new(); curve::error_curve( &truth.ranked, @@ -102,8 +118,25 @@ impl GroundTruth for SubpopCardinalityGT { ); metrics.insert("subpopulations".into(), truth.exact.len() as f64); metrics.insert("group_columns".into(), self.group_columns.len() as f64); + let groups = self.per_group(truth, probes, answers); + write_groups(&groups, truth.records, truth.schema_width, &mut metrics); metrics } + + /// Relative error per group, as `are_all` averages it. + fn per_group( + &self, + truth: &SubpopCardTruth, + probes: &[GroupKey], + answers: &[f64], + ) -> Vec { + let est: HashMap<&GroupKey, f64> = + probes.iter().zip(answers).map(|(p, a)| (p, *a)).collect(); + group_errors(truth.exact.iter().filter(|(_, &d)| d > 0).map(|(g, &d)| { + let err = (est.get(g).copied().unwrap_or(0.0) - d as f64).abs() / d as f64; + (g.clone(), truth.sizes[g], err) + })) + } } #[cfg(test)] @@ -120,7 +153,7 @@ mod tests { let truth = gt .truth(&records()) .expect("the table has the named columns"); - assert_eq!(truth.exact[&vec!["a".to_string()]], 2); - assert_eq!(truth.exact[&vec!["b".to_string()]], 1); + assert_eq!(truth.exact[&vec![Some("a".to_string())]], 2); + assert_eq!(truth.exact[&vec![Some("b".to_string())]], 1); } } diff --git a/aqpbm-core/src/accuracy/subpopulation/entropy.rs b/aqpbm-core/src/accuracy/subpopulation/entropy.rs index b768a08d..f96bb132 100644 --- a/aqpbm-core/src/accuracy/subpopulation/entropy.rs +++ b/aqpbm-core/src/accuracy/subpopulation/entropy.rs @@ -2,9 +2,11 @@ use std::collections::BTreeMap; use aqpbm_datagen::{DataGenError, GeneratedTable}; -use super::frequency_vector::{probes_over, score_over, truth_over, SubpopVectorTruth}; -use super::Group; -use crate::accuracy::GroundTruth; +use super::frequency_vector::{ + per_group_over, probes_over, score_over, truth_over, SubpopVectorTruth, +}; +use super::GroupKey; +use crate::accuracy::{GroundTruth, GroupError}; pub struct SubpopEntropyGT { /// Which label columns the subpopulation is taken over. @@ -29,25 +31,34 @@ pub fn entropy(counts: &[u64]) -> f64 { impl GroundTruth for SubpopEntropyGT { type Truth = SubpopVectorTruth; - type Probe = Group; + type Probe = GroupKey; type Answer = f64; fn truth(&self, table: &GeneratedTable) -> Result { truth_over(table, &self.group_columns, self.value_column, entropy) } - fn probes(&self, truth: &SubpopVectorTruth) -> Vec { + fn probes(&self, truth: &SubpopVectorTruth) -> Vec { probes_over(truth) } fn score( &self, truth: &SubpopVectorTruth, - probes: &[Group], + probes: &[GroupKey], answers: &[f64], ) -> BTreeMap { score_over(truth, probes, answers, self.group_columns.len()) } + + fn per_group( + &self, + truth: &SubpopVectorTruth, + probes: &[GroupKey], + answers: &[f64], + ) -> Vec { + per_group_over(truth, probes, answers) + } } #[cfg(test)] @@ -67,9 +78,9 @@ mod tests { let truth = over(vec![0]) .truth(&records()) .expect("the table has the named columns"); - assert_eq!(truth.exact[&vec!["b".to_string()]], 0.0); + assert_eq!(truth.exact[&vec![Some("b".to_string())]], 0.0); let expected = -(2.0 / 3.0 * (2.0f64 / 3.0).log2() + 1.0 / 3.0 * (1.0f64 / 3.0).log2()); - assert!((truth.exact[&vec!["a".to_string()]] - expected).abs() < 1e-12); + assert!((truth.exact[&vec![Some("a".to_string())]] - expected).abs() < 1e-12); } #[test] diff --git a/aqpbm-core/src/accuracy/subpopulation/frequency.rs b/aqpbm-core/src/accuracy/subpopulation/frequency.rs index 381cacd2..75a8b2d1 100644 --- a/aqpbm-core/src/accuracy/subpopulation/frequency.rs +++ b/aqpbm-core/src/accuracy/subpopulation/frequency.rs @@ -6,8 +6,8 @@ use std::marker::PhantomData; use aqpbm_datagen::{DataGenError, GeneratedTable}; -use super::{group_labels, owned, Group}; -use crate::accuracy::{curve, CountedValue, GroundTruth}; +use super::{group_errors, group_labels, owned, write_groups, GroupKey}; +use crate::accuracy::{curve, CountedValue, GroundTruth, GroupError}; pub struct SubpopFrequencyGT { /// Which label columns the subpopulation is taken over. A grouped sketch @@ -28,19 +28,23 @@ impl SubpopFrequencyGT { } } -/// The exact per-(group, value) counts, the ranking, and the unfiltered -/// population. Owned: the truth outlives the borrow of the table. +/// The exact per-(group, value) counts, the ranking, the unfiltered +/// population, each group's record count and the stream's shape. Owned: the +/// truth outlives the borrow of the table. pub struct SubpopFreqTruth { - exact: HashMap<(Group, V::CountKey), u64>, - ranked: Vec<(Group, V)>, - all: Vec<(Group, V)>, + exact: HashMap<(GroupKey, V::CountKey), u64>, + ranked: Vec<(GroupKey, V)>, + all: Vec<(GroupKey, V)>, subpopulations: usize, + sizes: HashMap, + records: u64, + schema_width: usize, } type CountedByGroup<'a, V> = HashMap, HashMap<::CountKey, (u64, V)>>; -fn counted_key(pair: &(Group, V)) -> (Group, V::CountKey) { +fn counted_key(pair: &(GroupKey, V)) -> (GroupKey, V::CountKey) { (pair.0.clone(), pair.1.count_key()) } @@ -49,7 +53,7 @@ where V: CountedValue, { type Truth = SubpopFreqTruth; - type Probe = (Group, V); + type Probe = (GroupKey, V); type Answer = f64; fn truth(&self, table: &GeneratedTable) -> Result, DataGenError> { @@ -76,10 +80,12 @@ where } } let subpopulations = counted.len(); - let mut exact: HashMap<(Group, V::CountKey), u64> = HashMap::new(); - let mut by_count: Vec<((Group, V), u64)> = Vec::new(); + let mut exact: HashMap<(GroupKey, V::CountKey), u64> = HashMap::new(); + let mut by_count: Vec<((GroupKey, V), u64)> = Vec::new(); + let mut sizes: HashMap = HashMap::with_capacity(subpopulations); for (group, seen) in counted { - let group = owned(&group); + let group = owned(&group, &self.group_columns); + sizes.insert(group.clone(), seen.values().map(|(count, _)| count).sum()); for (key, (count, value)) in seen { exact.insert((group.clone(), key), count); by_count.push(((group.clone(), value), count)); @@ -99,20 +105,23 @@ where ranked, all, subpopulations, + sizes, + records: table.row_num, + schema_width: self.value_column, }) } - fn probes(&self, truth: &SubpopFreqTruth) -> Vec<(Group, V)> { + fn probes(&self, truth: &SubpopFreqTruth) -> Vec<(GroupKey, V)> { curve::union_of(&truth.all, &truth.ranked, counted_key) } fn score( &self, truth: &SubpopFreqTruth, - probes: &[(Group, V)], + probes: &[(GroupKey, V)], answers: &[f64], ) -> BTreeMap { - let est: HashMap<(Group, V::CountKey), f64> = probes + let est: HashMap<(GroupKey, V::CountKey), f64> = probes .iter() .zip(answers) .map(|(p, a)| (counted_key(p), *a)) @@ -136,8 +145,37 @@ where // be read. metrics.insert("subpopulations".into(), truth.subpopulations as f64); metrics.insert("group_columns".into(), self.group_columns.len() as f64); + let groups = self.per_group(truth, probes, answers); + write_groups(&groups, truth.records, truth.schema_width, &mut metrics); metrics } + + /// A group's error is the relative error of its (group, value) pairs, + /// averaged over the group's pairs as `are_all` averages over every pair. + fn per_group( + &self, + truth: &SubpopFreqTruth, + probes: &[(GroupKey, V)], + answers: &[f64], + ) -> Vec { + let est: HashMap<(GroupKey, V::CountKey), f64> = probes + .iter() + .zip(answers) + .map(|(p, a)| (counted_key(p), *a)) + .collect(); + let mut per_group: HashMap<&GroupKey, (f64, usize)> = HashMap::new(); + for (key, &count) in &truth.exact { + let err = (est.get(key).copied().unwrap_or(0.0) - count as f64).abs() / count as f64; + let held = per_group.entry(&key.0).or_insert((0.0, 0)); + held.0 += err; + held.1 += 1; + } + group_errors( + per_group + .into_iter() + .map(|(g, (sum, pairs))| (g.clone(), truth.sizes[g], sum / pairs as f64)), + ) + } } #[cfg(test)] @@ -151,31 +189,17 @@ mod tests { /// and this double answers 0 whatever it was fed, so nothing needs to go /// into it. struct NullSubpop; - impl NullSubpop { - fn estimate_subpop_frequency(&self, _: &[&str], _: &i64) -> f64 { - 0.0 - } - } - fn query_null(s: &mut NullSubpop, p: &(Group, i64)) -> f64 { - s.estimate_subpop_frequency(&labels(&p.0), &p.1) - } - - fn labels(group: &Group) -> Vec<&str> { - group.iter().map(String::as_str).collect() + fn query_null(_: &mut NullSubpop, _: &(GroupKey, i64)) -> f64 { + 0.0 } /// The estimator that is exactly right, built from the same rows the /// comparator reads. Pins that the comparator's own truth is self-consistent. struct ExactSubpop { - counts: HashMap<(Group, i64), u64>, - } - impl ExactSubpop { - fn estimate_subpop_frequency(&self, group: &[&str], value: &i64) -> f64 { - *self.counts.get(&(owned(group), *value)).unwrap_or(&0) as f64 - } + counts: HashMap<(GroupKey, i64), u64>, } - fn query_exact(s: &mut ExactSubpop, p: &(Group, i64)) -> f64 { - s.estimate_subpop_frequency(&labels(&p.0), &p.1) + fn query_exact(s: &mut ExactSubpop, p: &(GroupKey, i64)) -> f64 { + *s.counts.get(p).unwrap_or(&0) as f64 } fn over(group_columns: Vec) -> SubpopFrequencyGT { @@ -258,6 +282,27 @@ mod tests { assert_eq!(cmp["group_columns"], 2.0); } + /// A group's error averages its own pairs: `a` holds (a,10) twice and + /// (a,20) once, so overcounting (a,10) by one misses that pair by 0.5 and + /// group `a` by 0.25, while `b` is exact. Over groups the mean is 0.125, + /// not `are_all`'s 0.5 / 3 over pairs. + #[test] + fn a_group_error_is_the_mean_over_its_pairs() { + let table = records(); + let gt = over(vec![0]); + let truth = gt.truth(&table).expect("the table has the named columns"); + let mut counts = truth.exact.clone(); + *counts + .get_mut(&(vec![Some("a".to_string())], 10)) + .expect("(a, 10) occurs") += 1; + let cmp = + crate::accuracy::score_with(>, &query_exact, &mut ExactSubpop { counts }, &table); + assert_eq!(cmp["err_mean"], 0.125); + assert_eq!(cmp["err_max"], 0.25); + assert_eq!(cmp["groups_scored"], 2.0); + assert!((cmp["are_all"] - 0.5 / 3.0).abs() < 1e-12); + } + #[test] fn a_grouping_column_that_is_not_text_is_refused() { let Err(err) = over(vec![2]).truth(&records()) else { diff --git a/aqpbm-core/src/accuracy/subpopulation/frequency_vector.rs b/aqpbm-core/src/accuracy/subpopulation/frequency_vector.rs index 76f8b287..43d7b209 100644 --- a/aqpbm-core/src/accuracy/subpopulation/frequency_vector.rs +++ b/aqpbm-core/src/accuracy/subpopulation/frequency_vector.rs @@ -4,22 +4,25 @@ use std::hash::Hash; use aqpbm_datagen::{ColumnData, DataGenError, GeneratedTable}; -use super::{group_labels, owned, Group}; -use crate::accuracy::curve; +use super::{group_errors, group_labels, owned, write_groups, GroupKey}; +use crate::accuracy::{curve, GroupError}; pub struct SubpopVectorTruth { - pub(super) exact: HashMap, - ranked: Vec, - all: Vec, + pub(super) exact: HashMap, + ranked: Vec, + all: Vec, + sizes: HashMap, + records: u64, + schema_width: usize, } fn histogram_per_group( labels: &[&str], - width: usize, + columns: &[usize], values: &[V], -) -> HashMap> { +) -> HashMap> { let mut per_group: HashMap, HashMap<&V, u64>> = HashMap::new(); - for (group, value) in labels.chunks(width).zip(values) { + for (group, value) in labels.chunks(columns.len()).zip(values) { match per_group.get_mut(group) { Some(seen) => *seen.entry(value).or_insert(0) += 1, None => { @@ -29,7 +32,7 @@ fn histogram_per_group( } per_group .into_iter() - .map(|(group, seen)| (owned(&group), seen.into_values().collect())) + .map(|(group, seen)| (owned(&group, columns), seen.into_values().collect())) .collect() } @@ -37,16 +40,16 @@ pub(super) fn counts_per_group( table: &GeneratedTable, group_columns: &[usize], value_column: usize, -) -> Result>, DataGenError> { +) -> Result>, DataGenError> { let labels = group_labels(table, group_columns)?; - let width = group_columns.len(); + let columns = group_columns; Ok(match table.column(value_column)? { - ColumnData::Int64(v) => histogram_per_group(&labels, width, v), - ColumnData::Unsigned64(v) => histogram_per_group(&labels, width, v), - ColumnData::String(v) => histogram_per_group(&labels, width, v), + ColumnData::Int64(v) => histogram_per_group(&labels, columns, v), + ColumnData::Unsigned64(v) => histogram_per_group(&labels, columns, v), + ColumnData::String(v) => histogram_per_group(&labels, columns, v), ColumnData::Float64(v) => { let bits: Vec = v.iter().map(|x| x.to_bits()).collect(); - histogram_per_group(&labels, width, &bits) + histogram_per_group(&labels, columns, &bits) } }) } @@ -58,14 +61,15 @@ pub(super) fn truth_over( fold: impl Fn(&[u64]) -> f64, ) -> Result { let counts = counts_per_group(table, group_columns, value_column)?; - let exact: HashMap = counts + let exact: HashMap = counts .iter() .map(|(group, counts)| (group.clone(), fold(counts))) .collect(); - let mut by_statistic: Vec<(Group, f64)> = exact.iter().map(|(g, v)| (g.clone(), *v)).collect(); + let mut by_statistic: Vec<(GroupKey, f64)> = + exact.iter().map(|(g, v)| (g.clone(), *v)).collect(); by_statistic.sort_unstable_by(|a, b| b.1.total_cmp(&a.1).then_with(|| a.0.cmp(&b.0))); - let sized: Vec<(Group, u64)> = by_statistic + let sized: Vec<(GroupKey, u64)> = by_statistic .iter() .map(|(g, _)| { let size = counts.get(g).map(|c| c.iter().sum()).unwrap_or(0); @@ -73,21 +77,29 @@ pub(super) fn truth_over( }) .collect(); let all = curve::shuffled(&sized); + let sizes = sized.into_iter().collect(); let ranked = by_statistic.into_iter().map(|(g, _)| g).collect(); - Ok(SubpopVectorTruth { exact, ranked, all }) + Ok(SubpopVectorTruth { + exact, + ranked, + all, + sizes, + records: table.row_num, + schema_width: value_column, + }) } -pub(super) fn probes_over(truth: &SubpopVectorTruth) -> Vec { +pub(super) fn probes_over(truth: &SubpopVectorTruth) -> Vec { curve::union_of(&truth.all, &truth.ranked, Clone::clone) } pub(super) fn score_over( truth: &SubpopVectorTruth, - probes: &[Group], + probes: &[GroupKey], answers: &[f64], group_columns: usize, ) -> BTreeMap { - let est: HashMap<&Group, f64> = probes.iter().zip(answers).map(|(p, a)| (p, *a)).collect(); + let est: HashMap<&GroupKey, f64> = probes.iter().zip(answers).map(|(p, a)| (p, *a)).collect(); let mut metrics: BTreeMap = BTreeMap::new(); curve::error_curve( &truth.ranked, @@ -102,16 +114,33 @@ pub(super) fn score_over( ); metrics.insert("subpopulations".into(), truth.exact.len() as f64); metrics.insert("group_columns".into(), group_columns as f64); + let groups = per_group_over(truth, probes, answers); + write_groups(&groups, truth.records, truth.schema_width, &mut metrics); metrics } +/// Relative error per group, as `are_all` averages it: a group whose true +/// statistic is zero (entropy over one value) has no relative error and is +/// not scored. +pub(super) fn per_group_over( + truth: &SubpopVectorTruth, + probes: &[GroupKey], + answers: &[f64], +) -> Vec { + let est: HashMap<&GroupKey, f64> = probes.iter().zip(answers).map(|(p, a)| (p, *a)).collect(); + group_errors(truth.exact.iter().filter(|(_, &v)| v > 0.0).map(|(g, &v)| { + let err = (est.get(g).copied().unwrap_or(0.0) - v).abs() / v; + (g.clone(), truth.sizes[g], err) + })) +} + #[cfg(test)] mod tests { use super::*; use crate::accuracy::subpopulation::records; - fn group(label: &str) -> Group { - vec![label.to_string()] + fn group(label: &str) -> GroupKey { + vec![Some(label.to_string())] } #[test] diff --git a/aqpbm-core/src/accuracy/subpopulation/l1_norm.rs b/aqpbm-core/src/accuracy/subpopulation/l1_norm.rs index 9a9710ca..28ca38e2 100644 --- a/aqpbm-core/src/accuracy/subpopulation/l1_norm.rs +++ b/aqpbm-core/src/accuracy/subpopulation/l1_norm.rs @@ -2,9 +2,11 @@ use std::collections::BTreeMap; use aqpbm_datagen::{DataGenError, GeneratedTable}; -use super::frequency_vector::{probes_over, score_over, truth_over, SubpopVectorTruth}; -use super::Group; -use crate::accuracy::GroundTruth; +use super::frequency_vector::{ + per_group_over, probes_over, score_over, truth_over, SubpopVectorTruth, +}; +use super::GroupKey; +use crate::accuracy::{GroundTruth, GroupError}; pub struct SubpopL1NormGT { /// Which label columns the subpopulation is taken over. @@ -18,25 +20,34 @@ pub fn l1_norm(counts: &[u64]) -> f64 { impl GroundTruth for SubpopL1NormGT { type Truth = SubpopVectorTruth; - type Probe = Group; + type Probe = GroupKey; type Answer = f64; fn truth(&self, table: &GeneratedTable) -> Result { truth_over(table, &self.group_columns, self.value_column, l1_norm) } - fn probes(&self, truth: &SubpopVectorTruth) -> Vec { + fn probes(&self, truth: &SubpopVectorTruth) -> Vec { probes_over(truth) } fn score( &self, truth: &SubpopVectorTruth, - probes: &[Group], + probes: &[GroupKey], answers: &[f64], ) -> BTreeMap { score_over(truth, probes, answers, self.group_columns.len()) } + + fn per_group( + &self, + truth: &SubpopVectorTruth, + probes: &[GroupKey], + answers: &[f64], + ) -> Vec { + per_group_over(truth, probes, answers) + } } #[cfg(test)] @@ -56,8 +67,8 @@ mod tests { let truth = over(vec![0]) .truth(&records()) .expect("the table has the named columns"); - assert_eq!(truth.exact[&vec!["a".to_string()]], 3.0); - assert_eq!(truth.exact[&vec!["b".to_string()]], 3.0); + assert_eq!(truth.exact[&vec![Some("a".to_string())]], 3.0); + assert_eq!(truth.exact[&vec![Some("b".to_string())]], 3.0); } #[test] @@ -74,6 +85,65 @@ mod tests { assert_eq!(cmp["aae_all"], 0.0); } + /// The per-group summary on a fixed case. Over both columns the groups + /// carry 2, 1, 1 and 2 records; answering 2, 2, 1.5 and 3 misses them by + /// 0, 1, 0.5 and 0.5 relatively, so sorted `[0, 0.5, 0.5, 1]`: mean 0.5, + /// p50 (index round(1.5) = 2) 0.5, p90 (index round(2.7) = 3) 1, max 1. + #[test] + fn per_group_errors_are_summarised_over_groups() { + let gt = over(vec![0, 1]); + let table = records(); + let key = |a: &str, b: &str| vec![Some(a.to_string()), Some(b.to_string())]; + let mut answers = std::collections::HashMap::from([ + (key("a", "x"), 2.0), + (key("a", "y"), 2.0), + (key("b", "x"), 1.5), + (key("b", "y"), 3.0), + ]); + let ask = |held: &mut std::collections::HashMap, p: &GroupKey| held[p]; + let cmp = crate::accuracy::score_with(>, &ask, &mut answers, &table); + assert_eq!(cmp["err_mean"], 0.5); + assert_eq!(cmp["err_p50"], 0.5); + assert_eq!(cmp["err_p90"], 1.0); + assert_eq!(cmp["err_max"], 1.0); + assert_eq!(cmp["groups_scored"], 4.0); + assert_eq!(cmp["schema_width"], 2.0); + assert_eq!(cmp["records"], 6.0); + assert_eq!(cmp["fanned_mass"], 18.0); + + let truth = gt.truth(&table).expect("the table has the named columns"); + let probes = gt.probes(&truth); + let asked: Vec = probes.iter().map(|p| answers[p]).collect(); + let rows: Vec<(String, u64, f64)> = gt + .per_group(&truth, &probes, &asked) + .into_iter() + .map(|g| (g.group, g.n_q, g.error)) + .collect(); + assert_eq!( + rows, + vec![ + ("label0:a;label1:x".to_string(), 2, 0.0), + ("label0:b;label1:y".to_string(), 2, 0.5), + ("label0:a;label1:y".to_string(), 1, 1.0), + ("label0:b;label1:x".to_string(), 1, 0.5), + ] + ); + } + + /// Column 1 alone is asked with column 0 left open, and named by column 1. + #[test] + fn a_group_on_a_later_column_leaves_the_earlier_ones_open() { + let gt = over(vec![1]); + let truth = gt + .truth(&records()) + .expect("the table has the named columns"); + assert_eq!(truth.exact[&vec![None, Some("x".to_string())]], 3.0); + let probes = gt.probes(&truth); + let groups = gt.per_group(&truth, &probes, &vec![0.0; probes.len()]); + let keys: Vec<&str> = groups.iter().map(|g| g.group.as_str()).collect(); + assert_eq!(keys, ["label1:x", "label1:y"]); + } + #[test] fn a_different_grouping_is_a_different_population() { let cmp = null_over(&over(vec![0, 1]), &records()); diff --git a/aqpbm-core/src/accuracy/subpopulation/l2_norm.rs b/aqpbm-core/src/accuracy/subpopulation/l2_norm.rs index a2069e22..267da2da 100644 --- a/aqpbm-core/src/accuracy/subpopulation/l2_norm.rs +++ b/aqpbm-core/src/accuracy/subpopulation/l2_norm.rs @@ -2,9 +2,11 @@ use std::collections::BTreeMap; use aqpbm_datagen::{DataGenError, GeneratedTable}; -use super::frequency_vector::{probes_over, score_over, truth_over, SubpopVectorTruth}; -use super::Group; -use crate::accuracy::GroundTruth; +use super::frequency_vector::{ + per_group_over, probes_over, score_over, truth_over, SubpopVectorTruth, +}; +use super::GroupKey; +use crate::accuracy::{GroundTruth, GroupError}; pub struct SubpopL2NormGT { /// Which label columns the subpopulation is taken over. @@ -22,25 +24,34 @@ pub fn l2_norm(counts: &[u64]) -> f64 { impl GroundTruth for SubpopL2NormGT { type Truth = SubpopVectorTruth; - type Probe = Group; + type Probe = GroupKey; type Answer = f64; fn truth(&self, table: &GeneratedTable) -> Result { truth_over(table, &self.group_columns, self.value_column, l2_norm) } - fn probes(&self, truth: &SubpopVectorTruth) -> Vec { + fn probes(&self, truth: &SubpopVectorTruth) -> Vec { probes_over(truth) } fn score( &self, truth: &SubpopVectorTruth, - probes: &[Group], + probes: &[GroupKey], answers: &[f64], ) -> BTreeMap { score_over(truth, probes, answers, self.group_columns.len()) } + + fn per_group( + &self, + truth: &SubpopVectorTruth, + probes: &[GroupKey], + answers: &[f64], + ) -> Vec { + per_group_over(truth, probes, answers) + } } #[cfg(test)] @@ -60,8 +71,8 @@ mod tests { let truth = over(vec![0]) .truth(&records()) .expect("the table has the named columns"); - assert!((truth.exact[&vec!["a".to_string()]] - 5.0f64.sqrt()).abs() < 1e-12); - assert_eq!(truth.exact[&vec!["b".to_string()]], 3.0); + assert!((truth.exact[&vec![Some("a".to_string())]] - 5.0f64.sqrt()).abs() < 1e-12); + assert_eq!(truth.exact[&vec![Some("b".to_string())]], 3.0); } #[test] diff --git a/aqpbm-core/src/accuracy/subpopulation/mod.rs b/aqpbm-core/src/accuracy/subpopulation/mod.rs index 76d7f1ad..2f16d227 100644 --- a/aqpbm-core/src/accuracy/subpopulation/mod.rs +++ b/aqpbm-core/src/accuracy/subpopulation/mod.rs @@ -17,10 +17,21 @@ pub use l1_norm::{l1_norm, SubpopL1NormGT}; pub use l2_norm::{l2_norm, SubpopL2NormGT}; pub use rank_error::{SubpopRankErrorGT, SubpopRankTruth}; +use std::collections::BTreeMap; + use aqpbm_datagen::{ColumnItem, DataGenError, GeneratedTable}; +use crate::accuracy::curve::percentile; +use crate::accuracy::GroupError; + pub type Group = Vec; +/// A subpopulation as a grid is asked for it: one slot per label column up to +/// the last one grouped on, `Some` at the grouped columns and `None` at the +/// rest. Grouping on column 1 alone asks `[None, Some(v)]`, which a wrapper +/// can tell apart from grouping on column 0. +pub type GroupKey = Vec>; + pub(crate) fn group_labels<'a>( table: &'a GeneratedTable, columns: &[usize], @@ -45,8 +56,67 @@ pub(crate) fn group_labels<'a>( Ok(flat) } -fn owned(group: &[&str]) -> Group { - group.iter().map(|label| (*label).to_string()).collect() +/// `group` holds one label per entry of `columns`, in that order. +fn owned(group: &[&str], columns: &[usize]) -> GroupKey { + let mut key = vec![None; columns.iter().max().map_or(0, |last| last + 1)]; + for (label, &column) in group.iter().zip(columns) { + key[column] = Some((*label).to_string()); + } + key +} + +/// How a group is named in `--per-group-out`: each grouped column's value +/// tagged with its column, as the grid's subkeys are (`label1:x`). +fn rendered(key: &GroupKey) -> String { + key.iter() + .enumerate() + .filter_map(|(j, label)| label.as_ref().map(|l| format!("label{j}:{l}"))) + .collect::>() + .join(";") +} + +/// One [`GroupError`] per group, heaviest first (ties on the key), so the +/// per-group file reads the same across runs. +fn group_errors(errors: impl Iterator) -> Vec { + let mut out: Vec = errors + .map(|(key, n_q, error)| GroupError { + group: rendered(&key), + n_q, + error, + }) + .collect(); + out.sort_by(|a, b| b.n_q.cmp(&a.n_q).then_with(|| a.group.cmp(&b.group))); + out +} + +/// The per-group summary every grouped comparator reports, in its own metric: +/// `err_mean` / `err_p50` / `err_p90` / `err_max` over the scored groups (none +/// written when no group was scored), `groups_scored`, and the stream's shape +/// against which a group's error is read: `schema_width` (`d`, the label +/// columns before the value), `records` (`N`) and `fanned_mass` +/// (`N·(2^d − 1)`, every subset a record is inserted into). +fn write_groups( + groups: &[GroupError], + records: u64, + schema_width: usize, + metrics: &mut BTreeMap, +) { + let mut errs: Vec = groups.iter().map(|g| g.error).collect(); + errs.sort_by(f64::total_cmp); + if let Some(&max) = errs.last() { + metrics.insert( + "err_mean".into(), + errs.iter().sum::() / errs.len() as f64, + ); + metrics.insert("err_p50".into(), percentile(&errs, 0.5)); + metrics.insert("err_p90".into(), percentile(&errs, 0.9)); + metrics.insert("err_max".into(), max); + } + metrics.insert("groups_scored".into(), errs.len() as f64); + metrics.insert("schema_width".into(), schema_width as f64); + metrics.insert("records".into(), records as f64); + let subsets = 2f64.powi(schema_width as i32) - 1.0; + metrics.insert("fanned_mass".into(), records as f64 * subsets); } /// key1 ∈ {a, b}, key2 ∈ {x, y}; the value column is what gets counted. @@ -76,15 +146,15 @@ fn records() -> GeneratedTable { #[cfg(test)] fn null_over(gt: &G, table: &GeneratedTable) -> std::collections::BTreeMap where - G: crate::accuracy::GroundTruth, + G: crate::accuracy::GroundTruth, { - crate::accuracy::score_with(gt, &|_: &mut (), _: &Group| 0.0, &mut (), table) + crate::accuracy::score_with(gt, &|_: &mut (), _: &GroupKey| 0.0, &mut (), table) } #[cfg(test)] fn exact_over(gt: &G, table: &GeneratedTable) -> std::collections::BTreeMap where - G: crate::accuracy::GroundTruth, + G: crate::accuracy::GroundTruth, { let mut exact = gt .truth(table) @@ -92,7 +162,7 @@ where .exact; crate::accuracy::score_with( gt, - &|held: &mut std::collections::HashMap, p: &Group| { + &|held: &mut std::collections::HashMap, p: &GroupKey| { held.get(p).copied().unwrap_or(0.0) }, &mut exact, diff --git a/aqpbm-core/src/accuracy/subpopulation/rank_error.rs b/aqpbm-core/src/accuracy/subpopulation/rank_error.rs index 5364184c..4c2f65fa 100644 --- a/aqpbm-core/src/accuracy/subpopulation/rank_error.rs +++ b/aqpbm-core/src/accuracy/subpopulation/rank_error.rs @@ -5,9 +5,9 @@ use std::collections::HashMap; use aqpbm_datagen::{DataGenError, GeneratedTable}; -use super::{group_labels, owned, Group}; +use super::{group_errors, group_labels, owned, write_groups, GroupKey}; use crate::accuracy::quantile::rank_err; -use crate::accuracy::{curve, f64_values, GroundTruth}; +use crate::accuracy::{curve, f64_values, GroundTruth, GroupError}; /// Number of quantiles probed per group — the same grid the ungrouped /// comparator uses, named once in [`super::quantile`]. @@ -24,17 +24,18 @@ pub struct SubpopRankErrorGT { /// The ordered statistic inside each group, plus the groups worth probing. pub struct SubpopRankTruth { - per_group: HashMap>, + per_group: HashMap>, /// Every group, shuffled: probe order must not hand the baseline the /// locality the encounter order would. - probed: Vec, + probed: Vec, items: usize, + schema_width: usize, } impl GroundTruth for SubpopRankErrorGT { type Truth = SubpopRankTruth; /// One (group, fraction) question. - type Probe = (Group, f64); + type Probe = (GroupKey, f64); type Answer = f64; fn truth(&self, table: &GeneratedTable) -> Result { @@ -53,13 +54,13 @@ impl GroundTruth for SubpopRankErrorGT { } } } - let mut per_group: HashMap> = HashMap::with_capacity(borrowed.len()); + let mut per_group: HashMap> = HashMap::with_capacity(borrowed.len()); for (group, mut held) in borrowed { held.sort_by(f64::total_cmp); - per_group.insert(owned(&group), held); + per_group.insert(owned(&group, &self.group_columns), held); } - let mut by_size: Vec<(Group, u64)> = per_group + let mut by_size: Vec<(GroupKey, u64)> = per_group .iter() .map(|(g, v)| (g.clone(), v.len() as u64)) .collect(); @@ -69,10 +70,11 @@ impl GroundTruth for SubpopRankErrorGT { per_group, probed, items: table.row_num as usize, + schema_width: self.value_column, }) } - fn probes(&self, truth: &SubpopRankTruth) -> Vec<(Group, f64)> { + fn probes(&self, truth: &SubpopRankTruth) -> Vec<(GroupKey, f64)> { let mut out = Vec::with_capacity(truth.probed.len() * GROUP_GRID_POINTS); for group in &truth.probed { match truth.per_group.get(group) { @@ -89,41 +91,16 @@ impl GroundTruth for SubpopRankErrorGT { fn score( &self, truth: &SubpopRankTruth, - probes: &[(Group, f64)], + probes: &[(GroupKey, f64)], answers: &[f64], ) -> BTreeMap { - let mut sum_mean = 0.0f64; - let mut max_err = 0.0f64; - let mut scored_groups = 0usize; - - // The probe set is one contiguous grid per group, in order. - for (chunk_p, chunk_a) in probes - .chunks(GROUP_GRID_POINTS) - .zip(answers.chunks(GROUP_GRID_POINTS)) - { - let Some((group, _)) = chunk_p.first() else { - continue; - }; - let Some(sorted) = truth.per_group.get(group) else { - continue; - }; - let mut group_sum = 0.0f64; - for ((_, q), est) in chunk_p.iter().zip(chunk_a) { - let err = rank_err(sorted, *est, *q); - group_sum += err; - if err > max_err { - max_err = err; - } - } - sum_mean += group_sum / GROUP_GRID_POINTS as f64; - scored_groups += 1; - } - + let (groups, max_err) = rank_errors(truth, probes, answers); + let scored_groups = groups.len(); let mut metrics: BTreeMap = BTreeMap::new(); metrics.insert( "mean_rank_err".into(), if scored_groups > 0 { - sum_mean / scored_groups as f64 + groups.iter().map(|g| g.error).sum::() / scored_groups as f64 } else { 0.0 }, @@ -134,8 +111,63 @@ impl GroundTruth for SubpopRankErrorGT { metrics.insert("probes".into(), scored_groups as f64); metrics.insert("subpopulations".into(), truth.per_group.len() as f64); metrics.insert("group_columns".into(), self.group_columns.len() as f64); + write_groups( + &groups, + truth.items as u64, + truth.schema_width, + &mut metrics, + ); metrics } + + /// A group's error is its mean rank error over the grid, the number + /// `mean_rank_err` averages over groups. + fn per_group( + &self, + truth: &SubpopRankTruth, + probes: &[(GroupKey, f64)], + answers: &[f64], + ) -> Vec { + rank_errors(truth, probes, answers).0 + } +} + +/// Each group's mean rank error over its grid, plus the largest rank error any +/// single probe saw. +fn rank_errors( + truth: &SubpopRankTruth, + probes: &[(GroupKey, f64)], + answers: &[f64], +) -> (Vec, f64) { + let mut max_err = 0.0f64; + let mut groups = Vec::new(); + + // The probe set is one contiguous grid per group, in order. + for (chunk_p, chunk_a) in probes + .chunks(GROUP_GRID_POINTS) + .zip(answers.chunks(GROUP_GRID_POINTS)) + { + let Some((group, _)) = chunk_p.first() else { + continue; + }; + let Some(sorted) = truth.per_group.get(group) else { + continue; + }; + let mut group_sum = 0.0f64; + for ((_, q), est) in chunk_p.iter().zip(chunk_a) { + let err = rank_err(sorted, *est, *q); + group_sum += err; + if err > max_err { + max_err = err; + } + } + groups.push(( + group.clone(), + sorted.len() as u64, + group_sum / GROUP_GRID_POINTS as f64, + )); + } + (group_errors(groups.into_iter()), max_err) } #[cfg(test)] @@ -154,11 +186,11 @@ mod tests { .expect("the table has the named columns"); assert_eq!(truth.items, 6); assert_eq!( - truth.per_group[&vec!["a".to_string()]], + truth.per_group[&vec![Some("a".to_string())]], vec![10.0, 10.0, 20.0] ); assert_eq!( - truth.per_group[&vec!["b".to_string()]], + truth.per_group[&vec![Some("b".to_string())]], vec![30.0, 30.0, 30.0] ); } diff --git a/configs/datagen/hydra_columns.yaml b/configs/datagen/hydra_columns.yaml index 50cb4b5c..88b098f0 100644 --- a/configs/datagen/hydra_columns.yaml +++ b/configs/datagen/hydra_columns.yaml @@ -40,7 +40,8 @@ column_label: [key1, key2, value] row_num: 200000 column_spec: - # key1: the column the comparator groups by. + # key1: the column the comparator groups by unless `--group-columns` names + # another subset (`1`, `0,1`). - data_type: string distribution: kind: uniform diff --git a/configs/datagen/hydra_columns_f64.yaml b/configs/datagen/hydra_columns_f64.yaml index ac22d2a6..15b54921 100644 --- a/configs/datagen/hydra_columns_f64.yaml +++ b/configs/datagen/hydra_columns_f64.yaml @@ -21,7 +21,8 @@ column_label: [key1, key2, value] row_num: 200000 column_spec: - # key1: the column the comparator groups by. + # key1: the column the comparator groups by unless `--group-columns` names + # another subset (`1`, `0,1`). - data_type: string distribution: kind: uniform diff --git a/configs/datagen/hydra_columns_string.yaml b/configs/datagen/hydra_columns_string.yaml index 9997da4c..132e5c2c 100644 --- a/configs/datagen/hydra_columns_string.yaml +++ b/configs/datagen/hydra_columns_string.yaml @@ -23,7 +23,8 @@ column_label: [key1, key2, value] row_num: 200000 column_spec: - # key1: the column the comparator groups by. + # key1: the column the comparator groups by unless `--group-columns` names + # another subset (`1`, `0,1`). - data_type: string distribution: kind: uniform diff --git a/configs/datagen/hydra_columns_u64.yaml b/configs/datagen/hydra_columns_u64.yaml index 11a04211..8e6b57d8 100644 --- a/configs/datagen/hydra_columns_u64.yaml +++ b/configs/datagen/hydra_columns_u64.yaml @@ -21,7 +21,8 @@ column_label: [key1, key2, value] row_num: 200000 column_spec: - # key1: the column the comparator groups by. + # key1: the column the comparator groups by unless `--group-columns` names + # another subset (`1`, `0,1`). - data_type: string distribution: kind: uniform diff --git a/configs/datagen/hydra_kll_disjoint_labels.yaml b/configs/datagen/hydra_kll_disjoint_labels.yaml index cff3f56d..b98a7559 100644 --- a/configs/datagen/hydra_kll_disjoint_labels.yaml +++ b/configs/datagen/hydra_kll_disjoint_labels.yaml @@ -10,7 +10,8 @@ column_label: [key1, key2, value] row_num: 200000 column_spec: - # key1: the column the comparator groups by. + # key1: the column the comparator groups by unless `--group-columns` names + # another subset (`1`, `0,1`). - data_type: string distribution: kind: uniform diff --git a/configs/datagen/hydra_univmon_columns.yaml b/configs/datagen/hydra_univmon_columns.yaml index ee4d1356..adad591e 100644 --- a/configs/datagen/hydra_univmon_columns.yaml +++ b/configs/datagen/hydra_univmon_columns.yaml @@ -25,7 +25,8 @@ column_label: [key1, key2, value] row_num: 20000 column_spec: - # key1: the column the comparator groups by. + # key1: the column the comparator groups by unless `--group-columns` names + # another subset (`1`, `0,1`). - data_type: string distribution: kind: uniform diff --git a/sketch-bench/src/request.rs b/sketch-bench/src/request.rs index 1a13d182..421990a7 100644 --- a/sketch-bench/src/request.rs +++ b/sketch-bench/src/request.rs @@ -118,6 +118,10 @@ pub struct Requirement { pub workers: usize, /// How many shards a merge measurement folds pub merge_shards: usize, + /// The label columns a grouped (`hydra-*`) row asks and scores, ascending + pub group_columns: Vec, + /// Where a grouped row writes each scored group's error, if anywhere + pub per_group_out: Option, /// A named comparator, or `None` for the row's default. pub comparator: Option, /// Measured runs, and the warm-ups run and discarded before them diff --git a/sketch-bench/src/wrappers/hydra_cms/mod.rs b/sketch-bench/src/wrappers/hydra_cms/mod.rs index 6a01f62c..a3eac34b 100644 --- a/sketch-bench/src/wrappers/hydra_cms/mod.rs +++ b/sketch-bench/src/wrappers/hydra_cms/mod.rs @@ -60,11 +60,14 @@ mod tests { fed(&mut h, &r); } // (a, 10) occurs twice, under two different second labels. - assert_eq!(h.estimate_subpop_frequency(&["a"], &10i64), 2.0); - assert_eq!(h.estimate_subpop_frequency(&["a"], &20i64), 1.0); - assert_eq!(h.estimate_subpop_frequency(&["b"], &30i64), 1.0); + assert_eq!(h.estimate_subpop_frequency(&[Some("a")], &10i64), 2.0); + assert_eq!(h.estimate_subpop_frequency(&[Some("a")], &20i64), 1.0); + assert_eq!(h.estimate_subpop_frequency(&[Some("b")], &30i64), 1.0); // The full key is its own subpopulation and is stored too. - assert_eq!(h.estimate_subpop_frequency(&["a", "x"], &10i64), 1.0); + assert_eq!( + h.estimate_subpop_frequency(&[Some("a"), Some("x")], &10i64), + 1.0 + ); } /// A group that never occurred must estimate zero. This is the failure mode @@ -74,7 +77,7 @@ mod tests { fn an_absent_group_estimates_zero() { let mut h = built(); fed(&mut h, &record("a;x", 10)); - assert_eq!(h.estimate_subpop_frequency(&["zzz"], &10i64), 0.0); + assert_eq!(h.estimate_subpop_frequency(&[Some("zzz")], &10i64), 0.0); } /// Count-Min is linear and the grid is cell-wise, so folding two shards is @@ -91,7 +94,7 @@ mod tests { left.inner .merge(&right.inner) .expect("both operands built from one ParamSet, so shapes match"); - assert_eq!(left.estimate_subpop_frequency(&["a"], &10i64), 7.0); + assert_eq!(left.estimate_subpop_frequency(&[Some("a")], &10i64), 7.0); } /// The grid's key columns are fixed at construction: a record of another @@ -126,8 +129,8 @@ mod tests { fn an_empty_label_keeps_its_column() { let mut h = built(); fed(&mut h, &record(";x", 10)); - assert_eq!(h.estimate_subpop_frequency(&[""], &10i64), 1.0); - assert_eq!(h.estimate_subpop_frequency(&["x"], &10i64), 0.0); + assert_eq!(h.estimate_subpop_frequency(&[Some("")], &10i64), 1.0); + assert_eq!(h.estimate_subpop_frequency(&[Some("x")], &10i64), 0.0); } #[test] diff --git a/sketch-bench/src/wrappers/hydra_cms/polars.rs b/sketch-bench/src/wrappers/hydra_cms/polars.rs index 6bcacd40..706143dc 100644 --- a/sketch-bench/src/wrappers/hydra_cms/polars.rs +++ b/sketch-bench/src/wrappers/hydra_cms/polars.rs @@ -33,7 +33,7 @@ pub fn build_polars_subpop_frequency_cms( } impl PolarsSubpopFrequencyCms { - pub fn estimate_subpop_frequency(&self, labels: &[&str], value: &T) -> f64 { + pub fn estimate_subpop_frequency(&self, labels: &[Option<&str>], value: &T) -> f64 { self.0.query(labels, value) } } @@ -92,7 +92,7 @@ pub fn insert_step_polars_subpop_frequency_cms( pub fn query_polars_subpop_frequency_cms( params: &ParamSet, items: Rc>, - probes: Rc, T)>>, + probes: Rc>, T)>>, passes: usize, ) -> Result>, BuildError> { let mut out = Vec::with_capacity(passes); diff --git a/sketch-bench/src/wrappers/hydra_cms/sketchlib.rs b/sketch-bench/src/wrappers/hydra_cms/sketchlib.rs index dc3a629d..243ed3e4 100644 --- a/sketch-bench/src/wrappers/hydra_cms/sketchlib.rs +++ b/sketch-bench/src/wrappers/hydra_cms/sketchlib.rs @@ -39,7 +39,11 @@ pub fn build_hydra_cms(config: &ParamSet, label_columns: usize) -> Result(&self, labels: &[&str], value: &V) -> f64 { + pub fn estimate_subpop_frequency( + &self, + labels: &[Option<&str>], + value: &V, + ) -> f64 { query( &self.inner, labels, @@ -106,7 +110,7 @@ pub fn insert_step_hydra_cms( pub fn query_hydra_cms( params: &ParamSet, items: Rc>, - probes: Rc, V)>>, + probes: Rc>, V)>>, passes: usize, ) -> Result>, BuildError> { merge_query_hydra_cms(params, items, probes, 1, passes) @@ -117,7 +121,7 @@ pub fn query_hydra_cms( pub fn merge_query_hydra_cms( params: &ParamSet, items: Rc>, - probes: Rc, V)>>, + probes: Rc>, V)>>, shards: usize, passes: usize, ) -> Result>, BuildError> { diff --git a/sketch-bench/src/wrappers/hydra_cs/mod.rs b/sketch-bench/src/wrappers/hydra_cs/mod.rs index 6d49033b..270e903b 100644 --- a/sketch-bench/src/wrappers/hydra_cs/mod.rs +++ b/sketch-bench/src/wrappers/hydra_cs/mod.rs @@ -60,11 +60,14 @@ mod tests { fed(&mut h, &r); } // (a, 10) occurs twice, under two different second labels. - assert_eq!(h.estimate_subpop_frequency(&["a"], &10i64), 2.0); - assert_eq!(h.estimate_subpop_frequency(&["a"], &20i64), 1.0); - assert_eq!(h.estimate_subpop_frequency(&["b"], &30i64), 1.0); + assert_eq!(h.estimate_subpop_frequency(&[Some("a")], &10i64), 2.0); + assert_eq!(h.estimate_subpop_frequency(&[Some("a")], &20i64), 1.0); + assert_eq!(h.estimate_subpop_frequency(&[Some("b")], &30i64), 1.0); // The full key is its own subpopulation and is stored too. - assert_eq!(h.estimate_subpop_frequency(&["a", "x"], &10i64), 1.0); + assert_eq!( + h.estimate_subpop_frequency(&[Some("a"), Some("x")], &10i64), + 1.0 + ); } /// A group that never occurred must estimate zero. This is the failure mode @@ -74,7 +77,7 @@ mod tests { fn an_absent_group_estimates_zero() { let mut h = built(); fed(&mut h, &record("a;x", 10)); - assert_eq!(h.estimate_subpop_frequency(&["zzz"], &10i64), 0.0); + assert_eq!(h.estimate_subpop_frequency(&[Some("zzz")], &10i64), 0.0); } /// Count Sketch is linear and the grid is cell-wise, so folding two shards is @@ -91,7 +94,7 @@ mod tests { left.inner .merge(&right.inner) .expect("both operands built from one ParamSet, so shapes match"); - assert_eq!(left.estimate_subpop_frequency(&["a"], &10i64), 7.0); + assert_eq!(left.estimate_subpop_frequency(&[Some("a")], &10i64), 7.0); } #[test] diff --git a/sketch-bench/src/wrappers/hydra_cs/polars.rs b/sketch-bench/src/wrappers/hydra_cs/polars.rs index 73e0ad8b..adab3d57 100644 --- a/sketch-bench/src/wrappers/hydra_cs/polars.rs +++ b/sketch-bench/src/wrappers/hydra_cs/polars.rs @@ -33,7 +33,7 @@ pub fn build_polars_subpop_frequency_cs( } impl PolarsSubpopFrequencyCs { - pub fn estimate_subpop_frequency(&self, labels: &[&str], value: &T) -> f64 { + pub fn estimate_subpop_frequency(&self, labels: &[Option<&str>], value: &T) -> f64 { self.0.query(labels, value) } } @@ -91,7 +91,7 @@ pub fn insert_step_polars_subpop_frequency_cs( pub fn query_polars_subpop_frequency_cs( params: &ParamSet, items: Rc>, - probes: Rc, T)>>, + probes: Rc>, T)>>, passes: usize, ) -> Result>, BuildError> { let mut out = Vec::with_capacity(passes); diff --git a/sketch-bench/src/wrappers/hydra_cs/sketchlib.rs b/sketch-bench/src/wrappers/hydra_cs/sketchlib.rs index 554a3808..9c326e78 100644 --- a/sketch-bench/src/wrappers/hydra_cs/sketchlib.rs +++ b/sketch-bench/src/wrappers/hydra_cs/sketchlib.rs @@ -39,7 +39,11 @@ pub fn build_hydra_cs(config: &ParamSet, label_columns: usize) -> Result(&self, labels: &[&str], value: &V) -> f64 { + pub fn estimate_subpop_frequency( + &self, + labels: &[Option<&str>], + value: &V, + ) -> f64 { query( &self.inner, labels, @@ -106,7 +110,7 @@ pub fn insert_step_hydra_cs( pub fn query_hydra_cs( params: &ParamSet, items: Rc>, - probes: Rc, V)>>, + probes: Rc>, V)>>, passes: usize, ) -> Result>, BuildError> { merge_query_hydra_cs(params, items, probes, 1, passes) @@ -117,7 +121,7 @@ pub fn query_hydra_cs( pub fn merge_query_hydra_cs( params: &ParamSet, items: Rc>, - probes: Rc, V)>>, + probes: Rc>, V)>>, shards: usize, passes: usize, ) -> Result>, BuildError> { diff --git a/sketch-bench/src/wrappers/hydra_hll/mod.rs b/sketch-bench/src/wrappers/hydra_hll/mod.rs index 52ef926f..29efabf5 100644 --- a/sketch-bench/src/wrappers/hydra_hll/mod.rs +++ b/sketch-bench/src/wrappers/hydra_hll/mod.rs @@ -58,7 +58,7 @@ mod tests { } fed_hll(&mut h, &record("a;y", 20)); // Two distinct values under label `a`, seen four times. - let est = h.estimate_subpop_cardinality(&["a"]); + let est = h.estimate_subpop_cardinality(&[Some("a")]); assert!( (est - 2.0).abs() < 0.5, "two distinct values under `a`, estimated {est}" @@ -69,7 +69,7 @@ mod tests { fn hll_absent_group_estimates_zero() { let mut h = built_hll(); fed_hll(&mut h, &record("a;x", 10)); - assert_eq!(h.estimate_subpop_cardinality(&["zzz"]), 0.0); + assert_eq!(h.estimate_subpop_cardinality(&[Some("zzz")]), 0.0); } /// A HyperLogLog is register-wise mergeable, so two shards fold without @@ -84,7 +84,7 @@ mod tests { left.inner .merge(&right.inner) .expect("both operands built from one ParamSet, so shapes match"); - let est = left.estimate_subpop_cardinality(&["a"]); + let est = left.estimate_subpop_cardinality(&[Some("a")]); assert!( (est - 3.0).abs() < 0.5, "three distinct values across both shards, estimated {est}" diff --git a/sketch-bench/src/wrappers/hydra_hll/polars.rs b/sketch-bench/src/wrappers/hydra_hll/polars.rs index 272b99c1..822445b0 100644 --- a/sketch-bench/src/wrappers/hydra_hll/polars.rs +++ b/sketch-bench/src/wrappers/hydra_hll/polars.rs @@ -34,7 +34,7 @@ pub fn build_polars_subpop_cardinality( } impl PolarsSubpopCardinality { - pub fn estimate_subpop_cardinality(&self, labels: &[&str]) -> f64 { + pub fn estimate_subpop_cardinality(&self, labels: &[Option<&str>]) -> f64 { self.0.query(labels) } } @@ -92,7 +92,7 @@ pub fn insert_step_polars_subpop_cardinality( pub fn query_polars_subpop_cardinality( params: &ParamSet, items: Rc>, - probes: Rc>>, + probes: Rc>>>, passes: usize, ) -> Result>, BuildError> { let mut out = Vec::with_capacity(passes); diff --git a/sketch-bench/src/wrappers/hydra_hll/sketchlib.rs b/sketch-bench/src/wrappers/hydra_hll/sketchlib.rs index fc4815c4..bbd303a6 100644 --- a/sketch-bench/src/wrappers/hydra_hll/sketchlib.rs +++ b/sketch-bench/src/wrappers/hydra_hll/sketchlib.rs @@ -35,7 +35,7 @@ pub fn build_hydra_hll(config: &ParamSet, label_columns: usize) -> Result f64 { + pub fn estimate_subpop_cardinality(&self, labels: &[Option<&str>]) -> f64 { query(&self.inner, labels, &HydraQuery::Cardinality, "hydra-hll") } } @@ -96,7 +96,7 @@ pub fn insert_step_hydra_hll( pub fn query_hydra_hll( params: &ParamSet, items: Rc>, - probes: Rc>>, + probes: Rc>>>, passes: usize, ) -> Result>, BuildError> { merge_query_hydra_hll(params, items, probes, 1, passes) @@ -107,7 +107,7 @@ pub fn query_hydra_hll( pub fn merge_query_hydra_hll( params: &ParamSet, items: Rc>, - probes: Rc>>, + probes: Rc>>>, shards: usize, passes: usize, ) -> Result>, BuildError> { diff --git a/sketch-bench/src/wrappers/hydra_kll/mod.rs b/sketch-bench/src/wrappers/hydra_kll/mod.rs index e6758577..de6f74f6 100644 --- a/sketch-bench/src/wrappers/hydra_kll/mod.rs +++ b/sketch-bench/src/wrappers/hydra_kll/mod.rs @@ -64,7 +64,7 @@ mod tests { for _ in 0..500 { fed_kll(&mut h, &frecord("b;x", 10_000.0)); } - let median = h.estimate_subpop_quantile(&["a"], 0.5); + let median = h.estimate_subpop_quantile(&[Some("a")], 0.5); assert!( (1.0..=101.0).contains(&median), "group `a` spans 1..=101, median estimated {median}" @@ -80,8 +80,8 @@ mod tests { for v in 1..=101 { fed_kll(&mut h, &frecord("a;x", v as f64)); } - assert_eq!(h.estimate_subpop_quantile(&["a"], 0.0), 1.0); - assert_eq!(h.estimate_subpop_quantile(&["a"], 1.0), 101.0); + assert_eq!(h.estimate_subpop_quantile(&[Some("a")], 0.0), 1.0); + assert_eq!(h.estimate_subpop_quantile(&[Some("a")], 1.0), 101.0); } /// The library allocates a cell's retained slots once, so the footprint is diff --git a/sketch-bench/src/wrappers/hydra_kll/polars.rs b/sketch-bench/src/wrappers/hydra_kll/polars.rs index 44ecb2f4..11a6a268 100644 --- a/sketch-bench/src/wrappers/hydra_kll/polars.rs +++ b/sketch-bench/src/wrappers/hydra_kll/polars.rs @@ -46,8 +46,8 @@ impl PolarsSubpopQuantile { /// `floor(phi * n)`, clamped to the last index. That is the index whose /// rank interval contains `phi * n`, which is what the rank-error /// comparator scores against, so an exact answer scores zero. - pub fn estimate_subpop_quantile(&self, labels: &[&str], phi: f64) -> f64 { - let Some(values) = self.sorted.get(&prefix_key(labels)) else { + pub fn estimate_subpop_quantile(&self, labels: &[Option<&str>], phi: f64) -> f64 { + let Some(values) = self.sorted.get(&group_key(labels)) else { return f64::NAN; }; if values.is_empty() { @@ -157,7 +157,7 @@ pub fn insert_step_polars_subpop_quantile( params: &ParamSet, items: Rc>, - probes: Rc, f64)>>, + probes: Rc>, f64)>>, passes: usize, ) -> Result>, BuildError> { let mut out = Vec::with_capacity(passes); diff --git a/sketch-bench/src/wrappers/hydra_kll/sketchlib.rs b/sketch-bench/src/wrappers/hydra_kll/sketchlib.rs index 672de091..7f0ea176 100644 --- a/sketch-bench/src/wrappers/hydra_kll/sketchlib.rs +++ b/sketch-bench/src/wrappers/hydra_kll/sketchlib.rs @@ -46,7 +46,7 @@ impl HydraKll { /// at a rank, which is what rank error is defined over. The `Cdf` variant /// answers the inverse question. #[inline] - pub fn estimate_subpop_quantile(&self, labels: &[&str], phi: f64) -> f64 { + pub fn estimate_subpop_quantile(&self, labels: &[Option<&str>], phi: f64) -> f64 { query(&self.inner, labels, &HydraQuery::Quantile(phi), "hydra-kll") } } @@ -112,7 +112,7 @@ pub fn insert_step_hydra_kll( pub fn query_hydra_kll( params: &ParamSet, items: Rc>, - probes: Rc, f64)>>, + probes: Rc>, f64)>>, passes: usize, ) -> Result>, BuildError> { merge_query_hydra_kll(params, items, probes, 1, passes) @@ -123,7 +123,7 @@ pub fn query_hydra_kll( pub fn merge_query_hydra_kll( params: &ParamSet, items: Rc>, - probes: Rc, f64)>>, + probes: Rc>, f64)>>, shards: usize, passes: usize, ) -> Result>, BuildError> { diff --git a/sketch-bench/src/wrappers/hydra_shared.rs b/sketch-bench/src/wrappers/hydra_shared.rs index 0521e338..1af9d7ef 100644 --- a/sketch-bench/src/wrappers/hydra_shared.rs +++ b/sketch-bench/src/wrappers/hydra_shared.rs @@ -6,8 +6,8 @@ use crate::wrappers::BuildError; use asap_sketchlib::input::{HydraCounter, HydraQuery}; use asap_sketchlib::{DataInput, Hydra}; -pub(crate) fn labels(group: &[String]) -> Vec<&str> { - group.iter().map(String::as_str).collect() +pub(crate) fn labels(group: &[Option]) -> Vec> { + group.iter().map(Option::as_deref).collect() } /// The label columns of a `;`-joined record stream, which every record must @@ -54,10 +54,11 @@ pub(crate) fn update(h: &mut Hydra, key: &str, value: &DataInput, variant: &str) } } -/// Asks `query` of the subpopulation `group` constrains: its values are the -/// leading key columns' (the scored subpopulations group by column 0), the -/// rest unconstrained. A probe wider than the grid is refused. -pub(crate) fn query(h: &Hydra, group: &[&str], query: &HydraQuery, variant: &str) -> f64 { +/// Asks `query` of the subpopulation `group` constrains: a value at each +/// grouped column, `None` (unconstrained) at the rest and past its end, so any +/// non-empty subset of the key columns can be asked. A probe wider than the +/// grid is refused. +pub(crate) fn query(h: &Hydra, group: &[Option<&str>], query: &HydraQuery, variant: &str) -> f64 { let width = h.schema().len(); if group.len() > width { panic!( @@ -65,7 +66,9 @@ pub(crate) fn query(h: &Hydra, group: &[&str], query: &HydraQuery, variant: &str group.len() ); } - let key: Vec> = (0..width).map(|i| group.get(i).copied()).collect(); + let key: Vec> = (0..width) + .map(|i| group.get(i).copied().flatten()) + .collect(); h.query_key(&key, query) .unwrap_or_else(|e| panic!("{variant}: probe {group:?}: {e}")) } diff --git a/sketch-bench/src/wrappers/hydra_univmon/mod.rs b/sketch-bench/src/wrappers/hydra_univmon/mod.rs index 181375ee..ab6073c8 100644 --- a/sketch-bench/src/wrappers/hydra_univmon/mod.rs +++ b/sketch-bench/src/wrappers/hydra_univmon/mod.rs @@ -59,7 +59,7 @@ mod tests { fed(&mut h, &record("a;x", 10)); } fed(&mut h, &record("a;y", 20)); - let est = h.estimate_subpop_l1_norm(&["a"]); + let est = h.estimate_subpop_l1_norm(&[Some("a")]); assert!( (est - 4.0).abs() < 0.5, "four records under `a`, estimated {est}" @@ -73,7 +73,7 @@ mod tests { fed(&mut h, &record("a;x", 10)); } fed(&mut h, &record("a;y", 20)); - let est = h.estimate_subpop_cardinality(&["a"]); + let est = h.estimate_subpop_cardinality(&[Some("a")]); assert!( (est - 2.0).abs() < 0.5, "two distinct values under `a`, estimated {est}" @@ -86,7 +86,7 @@ mod tests { for _ in 0..8 { fed(&mut h, &record("a;x", 10)); } - let est = h.estimate_subpop_entropy(&["a"]); + let est = h.estimate_subpop_entropy(&[Some("a")]); assert!( est.abs() < 0.1, "one value, so no uncertainty, estimated {est}" @@ -99,8 +99,8 @@ mod tests { for _ in 0..4 { fed(&mut h, &record("a;x", 10)); } - let l1 = h.estimate_subpop_l1_norm(&["a"]); - let l2 = h.estimate_subpop_l2_norm(&["a"]); + let l1 = h.estimate_subpop_l1_norm(&[Some("a")]); + let l2 = h.estimate_subpop_l2_norm(&[Some("a")]); assert!((l1 - l2).abs() < 0.5, "l1 {l1}, l2 {l2}"); } @@ -116,7 +116,7 @@ mod tests { left.inner .merge(&right.inner) .expect("both operands built from one ParamSet, so shapes match"); - let est = left.estimate_subpop_l1_norm(&["a"]); + let est = left.estimate_subpop_l1_norm(&[Some("a")]); assert!( (est - 7.0).abs() < 0.5, "seven records in all, estimated {est}" @@ -174,14 +174,17 @@ mod baseline_tests { type Asked = fn( &ParamSet, Rc>, - Rc>>, + Rc>>>, usize, ) -> Result>, crate::wrappers::BuildError>; fn answers_for(query: Asked) -> Vec { let params = ParamSet::of(&HydraUnivmonParams::canonical()); - let probes = Rc::new(vec![vec!["a".to_string()], vec!["b".to_string()]]); + let probes = Rc::new(vec![ + vec![Some("a".to_string())], + vec![Some("b".to_string())], + ]); let mut passes = query(¶ms, stream(), probes, 1).expect("baseline builds"); let pass = passes.pop().expect("one pass"); pass().0 diff --git a/sketch-bench/src/wrappers/hydra_univmon/polars.rs b/sketch-bench/src/wrappers/hydra_univmon/polars.rs index eb71f9f1..0e9dd152 100644 --- a/sketch-bench/src/wrappers/hydra_univmon/polars.rs +++ b/sketch-bench/src/wrappers/hydra_univmon/polars.rs @@ -39,7 +39,7 @@ impl Default for PolarsSubpopCardinalityUnivmon { } impl PolarsSubpopCardinalityUnivmon { - pub fn estimate_subpop_cardinality(&self, labels: &[&str]) -> f64 { + pub fn estimate_subpop_cardinality(&self, labels: &[Option<&str>]) -> f64 { self.0.query(labels) } } @@ -59,7 +59,7 @@ fn build_polars_subpop_vector( } impl PolarsSubpopUnivmon { - pub fn estimate_subpop_statistic(&self, labels: &[&str]) -> f64 { + pub fn estimate_subpop_statistic(&self, labels: &[Option<&str>]) -> f64 { self.0.query(labels) } } @@ -116,7 +116,7 @@ pub fn insert_step_polars_subpop_cardinality_univmon( pub fn query_polars_subpop_cardinality_univmon( params: &ParamSet, items: Rc>, - probes: Rc>>, + probes: Rc>>>, passes: usize, ) -> Result>, BuildError> { let mut out = Vec::with_capacity(passes); @@ -209,7 +209,7 @@ fn insert_step_polars_subpop_vector( fn query_polars_subpop_vector( params: &ParamSet, items: Rc>, - probes: Rc>>, + probes: Rc>>>, passes: usize, fold: fn(&[u64]) -> f64, ) -> Result>, BuildError> { @@ -274,7 +274,7 @@ pub fn insert_step_polars_subpop_l1_norm( pub fn query_polars_subpop_l1_norm( params: &ParamSet, items: Rc>, - probes: Rc>>, + probes: Rc>>>, passes: usize, ) -> Result>, BuildError> { query_polars_subpop_vector(params, items, probes, passes, l1_norm) @@ -307,7 +307,7 @@ pub fn insert_step_polars_subpop_l2_norm( pub fn query_polars_subpop_l2_norm( params: &ParamSet, items: Rc>, - probes: Rc>>, + probes: Rc>>>, passes: usize, ) -> Result>, BuildError> { query_polars_subpop_vector(params, items, probes, passes, l2_norm) @@ -340,7 +340,7 @@ pub fn insert_step_polars_subpop_entropy( pub fn query_polars_subpop_entropy( params: &ParamSet, items: Rc>, - probes: Rc>>, + probes: Rc>>>, passes: usize, ) -> Result>, BuildError> { query_polars_subpop_vector(params, items, probes, passes, entropy) diff --git a/sketch-bench/src/wrappers/hydra_univmon/sketchlib.rs b/sketch-bench/src/wrappers/hydra_univmon/sketchlib.rs index 11c689fa..e7465088 100644 --- a/sketch-bench/src/wrappers/hydra_univmon/sketchlib.rs +++ b/sketch-bench/src/wrappers/hydra_univmon/sketchlib.rs @@ -49,7 +49,7 @@ pub fn build_hydra_univmon( impl HydraUnivmon { #[inline] - pub fn estimate_subpop_cardinality(&self, labels: &[&str]) -> f64 { + pub fn estimate_subpop_cardinality(&self, labels: &[Option<&str>]) -> f64 { query( &self.inner, labels, @@ -59,17 +59,17 @@ impl HydraUnivmon { } #[inline] - pub fn estimate_subpop_l1_norm(&self, labels: &[&str]) -> f64 { + pub fn estimate_subpop_l1_norm(&self, labels: &[Option<&str>]) -> f64 { query(&self.inner, labels, &HydraQuery::L1Norm, "hydra-univmon") } #[inline] - pub fn estimate_subpop_l2_norm(&self, labels: &[&str]) -> f64 { + pub fn estimate_subpop_l2_norm(&self, labels: &[Option<&str>]) -> f64 { query(&self.inner, labels, &HydraQuery::L2Norm, "hydra-univmon") } #[inline] - pub fn estimate_subpop_entropy(&self, labels: &[&str]) -> f64 { + pub fn estimate_subpop_entropy(&self, labels: &[Option<&str>]) -> f64 { query(&self.inner, labels, &HydraQuery::Entropy, "hydra-univmon") } } @@ -131,10 +131,10 @@ pub fn insert_step_hydra_univmon( fn asked_hydra_univmon( params: &ParamSet, items: Rc>, - probes: Rc>>, + probes: Rc>>>, shards: usize, passes: usize, - estimate: fn(&HydraUnivmon, &[&str]) -> f64, + estimate: fn(&HydraUnivmon, &[Option<&str>]) -> f64, ) -> Result>, BuildError> { let mut out = Vec::with_capacity(passes); for _ in 0..passes { @@ -162,7 +162,7 @@ fn asked_hydra_univmon( pub fn query_hydra_univmon_cardinality( params: &ParamSet, items: Rc>, - probes: Rc>>, + probes: Rc>>>, passes: usize, ) -> Result>, BuildError> { asked_hydra_univmon( @@ -180,7 +180,7 @@ pub fn query_hydra_univmon_cardinality( pub fn merge_query_hydra_univmon_cardinality( params: &ParamSet, items: Rc>, - probes: Rc>>, + probes: Rc>>>, shards: usize, passes: usize, ) -> Result>, BuildError> { @@ -197,7 +197,7 @@ pub fn merge_query_hydra_univmon_cardinality( pub fn query_hydra_univmon_l1_norm( params: &ParamSet, items: Rc>, - probes: Rc>>, + probes: Rc>>>, passes: usize, ) -> Result>, BuildError> { asked_hydra_univmon( @@ -215,7 +215,7 @@ pub fn query_hydra_univmon_l1_norm( pub fn merge_query_hydra_univmon_l1_norm( params: &ParamSet, items: Rc>, - probes: Rc>>, + probes: Rc>>>, shards: usize, passes: usize, ) -> Result>, BuildError> { @@ -232,7 +232,7 @@ pub fn merge_query_hydra_univmon_l1_norm( pub fn query_hydra_univmon_l2_norm( params: &ParamSet, items: Rc>, - probes: Rc>>, + probes: Rc>>>, passes: usize, ) -> Result>, BuildError> { asked_hydra_univmon( @@ -250,7 +250,7 @@ pub fn query_hydra_univmon_l2_norm( pub fn merge_query_hydra_univmon_l2_norm( params: &ParamSet, items: Rc>, - probes: Rc>>, + probes: Rc>>>, shards: usize, passes: usize, ) -> Result>, BuildError> { @@ -267,7 +267,7 @@ pub fn merge_query_hydra_univmon_l2_norm( pub fn query_hydra_univmon_entropy( params: &ParamSet, items: Rc>, - probes: Rc>>, + probes: Rc>>>, passes: usize, ) -> Result>, BuildError> { asked_hydra_univmon( @@ -285,7 +285,7 @@ pub fn query_hydra_univmon_entropy( pub fn merge_query_hydra_univmon_entropy( params: &ParamSet, items: Rc>, - probes: Rc>>, + probes: Rc>>>, shards: usize, passes: usize, ) -> Result>, BuildError> { diff --git a/sketch-bench/src/wrappers/polars_shared.rs b/sketch-bench/src/wrappers/polars_shared.rs index d049a1b9..7e9e3c52 100644 --- a/sketch-bench/src/wrappers/polars_shared.rs +++ b/sketch-bench/src/wrappers/polars_shared.rs @@ -287,9 +287,17 @@ pub fn subset_key(parts: &[&str], mask: usize) -> String { out } -/// The key a probe of the leading label columns `labels` reads. -pub fn prefix_key(labels: &[&str]) -> String { - subset_key(labels, (1 << labels.len()) - 1) +/// The key a probe reads: `labels` holds a value at each grouped column and +/// `None` at the rest, so any subset of the columns is a group, not only a +/// prefix of them. +pub fn group_key(labels: &[Option<&str>]) -> String { + let parts: Vec<&str> = labels.iter().map(|l| l.unwrap_or("")).collect(); + let mask = labels + .iter() + .enumerate() + .filter(|(_, l)| l.is_some()) + .fold(0, |mask, (j, _)| mask | (1 << j)); + subset_key(&parts, mask) } /// Expand one record into `(subset_key, value)` rows, one per non-empty subset @@ -360,9 +368,9 @@ impl PolarsSubpopFrequencyCore { } } - pub fn query(&self, labels: &[&str], value: &T) -> f64 { + pub fn query(&self, labels: &[Option<&str>], value: &T) -> f64 { self.counts - .get(&(prefix_key(labels), value.count_key())) + .get(&(group_key(labels), value.count_key())) .copied() .unwrap_or(0) as f64 } @@ -426,8 +434,8 @@ impl PolarsSubpopCardinalityCore { } } - pub fn query(&self, labels: &[&str]) -> f64 { - self.distinct.get(&prefix_key(labels)).copied().unwrap_or(0) as f64 + pub fn query(&self, labels: &[Option<&str>]) -> f64 { + self.distinct.get(&group_key(labels)).copied().unwrap_or(0) as f64 } pub fn memory_bytes(&self) -> usize { @@ -495,8 +503,8 @@ impl PolarsSubpopVectorCore { .collect(); } - pub fn query(&self, labels: &[&str]) -> f64 { - self.exact.get(&prefix_key(labels)).copied().unwrap_or(0.0) + pub fn query(&self, labels: &[Option<&str>]) -> f64 { + self.exact.get(&group_key(labels)).copied().unwrap_or(0.0) } pub fn memory_bytes(&self) -> usize { @@ -542,7 +550,8 @@ mod tests { fn subset_keys_tag_each_value_with_its_column() { assert_eq!(subset_key(&["a", "b", "c"], 0b101), "label0:a;label2:c"); assert_eq!(subset_key(&["x:y;z", ""], 0b11), r"label0:x\:y\;z;label1:"); - assert_eq!(prefix_key(&["a", "b"]), "label0:a;label1:b"); + assert_eq!(group_key(&[Some("a"), Some("b")]), "label0:a;label1:b"); + assert_eq!(group_key(&[None, Some("b")]), "label1:b"); } /// #74: the column-0 group `a` holds 2 records (`a;a`, `a;b`). The old @@ -553,8 +562,11 @@ mod tests { core.update(&("a;a".to_string(), 7)); core.update(&("a;b".to_string(), 7)); core.finalize(); - assert_eq!(core.query(&["a"], &7), 2.0); - assert_eq!(core.query(&["a", "a"], &7), 1.0); - assert_eq!(core.query(&["b"], &7), 0.0); + assert_eq!(core.query(&[Some("a")], &7), 2.0); + assert_eq!(core.query(&[Some("a"), Some("a")], &7), 1.0); + assert_eq!(core.query(&[Some("b")], &7), 0.0); + // Column 1 alone: `a` once, `b` once, with column 0 unconstrained. + assert_eq!(core.query(&[None, Some("a")], &7), 1.0); + assert_eq!(core.query(&[None, Some("b")], &7), 1.0); } } From 4b40e31c995a708d0dbf0e935544f02b45cdfdc2 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 10 Oct 2026 04:18:40 +0000 Subject: [PATCH 06/18] datagen: whole uniform bounds for child_of; 2- and 3-label hierarchy cuts A uniform draw is continuous, so `uniform{0.0, 2.9}` with fan_out 2 drew child indices 0, 1 and 2; child_of now requires whole uniform bounds. hydra_hier_d2.yaml and hydra_hier_d3.yaml cut hydra_hier.yaml to its first 2 and 3 labels for the Hydra study's schema-width sweep. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FhcWJExZcmVqS6r6rEtjis --- aqpbm-datagen/src/column.rs | 10 +++++++ aqpbm-datagen/src/lib.rs | 16 ++++++++++ configs/datagen/hydra_hier_d2.yaml | 38 +++++++++++++++++++++++ configs/datagen/hydra_hier_d3.yaml | 48 ++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+) create mode 100644 configs/datagen/hydra_hier_d2.yaml create mode 100644 configs/datagen/hydra_hier_d3.yaml diff --git a/aqpbm-datagen/src/column.rs b/aqpbm-datagen/src/column.rs index 07e8899d..092bea87 100644 --- a/aqpbm-datagen/src/column.rs +++ b/aqpbm-datagen/src/column.rs @@ -134,6 +134,16 @@ impl ColumnSpec { if fan_out == 0 { return Err(DataGenError::BadParam("fan_out must be > 0".into())); } + // A uniform draw is continuous; only whole bounds keep its index in + // `[0, fan_out)` (`uniform{0.0, 2.9}` would draw 0, 1 and 2). + if let DataDistribution::Uniform(p) = &self.distribution { + if p.lower_bound.fract() != 0.0 || p.upper_bound.fract() != 0.0 { + return Err(DataGenError::BadParam(format!( + "child_of: uniform bounds must be whole numbers, got [{}, {})", + p.lower_bound, p.upper_bound + ))); + } + } match self.distribution.domain() { Some(domain) if domain.size == fan_out => {} Some(domain) => { diff --git a/aqpbm-datagen/src/lib.rs b/aqpbm-datagen/src/lib.rs index f5e2679d..e90ac559 100644 --- a/aqpbm-datagen/src/lib.rs +++ b/aqpbm-datagen/src/lib.rs @@ -610,6 +610,19 @@ mod tests { assert_eq!(children_by_parent(col(1), col(2), 25).len(), 100); } + /// The schema-width cuts keep the first labels and the value column. + #[test] + fn hydra_hier_cuts_keep_their_leading_labels() { + for (file, labels) in [("hydra_hier_d2.yaml", 2), ("hydra_hier_d3.yaml", 3)] { + let path = format!("../configs/datagen/{file}"); + let d = TableDescription::from_path(std::path::Path::new(&path)).unwrap(); + assert_eq!(d.column_num, labels + 1, "{file}"); + let t = d.generate().unwrap(); + let col = |i: usize| t.data[i].as_string().unwrap(); + assert_eq!(children_by_parent(col(0), col(1), 25).len(), 4, "{file}"); + } + } + #[test] fn bad_child_columns_are_refused_by_name() { type Edit = fn(&mut TableDescription); @@ -633,6 +646,9 @@ mod tests { ("has to be `string`", |d| { d.column_spec[1].data_type = "u64".into() }), + ("whole numbers", |d| { + d.column_spec[2].distribution = uniform(0.0, 5.9, 3) + }), ("cardinality", |d| d.column_spec[1].cardinality = Some(25)), ("`string:` block", |d| { d.column_spec[1].string = Some(StringOpts::default()) diff --git a/configs/datagen/hydra_hier_d2.yaml b/configs/datagen/hydra_hier_d2.yaml new file mode 100644 index 00000000..e0454cc6 --- /dev/null +++ b/configs/datagen/hydra_hier_d2.yaml @@ -0,0 +1,38 @@ +# `hydra_hier.yaml` cut to 2 labels (region, service), for the Hydra +# study's schema-width sweep: `2^2 - 1 = 3` cell insertions per record. +# Same columns, distributions and seeds as `hydra_hier.yaml`, which has 4 labels. + +column_num: 3 +column_label: [region, service, user_id] +row_num: 200000 + +column_spec: + # region: 4 values, equally likely. + - data_type: string + string: + min_len: 4 + max_len: 4 + distribution: + kind: uniform + lower_bound: 0.0 + upper_bound: 4.0 + seed: 1 + + # service: 25 per region, skewed, so a few services carry most traffic. + - data_type: string + child_of: 0 + fan_out: 25 + distribution: + kind: zipf + skewness: 1.1 + population_size: 25 + seed: 2 + + # user_id: the value. Count-Min counts its occurrences per group, + # HyperLogLog counts how many are distinct. + - data_type: i64 + distribution: + kind: zipf + skewness: 0.8 + population_size: 1000000 + seed: 5 diff --git a/configs/datagen/hydra_hier_d3.yaml b/configs/datagen/hydra_hier_d3.yaml new file mode 100644 index 00000000..b9257cb7 --- /dev/null +++ b/configs/datagen/hydra_hier_d3.yaml @@ -0,0 +1,48 @@ +# `hydra_hier.yaml` cut to 3 labels (region, service, endpoint), for the Hydra +# study's schema-width sweep: `2^3 - 1 = 7` cell insertions per record. +# Same columns, distributions and seeds as `hydra_hier.yaml`, which has 4 labels. + +column_num: 4 +column_label: [region, service, endpoint, user_id] +row_num: 200000 + +column_spec: + # region: 4 values, equally likely. + - data_type: string + string: + min_len: 4 + max_len: 4 + distribution: + kind: uniform + lower_bound: 0.0 + upper_bound: 4.0 + seed: 1 + + # service: 25 per region, skewed, so a few services carry most traffic. + - data_type: string + child_of: 0 + fan_out: 25 + distribution: + kind: zipf + skewness: 1.1 + population_size: 25 + seed: 2 + + # endpoint: 25 per service, skewed. + - data_type: string + child_of: 1 + fan_out: 25 + distribution: + kind: zipf + skewness: 1.2 + population_size: 25 + seed: 3 + + # user_id: the value. Count-Min counts its occurrences per group, + # HyperLogLog counts how many are distinct. + - data_type: i64 + distribution: + kind: zipf + skewness: 0.8 + population_size: 1000000 + seed: 5 From 0f99c402c55cf5da29208feeec8fd709dee4a3c2 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 10 Oct 2026 04:39:55 +0000 Subject: [PATCH 07/18] datagen: the hydra_hier usage example passes --dtype Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FhcWJExZcmVqS6r6rEtjis --- configs/datagen/hydra_hier.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/datagen/hydra_hier.yaml b/configs/datagen/hydra_hier.yaml index 968e6ecd..62c6a6aa 100644 --- a/configs/datagen/hydra_hier.yaml +++ b/configs/datagen/hydra_hier.yaml @@ -4,7 +4,7 @@ # before it are labels. # # approxbench sketchbench --variant hydra-cms --library lib \ -# --spec configs/datagen/hydra_hier.yaml \ +# --spec configs/datagen/hydra_hier.yaml --dtype i64 \ # --config "rows=3 cols=1024 cell_rows=3 cell_cols=1024" \ # --operations query --metrics accuracy # From 8f73ed39a485dd4a2be488b88dacd4977e8d30cb Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 10 Oct 2026 04:45:46 +0000 Subject: [PATCH 08/18] hydra: list every group in --per-group-out; escape keys; refuse no accuracy - Group keys escape `\`, `:` and `;` as polars_shared::subset_key does. - Groups that cannot be scored (zero true statistic, e.g. zero entropy) are written with an empty `error`, so n_q sums to the stream's records. - --per-group-out with no accuracy measurement is refused instead of writing an empty file. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FhcWJExZcmVqS6r6rEtjis --- README.md | 4 +- aqpbm-cli/src/cli.rs | 10 +++-- aqpbm-cli/src/main.rs | 8 ++-- aqpbm-cli/src/rows/statistic.rs | 7 ++-- aqpbm-cli/tests/hydra_grouping.rs | 32 +++++++++++++-- aqpbm-core/src/accuracy/mod.rs | 12 +++--- .../src/accuracy/subpopulation/cardinality.rs | 8 ++-- .../src/accuracy/subpopulation/entropy.rs | 28 +++++++++++++ .../src/accuracy/subpopulation/frequency.rs | 2 +- .../subpopulation/frequency_vector.rs | 8 ++-- .../src/accuracy/subpopulation/l1_norm.rs | 10 ++--- aqpbm-core/src/accuracy/subpopulation/mod.rs | 41 +++++++++++++++---- .../src/accuracy/subpopulation/rank_error.rs | 4 +- 13 files changed, 132 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 06118710..0fd6850d 100644 --- a/README.md +++ b/README.md @@ -453,7 +453,9 @@ The file can be found [under configs directory](./configs/datagen/hydra_columns. Any non-empty subset works, since the grid stores every subset of a record's labels; it picks the one the `hydra-*` row (`lib` and `polars`) asks and scores. With `--metrics accuracy`, the record also summarises each scored group's error in the row's own metric (relative error; mean rank error for `hydra-kll`): `err_mean`, `err_p50`, `err_p90`, `err_max`, `groups_scored`, plus `schema_width` (`d`), `records` (`N`) and `fanned_mass` (`N·(2^d − 1)`). -**`--per-group-out PATH`** (Hydra rows only): write those per-group errors as a CSV, `group_key,n_q,error`, one row per scored group (`n_q` is its record count, `group_key` its tagged labels such as `label0:a;label1:x`). +**`--per-group-out PATH`** (Hydra rows only): write those per-group errors as a CSV, `group_key,n_q,error`, one row per group (`n_q` is its record count, `group_key` its tagged labels such as `label0:a;label1:x`, with `\`, `:` and `;` in a label escaped by `\`). +Every group is listed, so the `n_q` sum to the stream's records; a group that cannot be scored (its true statistic is zero, e.g. zero entropy) has an empty `error` and is left out of the `err_*` summary. +It takes exactly one accuracy measurement (query or merge). #### General Explanation diff --git a/aqpbm-cli/src/cli.rs b/aqpbm-cli/src/cli.rs index 1667a45c..65e02ee4 100644 --- a/aqpbm-cli/src/cli.rs +++ b/aqpbm-cli/src/cli.rs @@ -249,10 +249,12 @@ pub struct SketchbenchArgs { /// Path to append JSONL records to. `-` or omitted sends them to stdout. #[arg(long, help_heading = "Output")] pub report: Option, - /// Write a `hydra-*` row's per-group errors to this CSV, one row per scored - /// group: `group_key,n_q,error` (`n_q` is the group's record count, `error` - /// the row's own metric). The record's `err_*` fields summarise the same - /// rows. Takes one accuracy measurement: query or merge, not both. + /// Write a `hydra-*` row's per-group errors to this CSV, one row per group: + /// `group_key,n_q,error` (`n_q` is the group's record count, `error` the + /// row's own metric, empty for a group that cannot be scored, such as one + /// with zero entropy). The `n_q` sum to the stream's records; the record's + /// `err_*` fields summarise the scored rows. Takes exactly one accuracy + /// measurement: query or merge. #[arg(long, help_heading = "Output")] pub per_group_out: Option, /// Write one line for the whole invocation instead of one per measurement, diff --git a/aqpbm-cli/src/main.rs b/aqpbm-cli/src/main.rs index 105035df..44edd0f9 100644 --- a/aqpbm-cli/src/main.rs +++ b/aqpbm-cli/src/main.rs @@ -357,10 +357,12 @@ fn run_sketchbench(args: SketchbenchArgs) -> Result<()> { if !variant.starts_with("hydra-") { bail!("--per-group-out: only the hydra-* rows score groups, and {variant} is not one"); } - if want.iter().filter(|(_, m)| *m == Metric::Accuracy).count() > 1 { - bail!( + match want.iter().filter(|(_, m)| *m == Metric::Accuracy).count() { + 0 => bail!("--per-group-out writes an accuracy measurement, and none was asked for"), + 1 => {} + _ => bail!( "--per-group-out writes one accuracy measurement; ask for query or merge, not both" - ); + ), } } let req = Requirement { diff --git a/aqpbm-cli/src/rows/statistic.rs b/aqpbm-cli/src/rows/statistic.rs index 87a227aa..7724a139 100644 --- a/aqpbm-cli/src/rows/statistic.rs +++ b/aqpbm-cli/src/rows/statistic.rs @@ -428,8 +428,8 @@ where Ok(bodies) } -/// The scorer, also writing each scored group's error to `path` as -/// `group_key,n_q,error` whenever it scores. Every pass answers the same, so +/// The scorer, also writing every group to `path` as `group_key,n_q,error` +/// whenever it scores; `error` is empty for a group that cannot be scored. Every pass answers the same, so /// each rewrites the same rows. The file is created here, so a path that /// cannot be written fails before anything is measured. fn per_group_written( @@ -450,7 +450,8 @@ fn per_group_written( } else { g.group }; - csv.push_str(&format!("{key},{},{}\n", g.n_q, g.error)); + let error = g.error.map_or(String::new(), |e| e.to_string()); + csv.push_str(&format!("{key},{},{error}\n", g.n_q)); } std::fs::write(&path, csv) .unwrap_or_else(|e| panic!("--per-group-out {}: {e}", path.display())); diff --git a/aqpbm-cli/tests/hydra_grouping.rs b/aqpbm-cli/tests/hydra_grouping.rs index 50464482..281ccec5 100644 --- a/aqpbm-cli/tests/hydra_grouping.rs +++ b/aqpbm-cli/tests/hydra_grouping.rs @@ -65,8 +65,8 @@ fn scored( /// Every Hydra family, lib at a grid far larger than its 19 subkeys and the /// polars baseline, grouped by column 1 alone and by both columns: both score -/// (about) zero, and the per-group file has one row per scored group, keyed by -/// the columns asked. +/// (about) zero, and the per-group file has one row per group, keyed by the +/// columns asked. #[test] fn any_label_subset_is_asked_where_it_was_inserted() { let rows = [ @@ -93,7 +93,8 @@ fn any_label_subset_is_asked_where_it_was_inserted() { let mut lines = csv.lines(); assert_eq!(lines.next(), Some("group_key,n_q,error"), "{at}"); let rows: Vec<&str> = lines.collect(); - assert_eq!(rows.len() as f64, acc["groups_scored"].as_f64().expect(&at)); + let scored = rows.iter().filter(|r| !r.ends_with(',')).count(); + assert_eq!(scored as f64, acc["groups_scored"].as_f64().expect(&at)); let n_q: u64 = rows .iter() .map(|r| r.split(',').nth(1).and_then(|n| n.parse::().ok())) @@ -135,3 +136,28 @@ fn a_group_column_past_the_labels_is_refused() { let err = String::from_utf8_lossy(&out.stderr); assert!(err.contains("--group-columns 2"), "{err}"); } + +/// `--per-group-out` writes an accuracy measurement, so a run that asks for +/// none is refused rather than leaving an empty file. +#[test] +fn per_group_out_without_accuracy_is_refused() { + let spec = spec("i64"); + let csv = spec.with_extension("no-accuracy.csv"); + let out = Command::new(env!("CARGO_BIN_EXE_approxbench")) + .args(["sketchbench", "--variant", "hydra-cms", "--library", "lib"]) + .args([ + "--spec", + spec.to_str().expect("utf-8 path"), + "--dtype", + "i64", + ]) + .args(["--per-group-out", csv.to_str().expect("utf-8 path")]) + .args(["--operations", "query", "--metrics", "throughput"]) + .args(["--runs", "1", "--warmup-runs", "0"]) + .output() + .expect("approxbench runs"); + assert!(!out.status.success()); + let err = String::from_utf8_lossy(&out.stderr); + assert!(err.contains("none was asked for"), "{err}"); + assert!(!csv.exists(), "no file is left behind"); +} diff --git a/aqpbm-core/src/accuracy/mod.rs b/aqpbm-core/src/accuracy/mod.rs index 698b38d0..1ae393ed 100644 --- a/aqpbm-core/src/accuracy/mod.rs +++ b/aqpbm-core/src/accuracy/mod.rs @@ -103,9 +103,9 @@ pub trait GroundTruth { answers: &[Self::Answer], ) -> BTreeMap; - /// ④ Each scored group's error, for a comparator whose population is + /// ④ Every group with its error, for a comparator whose population is /// subpopulations; empty for every other. [`score`](Self::score) summarises - /// the same rows, and `--per-group-out` writes them. + /// the scored rows, and `--per-group-out` writes them all. fn per_group( &self, _truth: &Self::Truth, @@ -116,13 +116,15 @@ pub trait GroundTruth { } } -/// One scored subpopulation: its key, how many records it carried (`n_q`), and -/// its error in the comparator's own metric, lower is better. +/// One subpopulation: its key, how many records it carried (`n_q`), and its +/// error in the comparator's own metric, lower is better. `None` when the group +/// cannot be scored (its true statistic is zero, so it has no relative error); +/// it is still listed, so the groups' `n_q` sum to the stream's records. #[derive(Clone, Debug, PartialEq)] pub struct GroupError { pub group: String, pub n_q: u64, - pub error: f64, + pub error: Option, } /// How a row's answers turn into the named error metrics: the scorer diff --git a/aqpbm-core/src/accuracy/subpopulation/cardinality.rs b/aqpbm-core/src/accuracy/subpopulation/cardinality.rs index c3128b81..3cf99df0 100644 --- a/aqpbm-core/src/accuracy/subpopulation/cardinality.rs +++ b/aqpbm-core/src/accuracy/subpopulation/cardinality.rs @@ -123,7 +123,8 @@ impl GroundTruth for SubpopCardinalityGT { metrics } - /// Relative error per group, as `are_all` averages it. + /// Relative error per group, as `are_all` averages it; a group with no + /// distinct value is listed unscored. fn per_group( &self, truth: &SubpopCardTruth, @@ -132,8 +133,9 @@ impl GroundTruth for SubpopCardinalityGT { ) -> Vec { let est: HashMap<&GroupKey, f64> = probes.iter().zip(answers).map(|(p, a)| (p, *a)).collect(); - group_errors(truth.exact.iter().filter(|(_, &d)| d > 0).map(|(g, &d)| { - let err = (est.get(g).copied().unwrap_or(0.0) - d as f64).abs() / d as f64; + group_errors(truth.exact.iter().map(|(g, &d)| { + let err = + (d > 0).then(|| (est.get(g).copied().unwrap_or(0.0) - d as f64).abs() / d as f64); (g.clone(), truth.sizes[g], err) })) } diff --git a/aqpbm-core/src/accuracy/subpopulation/entropy.rs b/aqpbm-core/src/accuracy/subpopulation/entropy.rs index f96bb132..4d2bdbb3 100644 --- a/aqpbm-core/src/accuracy/subpopulation/entropy.rs +++ b/aqpbm-core/src/accuracy/subpopulation/entropy.rs @@ -102,6 +102,34 @@ mod tests { assert_eq!(cmp["aae_all"], 0.0); } + /// Over both columns only (a, x) holds two values; the other groups have + /// zero entropy and no relative error, but are still listed, so `n_q` + /// sums to the stream's 6 records. + #[test] + fn a_zero_entropy_group_is_listed_unscored() { + let gt = over(vec![0, 1]); + let truth = gt + .truth(&records()) + .expect("the table has the named columns"); + let probes = gt.probes(&truth); + let rows: Vec<(String, u64, Option)> = gt + .per_group(&truth, &probes, &vec![0.0; probes.len()]) + .into_iter() + .map(|g| (g.group, g.n_q, g.error)) + .collect(); + assert_eq!( + rows, + vec![ + ("label0:a;label1:x".to_string(), 2, Some(1.0)), + ("label0:b;label1:y".to_string(), 2, None), + ("label0:a;label1:y".to_string(), 1, None), + ("label0:b;label1:x".to_string(), 1, None), + ] + ); + let cmp = null_over(>, &records()); + assert_eq!(cmp["groups_scored"], 1.0); + } + #[test] fn a_different_grouping_is_a_different_population() { let cmp = null_over(&over(vec![0, 1]), &records()); diff --git a/aqpbm-core/src/accuracy/subpopulation/frequency.rs b/aqpbm-core/src/accuracy/subpopulation/frequency.rs index 75a8b2d1..588a286f 100644 --- a/aqpbm-core/src/accuracy/subpopulation/frequency.rs +++ b/aqpbm-core/src/accuracy/subpopulation/frequency.rs @@ -173,7 +173,7 @@ where group_errors( per_group .into_iter() - .map(|(g, (sum, pairs))| (g.clone(), truth.sizes[g], sum / pairs as f64)), + .map(|(g, (sum, pairs))| (g.clone(), truth.sizes[g], Some(sum / pairs as f64))), ) } } diff --git a/aqpbm-core/src/accuracy/subpopulation/frequency_vector.rs b/aqpbm-core/src/accuracy/subpopulation/frequency_vector.rs index 43d7b209..08421ec4 100644 --- a/aqpbm-core/src/accuracy/subpopulation/frequency_vector.rs +++ b/aqpbm-core/src/accuracy/subpopulation/frequency_vector.rs @@ -120,16 +120,16 @@ pub(super) fn score_over( } /// Relative error per group, as `are_all` averages it: a group whose true -/// statistic is zero (entropy over one value) has no relative error and is -/// not scored. +/// statistic is zero (entropy over one value) has no relative error, so it is +/// listed unscored. pub(super) fn per_group_over( truth: &SubpopVectorTruth, probes: &[GroupKey], answers: &[f64], ) -> Vec { let est: HashMap<&GroupKey, f64> = probes.iter().zip(answers).map(|(p, a)| (p, *a)).collect(); - group_errors(truth.exact.iter().filter(|(_, &v)| v > 0.0).map(|(g, &v)| { - let err = (est.get(g).copied().unwrap_or(0.0) - v).abs() / v; + group_errors(truth.exact.iter().map(|(g, &v)| { + let err = (v > 0.0).then(|| (est.get(g).copied().unwrap_or(0.0) - v).abs() / v); (g.clone(), truth.sizes[g], err) })) } diff --git a/aqpbm-core/src/accuracy/subpopulation/l1_norm.rs b/aqpbm-core/src/accuracy/subpopulation/l1_norm.rs index 28ca38e2..c32f9f5f 100644 --- a/aqpbm-core/src/accuracy/subpopulation/l1_norm.rs +++ b/aqpbm-core/src/accuracy/subpopulation/l1_norm.rs @@ -114,7 +114,7 @@ mod tests { let truth = gt.truth(&table).expect("the table has the named columns"); let probes = gt.probes(&truth); let asked: Vec = probes.iter().map(|p| answers[p]).collect(); - let rows: Vec<(String, u64, f64)> = gt + let rows: Vec<(String, u64, Option)> = gt .per_group(&truth, &probes, &asked) .into_iter() .map(|g| (g.group, g.n_q, g.error)) @@ -122,10 +122,10 @@ mod tests { assert_eq!( rows, vec![ - ("label0:a;label1:x".to_string(), 2, 0.0), - ("label0:b;label1:y".to_string(), 2, 0.5), - ("label0:a;label1:y".to_string(), 1, 1.0), - ("label0:b;label1:x".to_string(), 1, 0.5), + ("label0:a;label1:x".to_string(), 2, Some(0.0)), + ("label0:b;label1:y".to_string(), 2, Some(0.5)), + ("label0:a;label1:y".to_string(), 1, Some(1.0)), + ("label0:b;label1:x".to_string(), 1, Some(0.5)), ] ); } diff --git a/aqpbm-core/src/accuracy/subpopulation/mod.rs b/aqpbm-core/src/accuracy/subpopulation/mod.rs index 2f16d227..0f92c223 100644 --- a/aqpbm-core/src/accuracy/subpopulation/mod.rs +++ b/aqpbm-core/src/accuracy/subpopulation/mod.rs @@ -66,18 +66,30 @@ fn owned(group: &[&str], columns: &[usize]) -> GroupKey { } /// How a group is named in `--per-group-out`: each grouped column's value -/// tagged with its column, as the grid's subkeys are (`label1:x`). +/// tagged with its column, as the grid's subkeys are (`label1:x`), with `\`, +/// `:` and `;` escaped. The same encoding as sketch-bench's +/// `polars_shared::subset_key`, which this crate cannot depend on. fn rendered(key: &GroupKey) -> String { - key.iter() - .enumerate() - .filter_map(|(j, label)| label.as_ref().map(|l| format!("label{j}:{l}"))) - .collect::>() - .join(";") + let mut out = String::new(); + for (j, label) in key.iter().enumerate() { + let Some(label) = label else { continue }; + if !out.is_empty() { + out.push(';'); + } + out.push_str(&format!("label{j}:")); + for ch in label.chars() { + if matches!(ch, '\\' | ':' | ';') { + out.push('\\'); + } + out.push(ch); + } + } + out } /// One [`GroupError`] per group, heaviest first (ties on the key), so the /// per-group file reads the same across runs. -fn group_errors(errors: impl Iterator) -> Vec { +fn group_errors(errors: impl Iterator)>) -> Vec { let mut out: Vec = errors .map(|(key, n_q, error)| GroupError { group: rendered(&key), @@ -101,7 +113,7 @@ fn write_groups( schema_width: usize, metrics: &mut BTreeMap, ) { - let mut errs: Vec = groups.iter().map(|g| g.error).collect(); + let mut errs: Vec = groups.iter().filter_map(|g| g.error).collect(); errs.sort_by(f64::total_cmp); if let Some(&max) = errs.last() { metrics.insert( @@ -169,3 +181,16 @@ where table, ) } + +#[cfg(test)] +mod tests { + use super::*; + + /// A label holding the separators is escaped as the grid's subkeys are, + /// so it cannot be read as two columns. + #[test] + fn rendered_keys_escape_the_separators() { + let key = vec![None, Some(r"x:y;z\".to_string()), Some(String::new())]; + assert_eq!(rendered(&key), r"label1:x\:y\;z\\;label2:"); + } +} diff --git a/aqpbm-core/src/accuracy/subpopulation/rank_error.rs b/aqpbm-core/src/accuracy/subpopulation/rank_error.rs index 4c2f65fa..01ec1259 100644 --- a/aqpbm-core/src/accuracy/subpopulation/rank_error.rs +++ b/aqpbm-core/src/accuracy/subpopulation/rank_error.rs @@ -100,7 +100,7 @@ impl GroundTruth for SubpopRankErrorGT { metrics.insert( "mean_rank_err".into(), if scored_groups > 0 { - groups.iter().map(|g| g.error).sum::() / scored_groups as f64 + groups.iter().filter_map(|g| g.error).sum::() / scored_groups as f64 } else { 0.0 }, @@ -164,7 +164,7 @@ fn rank_errors( groups.push(( group.clone(), sorted.len() as u64, - group_sum / GROUP_GRID_POINTS as f64, + Some(group_sum / GROUP_GRID_POINTS as f64), )); } (group_errors(groups.into_iter()), max_err) From 2ba16fb6c1ab8ace36ec6342249298eb02c17e2e Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 10 Oct 2026 04:44:29 +0000 Subject: [PATCH 09/18] hydra: KLL CDF and UnivMon sum rows; KLL cell merge buffer - hydra-kll-cdf (lib, polars): asks the KLL grid HydraQuery::Cdf(x) at each group's own values at phi = 0.01..0.99, scored by absolute CDF error (SubpopCdfErrorGT, comparator subpop-cdf-error). The polars baseline is the exact share at or below x over the same sorted groups. - hydra-univmon-sum (lib, polars): each record inserted with count = value, so the UnivMon grid's L1 is the group's sum, scored by relative error against exact per-group sums (SubpopSumGT, comparator subpop-sum). Integer value dtypes only; f64/str and values outside [0, i32::MAX] are refused by name. - memory_hydra_kll counts each cell as kll_lib_bytes does, merge buffer of cell_k items included. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FhcWJExZcmVqS6r6rEtjis --- README.md | 5 +- aqpbm-cli/src/rows/hydra.rs | 62 ++++++ aqpbm-cli/src/rows/hydra_univmon.rs | 80 ++++++++ aqpbm-cli/src/rows/mod.rs | 8 +- aqpbm-cli/src/rows/statistic.rs | 32 +++ aqpbm-cli/tests/hydra_grouping.rs | 29 +++ .../src/accuracy/subpopulation/cdf_error.rs | 193 ++++++++++++++++++ .../subpopulation/frequency_vector.rs | 29 ++- aqpbm-core/src/accuracy/subpopulation/mod.rs | 4 + .../src/accuracy/subpopulation/rank_error.rs | 8 +- aqpbm-core/src/accuracy/subpopulation/sum.rs | 182 +++++++++++++++++ configs/datagen/hydra_columns_f64.yaml | 6 +- configs/datagen/hydra_univmon_columns.yaml | 2 +- docs/concepts.md | 2 + sketch-bench/src/registry.rs | 66 ++++++ sketch-bench/src/request.rs | 4 + sketch-bench/src/wrappers/hydra_kll/mod.rs | 61 +++++- sketch-bench/src/wrappers/hydra_kll/polars.rs | 46 ++++- .../src/wrappers/hydra_kll/sketchlib.rs | 67 +++++- sketch-bench/src/wrappers/hydra_shared.rs | 14 +- .../src/wrappers/hydra_univmon/mod.rs | 55 +++++ .../src/wrappers/hydra_univmon/polars.rs | 144 +++++++++++++ .../src/wrappers/hydra_univmon/sketchlib.rs | 185 ++++++++++++++++- sketch-bench/src/wrappers/kll/mod.rs | 5 +- sketch-bench/src/wrappers/polars_shared.rs | 2 +- 25 files changed, 1245 insertions(+), 46 deletions(-) create mode 100644 aqpbm-core/src/accuracy/subpopulation/cdf_error.rs create mode 100644 aqpbm-core/src/accuracy/subpopulation/sum.rs diff --git a/README.md b/README.md index 0fd6850d..1baf3318 100644 --- a/README.md +++ b/README.md @@ -451,7 +451,10 @@ The file can be found [under configs directory](./configs/datagen/hydra_columns. **`--group-columns`** (Hydra rows only): which label columns the subpopulation is taken over, as comma-separated indices into the columns before the value column, e.g. `1` or `0,1`; default `0`. Any non-empty subset works, since the grid stores every subset of a record's labels; it picks the one the `hydra-*` row (`lib` and `polars`) asks and scores. -With `--metrics accuracy`, the record also summarises each scored group's error in the row's own metric (relative error; mean rank error for `hydra-kll`): `err_mean`, `err_p50`, `err_p90`, `err_max`, `groups_scored`, plus `schema_width` (`d`), `records` (`N`) and `fanned_mass` (`N·(2^d − 1)`). +With `--metrics accuracy`, the record also summarises each scored group's error in the row's own metric (relative error; mean rank error for `hydra-kll`; mean absolute CDF error for `hydra-kll-cdf`): `err_mean`, `err_p50`, `err_p90`, `err_max`, `groups_scored`, plus `schema_width` (`d`), `records` (`N`) and `fanned_mass` (`N·(2^d − 1)`). + +Two Hydra rows ask a second statistic of an existing cell: `hydra-kll-cdf` asks the KLL grid `Cdf(x)`, the share of a group at or below `x`, at each group's own values at `phi` = 0.01..0.99; `hydra-univmon-sum` inserts each value counted by itself (i64 or u64, within an `i32` count), so the UnivMon grid's L1 is the group's sum. +Both have a `polars` exact baseline. **`--per-group-out PATH`** (Hydra rows only): write those per-group errors as a CSV, `group_key,n_q,error`, one row per group (`n_q` is its record count, `group_key` its tagged labels such as `label0:a;label1:x`, with `\`, `:` and `;` in a label escaped by `\`). Every group is listed, so the `n_q` sum to the stream's records; a group that cannot be scored (its true statistic is zero, e.g. zero entropy) has an empty `error` and is left out of the `err_*` summary. diff --git a/aqpbm-cli/src/rows/hydra.rs b/aqpbm-cli/src/rows/hydra.rs index a1a60241..0c1cdbdf 100644 --- a/aqpbm-cli/src/rows/hydra.rs +++ b/aqpbm-cli/src/rows/hydra.rs @@ -253,3 +253,65 @@ pub(crate) fn row_hydra_kll_polars( other => Err(no_build_at(req, other)), } } + +pub(crate) fn row_hydra_kll_cdf_lib( + req: &Requirement, + description: &TableDescription, + table: GeneratedTable, + want: &[(Operation, Metric)], +) -> Result { + macro_rules! at { + ($t:ty) => { + subpop_cdf_row::<$t>( + req, + description, + table, + want, + hks::insert_hydra_kll::<$t>, + hks::insert_step_hydra_kll::<$t>, + hks::query_hydra_kll_cdf::<$t>, + Some(( + hks::merge_hydra_kll::<$t>, + hks::merge_step_hydra_kll::<$t>, + hks::merge_query_hydra_kll_cdf::<$t>, + )), + None, + ) + }; + } + match req.width { + Dtype::I64 => at!(i64), + Dtype::U64 => at!(u64), + Dtype::F64 => at!(f64), + other => Err(no_build_at(req, other)), + } +} + +pub(crate) fn row_hydra_kll_cdf_polars( + req: &Requirement, + description: &TableDescription, + table: GeneratedTable, + want: &[(Operation, Metric)], +) -> Result { + macro_rules! at { + ($t:ty) => { + subpop_cdf_row::<$t>( + req, + description, + table, + want, + hkp::insert_polars_subpop_quantile::<$t>, + hkp::insert_step_polars_subpop_quantile::<$t>, + hkp::query_polars_subpop_cdf::<$t>, + None, + Some(hkp::prepare_polars_subpop_quantile::<$t>), + ) + }; + } + match req.width { + Dtype::I64 => at!(i64), + Dtype::U64 => at!(u64), + Dtype::F64 => at!(f64), + other => Err(no_build_at(req, other)), + } +} diff --git a/aqpbm-cli/src/rows/hydra_univmon.rs b/aqpbm-cli/src/rows/hydra_univmon.rs index 34320aa9..cd3f283c 100644 --- a/aqpbm-cli/src/rows/hydra_univmon.rs +++ b/aqpbm-cli/src/rows/hydra_univmon.rs @@ -272,3 +272,83 @@ pub(crate) fn row_hydra_univmon_entropy_polars( Dtype::Str => at!(String), } } + +/// The sum rows insert each value as its own integer count, so they build at +/// the integer widths only. +fn no_sum_at(req: &Requirement, got: Dtype) -> RunError { + RunError::Sketch(format!( + "{}/{} counts each value by itself, an integer weight, so builds at i64 or u64; \ + --dtype {} is not an integer", + req.variant, + req.library, + got.name(), + )) +} + +pub(crate) fn row_hydra_univmon_sum_lib( + req: &Requirement, + description: &TableDescription, + table: GeneratedTable, + want: &[(Operation, Metric)], +) -> Result { + macro_rules! at { + ($t:ty) => { + subpop_vector_row::<$t, _>( + req, + description, + table, + want, + SubpopSumGT { + group_columns: group_columns(req, description)?, + value_column: value_column(description), + }, + hus::insert_hydra_univmon_sum::<$t>, + hus::insert_step_hydra_univmon_sum::<$t>, + hus::query_hydra_univmon_sum::<$t>, + Some(( + hus::merge_hydra_univmon_sum::<$t>, + hus::merge_step_hydra_univmon_sum::<$t>, + hus::merge_query_hydra_univmon_sum::<$t>, + )), + None, + ) + }; + } + match req.width { + Dtype::I64 => at!(i64), + Dtype::U64 => at!(u64), + other => Err(no_sum_at(req, other)), + } +} + +pub(crate) fn row_hydra_univmon_sum_polars( + req: &Requirement, + description: &TableDescription, + table: GeneratedTable, + want: &[(Operation, Metric)], +) -> Result { + macro_rules! at { + ($t:ty) => { + subpop_vector_row::<$t, _>( + req, + description, + table, + want, + SubpopSumGT { + group_columns: group_columns(req, description)?, + value_column: value_column(description), + }, + hup::insert_polars_subpop_sum::<$t>, + hup::insert_step_polars_subpop_sum::<$t>, + hup::query_polars_subpop_sum::<$t>, + None, + Some(hup::prepare_polars_subpop_sum::<$t>), + ) + }; + } + match req.width { + Dtype::I64 => at!(i64), + Dtype::U64 => at!(u64), + other => Err(no_sum_at(req, other)), + } +} diff --git a/aqpbm-cli/src/rows/mod.rs b/aqpbm-cli/src/rows/mod.rs index 66524a42..f1937bfc 100644 --- a/aqpbm-cli/src/rows/mod.rs +++ b/aqpbm-cli/src/rows/mod.rs @@ -23,8 +23,8 @@ use aqpbm_core::accuracy::keyed::{ }; use aqpbm_core::accuracy::quantile::{RankErrorGT, RelativeValueErrorGT}; use aqpbm_core::accuracy::subpopulation::{ - GroupKey, SubpopCardinalityGT, SubpopEntropyGT, SubpopFrequencyGT, SubpopL1NormGT, - SubpopL2NormGT, SubpopRankErrorGT, + GroupKey, SubpopCardinalityGT, SubpopCdfErrorGT, SubpopEntropyGT, SubpopFrequencyGT, + SubpopL1NormGT, SubpopL2NormGT, SubpopRankErrorGT, SubpopSumGT, }; use aqpbm_core::accuracy::topk::TopkGT; use aqpbm_core::accuracy::Score; @@ -127,6 +127,8 @@ fn binding(variant: &str, library: &str) -> Option { ("hydra-hll", "polars") => row_hydra_hll_polars, ("hydra-kll", "lib") => row_hydra_kll_lib, ("hydra-kll", "polars") => row_hydra_kll_polars, + ("hydra-kll-cdf", "lib") => row_hydra_kll_cdf_lib, + ("hydra-kll-cdf", "polars") => row_hydra_kll_cdf_polars, ("hydra-univmon-cardinality", "lib") => row_hydra_univmon_cardinality_lib, ("hydra-univmon-cardinality", "polars") => row_hydra_univmon_cardinality_polars, ("hydra-univmon-l1-norm", "lib") => row_hydra_univmon_l1_norm_lib, @@ -135,6 +137,8 @@ fn binding(variant: &str, library: &str) -> Option { ("hydra-univmon-l2-norm", "polars") => row_hydra_univmon_l2_norm_polars, ("hydra-univmon-entropy", "lib") => row_hydra_univmon_entropy_lib, ("hydra-univmon-entropy", "polars") => row_hydra_univmon_entropy_polars, + ("hydra-univmon-sum", "lib") => row_hydra_univmon_sum_lib, + ("hydra-univmon-sum", "polars") => row_hydra_univmon_sum_polars, ("univmon-cardinality", "lib") => row_univmon_cardinality_lib, ("univmon-l1-norm", "lib") => row_univmon_l1_norm_lib, ("univmon-l1-norm", "oxide") => row_univmon_l1_norm_oxide, diff --git a/aqpbm-cli/src/rows/statistic.rs b/aqpbm-cli/src/rows/statistic.rs index 7724a139..232dfff0 100644 --- a/aqpbm-cli/src/rows/statistic.rs +++ b/aqpbm-cli/src/rows/statistic.rs @@ -259,6 +259,38 @@ pub(super) fn subpop_quantile_row( ) } +/// A row answering **subpopulation CDF**: the share of a group at or below a +/// value, scored in absolute CDF error. +#[allow(clippy::too_many_arguments, clippy::type_complexity)] +pub(super) fn subpop_cdf_row( + req: &Requirement, + description: &TableDescription, + table: GeneratedTable, + want: &[(Operation, Metric)], + insert: InsertBody<(String, V)>, + insert_step: InsertStepBody<(String, V)>, + query: QueryBody<(String, V), (GroupKey, f64), f64>, + merge: Option>, + prepare: Option>, +) -> Result { + scored_row( + req, + description, + table, + want, + SubpopCdfErrorGT { + group_columns: group_columns(req, description)?, + value_column: value_column(description), + }, + peel_labeled::, + insert, + insert_step, + query, + merge, + prepare, + ) +} + #[allow(clippy::too_many_arguments)] pub(super) fn keyed_row( req: &Requirement, diff --git a/aqpbm-cli/tests/hydra_grouping.rs b/aqpbm-cli/tests/hydra_grouping.rs index 281ccec5..5050bfb5 100644 --- a/aqpbm-cli/tests/hydra_grouping.rs +++ b/aqpbm-cli/tests/hydra_grouping.rs @@ -74,11 +74,17 @@ fn any_label_subset_is_asked_where_it_was_inserted() { ("hydra-cs", "rows=3 cols=1024 cell_rows=3 cell_cols=256", "i64"), ("hydra-hll", "rows=3 cols=1024", "i64"), ("hydra-kll", "rows=3 cols=1024 cell_k=4096", "f64"), + ("hydra-kll-cdf", "rows=3 cols=1024 cell_k=4096", "f64"), ( "hydra-univmon-l1-norm", "rows=3 cols=1024 cell_heap_size=64 cell_sketch_row=3 cell_sketch_col=256 cell_layer_size=4", "i64", ), + ( + "hydra-univmon-sum", + "rows=3 cols=1024 cell_heap_size=64 cell_sketch_row=3 cell_sketch_col=256 cell_layer_size=4", + "i64", + ), ]; for (variant, config, dtype) in rows { for group_columns in ["1", "0,1"] { @@ -161,3 +167,26 @@ fn per_group_out_without_accuracy_is_refused() { assert!(err.contains("none was asked for"), "{err}"); assert!(!csv.exists(), "no file is left behind"); } + +/// The sum rows count each value by itself, an integer weight: an `f64` +/// value column is refused by name, at both libraries. +#[test] +fn a_float_sum_is_refused_by_name() { + let spec = spec("f64"); + for library in ["lib", "polars"] { + let out = Command::new(env!("CARGO_BIN_EXE_approxbench")) + .args(["sketchbench", "--variant", "hydra-univmon-sum"]) + .args(["--library", library, "--dtype", "f64"]) + .args(["--spec", spec.to_str().expect("utf-8 path")]) + .args(["--operations", "query", "--metrics", "accuracy"]) + .args(["--runs", "1", "--warmup-runs", "0"]) + .output() + .expect("approxbench runs"); + assert!(!out.status.success(), "{library}"); + let err = String::from_utf8_lossy(&out.stderr); + assert!( + err.contains("f64") && err.contains("integer"), + "{library}: {err}" + ); + } +} diff --git a/aqpbm-core/src/accuracy/subpopulation/cdf_error.rs b/aqpbm-core/src/accuracy/subpopulation/cdf_error.rs new file mode 100644 index 00000000..9c734456 --- /dev/null +++ b/aqpbm-core/src/accuracy/subpopulation/cdf_error.rs @@ -0,0 +1,193 @@ +//! The CDF inside one subpopulation, in absolute CDF error: the inverse of the +//! rank-error question, asked at a value and answered with a fraction. + +use std::collections::BTreeMap; + +use aqpbm_datagen::{DataGenError, GeneratedTable}; + +use super::{group_errors, write_groups, GroupKey, SubpopRankErrorGT, SubpopRankTruth}; +use crate::accuracy::{GroundTruth, GroupError}; + +/// Thresholds probed per group: the group's own values at `phi` = 0.01..=0.99, +/// so every threshold sits inside the group's range whatever its scale. +const CDF_GRID_POINTS: usize = 99; + +/// Ground truth for a grouped CDF sketch: `F_q(x)`, the share of group `q`'s +/// values at or below `x`, scored as `|F̂_q(x) − F_q(x)|`. +pub struct SubpopCdfErrorGT { + /// Which label columns the subpopulation is taken over. + pub group_columns: Vec, + pub value_column: usize, +} + +/// `F(x)` over a sorted group: values at or below `x`, as a share. +fn cdf_at(sorted: &[f64], x: f64) -> f64 { + sorted.partition_point(|v| *v <= x) as f64 / sorted.len() as f64 +} + +impl GroundTruth for SubpopCdfErrorGT { + /// The same sorted values per group the rank-error comparator holds. + type Truth = SubpopRankTruth; + /// One (group, threshold) question. + type Probe = (GroupKey, f64); + type Answer = f64; + + fn truth(&self, table: &GeneratedTable) -> Result { + SubpopRankErrorGT { + group_columns: self.group_columns.clone(), + value_column: self.value_column, + } + .truth(table) + } + + /// The value at index `floor(phi * n)`, the index the exact quantile + /// baseline answers `phi` with. + fn probes(&self, truth: &SubpopRankTruth) -> Vec<(GroupKey, f64)> { + let mut out = Vec::with_capacity(truth.probed.len() * CDF_GRID_POINTS); + for group in &truth.probed { + let Some(sorted) = truth.per_group.get(group).filter(|s| !s.is_empty()) else { + continue; + }; + for i in 1..=CDF_GRID_POINTS { + let phi = i as f64 / 100.0; + let idx = ((phi * sorted.len() as f64).floor() as usize).min(sorted.len() - 1); + out.push((group.clone(), sorted[idx])); + } + } + out + } + + fn score( + &self, + truth: &SubpopRankTruth, + probes: &[(GroupKey, f64)], + answers: &[f64], + ) -> BTreeMap { + let (groups, max_err) = cdf_errors(truth, probes, answers); + let scored_groups = groups.len(); + let mut metrics: BTreeMap = BTreeMap::new(); + metrics.insert( + "mean_cdf_err".into(), + if scored_groups > 0 { + groups.iter().filter_map(|g| g.error).sum::() / scored_groups as f64 + } else { + 0.0 + }, + ); + metrics.insert("max_cdf_err".into(), max_err); + metrics.insert("grid_points".into(), CDF_GRID_POINTS as f64); + metrics.insert("items".into(), truth.items as f64); + metrics.insert("probes".into(), scored_groups as f64); + metrics.insert("subpopulations".into(), truth.per_group.len() as f64); + metrics.insert("group_columns".into(), self.group_columns.len() as f64); + write_groups( + &groups, + truth.items as u64, + truth.schema_width, + &mut metrics, + ); + metrics + } + + /// A group's error is its mean absolute CDF error over the grid, the + /// number `mean_cdf_err` averages over groups. Every group holds a record, + /// so every group is scored. + fn per_group( + &self, + truth: &SubpopRankTruth, + probes: &[(GroupKey, f64)], + answers: &[f64], + ) -> Vec { + cdf_errors(truth, probes, answers).0 + } +} + +/// Each group's mean absolute CDF error over its grid, plus the largest error +/// any single probe saw. +fn cdf_errors( + truth: &SubpopRankTruth, + probes: &[(GroupKey, f64)], + answers: &[f64], +) -> (Vec, f64) { + let mut max_err = 0.0f64; + let mut groups = Vec::new(); + // The probe set is one contiguous grid per group, in order. + for (chunk_p, chunk_a) in probes + .chunks(CDF_GRID_POINTS) + .zip(answers.chunks(CDF_GRID_POINTS)) + { + let Some((group, _)) = chunk_p.first() else { + continue; + }; + let Some(sorted) = truth.per_group.get(group) else { + continue; + }; + let mut group_sum = 0.0f64; + for ((_, x), est) in chunk_p.iter().zip(chunk_a) { + let err = (est - cdf_at(sorted, *x)).abs(); + group_sum += err; + max_err = max_err.max(err); + } + groups.push(( + group.clone(), + sorted.len() as u64, + Some(group_sum / CDF_GRID_POINTS as f64), + )); + } + (group_errors(groups.into_iter()), max_err) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::accuracy::subpopulation::records; + + fn over(group_columns: Vec) -> SubpopCdfErrorGT { + SubpopCdfErrorGT { + group_columns, + value_column: 2, + } + } + + /// Answering each threshold with its exact share scores zero, and the + /// thresholds are the group's own values: `a` holds 10, 10, 20, so + /// `F_a(10)` is 2/3 and `F_a(20)` is 1, at or below, ties included. + #[test] + fn the_exact_cdf_scores_zero() { + let gt = over(vec![0]); + let table = records(); + let mut truth = gt.truth(&table).expect("the table has the named columns"); + let probes = gt.probes(&truth); + assert_eq!(probes.len(), 2 * CDF_GRID_POINTS); + let a = vec![Some("a".to_string())]; + let sorted = truth.per_group[&a].clone(); + assert_eq!(cdf_at(&sorted, 10.0), 2.0 / 3.0); + assert_eq!(cdf_at(&sorted, 20.0), 1.0); + let cmp = crate::accuracy::score_with( + >, + &|t: &mut SubpopRankTruth, p: &(GroupKey, f64)| cdf_at(&t.per_group[&p.0], p.1), + &mut truth, + &table, + ); + assert_eq!(cmp["mean_cdf_err"], 0.0); + assert_eq!(cmp["err_max"], 0.0); + assert_eq!(cmp["groups_scored"], 2.0); + } + + /// Answering 0 everywhere misses each threshold by its whole share: over + /// `b`, which holds only 30, every threshold is 30 and `F_b(30)` is 1. + #[test] + fn a_zero_answer_misses_by_the_share() { + let gt = over(vec![0]); + let table = records(); + let truth = gt.truth(&table).expect("the table has the named columns"); + let probes = gt.probes(&truth); + let groups = gt.per_group(&truth, &probes, &vec![0.0; probes.len()]); + let b = groups + .iter() + .find(|g| g.group == "label0:b") + .expect("b is scored"); + assert_eq!(b.error, Some(1.0)); + assert_eq!(b.n_q, 3); + } +} diff --git a/aqpbm-core/src/accuracy/subpopulation/frequency_vector.rs b/aqpbm-core/src/accuracy/subpopulation/frequency_vector.rs index 08421ec4..ae4790c6 100644 --- a/aqpbm-core/src/accuracy/subpopulation/frequency_vector.rs +++ b/aqpbm-core/src/accuracy/subpopulation/frequency_vector.rs @@ -61,32 +61,39 @@ pub(super) fn truth_over( fold: impl Fn(&[u64]) -> f64, ) -> Result { let counts = counts_per_group(table, group_columns, value_column)?; - let exact: HashMap = counts + let exact = counts.iter().map(|(g, c)| (g.clone(), fold(c))).collect(); + let sizes = counts .iter() - .map(|(group, counts)| (group.clone(), fold(counts))) + .map(|(g, c)| (g.clone(), c.iter().sum())) .collect(); + Ok(truth_of(exact, sizes, table.row_num, value_column)) +} +/// The truth over each group's exact statistic and record count, however the +/// statistic was taken. `schema_width` is the label columns before the value. +pub(super) fn truth_of( + exact: HashMap, + sizes: HashMap, + records: u64, + schema_width: usize, +) -> SubpopVectorTruth { let mut by_statistic: Vec<(GroupKey, f64)> = exact.iter().map(|(g, v)| (g.clone(), *v)).collect(); by_statistic.sort_unstable_by(|a, b| b.1.total_cmp(&a.1).then_with(|| a.0.cmp(&b.0))); let sized: Vec<(GroupKey, u64)> = by_statistic .iter() - .map(|(g, _)| { - let size = counts.get(g).map(|c| c.iter().sum()).unwrap_or(0); - (g.clone(), size) - }) + .map(|(g, _)| (g.clone(), sizes.get(g).copied().unwrap_or(0))) .collect(); let all = curve::shuffled(&sized); - let sizes = sized.into_iter().collect(); let ranked = by_statistic.into_iter().map(|(g, _)| g).collect(); - Ok(SubpopVectorTruth { + SubpopVectorTruth { exact, ranked, all, sizes, - records: table.row_num, - schema_width: value_column, - }) + records, + schema_width, + } } pub(super) fn probes_over(truth: &SubpopVectorTruth) -> Vec { diff --git a/aqpbm-core/src/accuracy/subpopulation/mod.rs b/aqpbm-core/src/accuracy/subpopulation/mod.rs index 0f92c223..ce788a2f 100644 --- a/aqpbm-core/src/accuracy/subpopulation/mod.rs +++ b/aqpbm-core/src/accuracy/subpopulation/mod.rs @@ -2,20 +2,24 @@ //! subpopulation, one comparator per statistic. mod cardinality; +mod cdf_error; mod entropy; mod frequency; mod frequency_vector; mod l1_norm; mod l2_norm; mod rank_error; +mod sum; pub use cardinality::{SubpopCardTruth, SubpopCardinalityGT}; +pub use cdf_error::SubpopCdfErrorGT; pub use entropy::{entropy, SubpopEntropyGT}; pub use frequency::{SubpopFreqTruth, SubpopFrequencyGT}; pub use frequency_vector::SubpopVectorTruth; pub use l1_norm::{l1_norm, SubpopL1NormGT}; pub use l2_norm::{l2_norm, SubpopL2NormGT}; pub use rank_error::{SubpopRankErrorGT, SubpopRankTruth}; +pub use sum::SubpopSumGT; use std::collections::BTreeMap; diff --git a/aqpbm-core/src/accuracy/subpopulation/rank_error.rs b/aqpbm-core/src/accuracy/subpopulation/rank_error.rs index 01ec1259..5a3d612f 100644 --- a/aqpbm-core/src/accuracy/subpopulation/rank_error.rs +++ b/aqpbm-core/src/accuracy/subpopulation/rank_error.rs @@ -24,12 +24,12 @@ pub struct SubpopRankErrorGT { /// The ordered statistic inside each group, plus the groups worth probing. pub struct SubpopRankTruth { - per_group: HashMap>, + pub(super) per_group: HashMap>, /// Every group, shuffled: probe order must not hand the baseline the /// locality the encounter order would. - probed: Vec, - items: usize, - schema_width: usize, + pub(super) probed: Vec, + pub(super) items: usize, + pub(super) schema_width: usize, } impl GroundTruth for SubpopRankErrorGT { diff --git a/aqpbm-core/src/accuracy/subpopulation/sum.rs b/aqpbm-core/src/accuracy/subpopulation/sum.rs new file mode 100644 index 00000000..4c997eba --- /dev/null +++ b/aqpbm-core/src/accuracy/subpopulation/sum.rs @@ -0,0 +1,182 @@ +//! The sum of the values inside one subpopulation: the weighted L1 norm, each +//! value its own weight. + +use std::collections::BTreeMap; +use std::collections::HashMap; + +use aqpbm_datagen::{ColumnData, DataGenError, GeneratedTable}; + +use super::frequency_vector::{per_group_over, probes_over, score_over, truth_of}; +use super::{group_labels, owned, GroupKey, SubpopVectorTruth}; +use crate::accuracy::{GroundTruth, GroupError}; + +pub struct SubpopSumGT { + /// Which label columns the subpopulation is taken over. + pub group_columns: Vec, + pub value_column: usize, +} + +/// Each group's sum, over its record count. The value is an integer weight, +/// so an `f64` or text value column is refused by its type. +fn sums_per_group( + table: &GeneratedTable, + group_columns: &[usize], + value_column: usize, +) -> Result { + let labels = group_labels(table, group_columns)?; + let values: Vec = match table.column(value_column)? { + ColumnData::Int64(v) => v.iter().map(|&x| x as f64).collect(), + ColumnData::Unsigned64(v) => v.iter().map(|&x| x as f64).collect(), + other => { + return Err(DataGenError::TypeMismatch { + held: other.kind(), + wanted: "an integer value column (i64 or u64), the weights a sum adds", + }) + } + }; + let mut per_group: HashMap, (f64, u64)> = HashMap::new(); + for (group, value) in labels.chunks(group_columns.len()).zip(&values) { + match per_group.get_mut(group) { + Some((sum, n)) => { + *sum += value; + *n += 1; + } + None => { + per_group.insert(group.to_vec(), (*value, 1)); + } + } + } + let (mut sums, mut sizes) = (HashMap::new(), HashMap::new()); + for (group, (sum, n)) in per_group { + let key = owned(&group, group_columns); + sums.insert(key.clone(), sum); + sizes.insert(key, n); + } + Ok(truth_of(sums, sizes, table.row_num, value_column)) +} + +impl GroundTruth for SubpopSumGT { + type Truth = SubpopVectorTruth; + type Probe = GroupKey; + type Answer = f64; + + fn truth(&self, table: &GeneratedTable) -> Result { + sums_per_group(table, &self.group_columns, self.value_column) + } + + fn probes(&self, truth: &SubpopVectorTruth) -> Vec { + probes_over(truth) + } + + fn score( + &self, + truth: &SubpopVectorTruth, + probes: &[GroupKey], + answers: &[f64], + ) -> BTreeMap { + score_over(truth, probes, answers, self.group_columns.len()) + } + + fn per_group( + &self, + truth: &SubpopVectorTruth, + probes: &[GroupKey], + answers: &[f64], + ) -> Vec { + per_group_over(truth, probes, answers) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::accuracy::subpopulation::{exact_over, null_over, records}; + + fn over(group_columns: Vec) -> SubpopSumGT { + SubpopSumGT { + group_columns, + value_column: 2, + } + } + + /// `a` carries 10, 10 and 20; `b` 30 three times. Its L1 sibling would + /// answer 3 for both, the record count. + #[test] + fn the_sum_of_a_group_adds_its_values() { + let truth = over(vec![0]) + .truth(&records()) + .expect("the table has the named columns"); + assert_eq!(truth.exact[&vec![Some("a".to_string())]], 40.0); + assert_eq!(truth.exact[&vec![Some("b".to_string())]], 90.0); + let probes = over(vec![0]).probes(&truth); + let groups = over(vec![0]).per_group(&truth, &probes, &vec![0.0; probes.len()]); + let sizes: Vec = groups.iter().map(|g| g.n_q).collect(); + assert_eq!(sizes, [3, 3], "n_q is the group's records, not its sum"); + } + + #[test] + fn null_estimator_scores_exactly_one_on_are() { + let cmp = null_over(&over(vec![0]), &records()); + assert!((cmp["are_all"] - 1.0).abs() < 1e-12, "{}", cmp["are_all"]); + } + + #[test] + fn exact_estimator_scores_zero() { + let cmp = exact_over(&over(vec![0, 1]), &records()); + assert_eq!(cmp["are_all"], 0.0); + assert_eq!(cmp["err_max"], 0.0); + } + + /// A group whose values sum to zero has no relative error: it is listed + /// unscored, as a group with no distinct value is for cardinality. + #[test] + fn a_zero_sum_group_is_listed_unscored() { + use crate::accuracy::table_of; + let table = table_of( + &["key1", "value"], + vec![ + ColumnData::String(vec!["a".into(), "b".into(), "b".into()]), + ColumnData::Int64(vec![4, 0, 0]), + ], + ); + let gt = SubpopSumGT { + group_columns: vec![0], + value_column: 1, + }; + let truth = gt.truth(&table).expect("the table has the named columns"); + let probes = gt.probes(&truth); + let rows: Vec<(String, u64, Option)> = gt + .per_group(&truth, &probes, &vec![0.0; probes.len()]) + .into_iter() + .map(|g| (g.group, g.n_q, g.error)) + .collect(); + assert_eq!( + rows, + vec![ + ("label0:b".to_string(), 2, None), + ("label0:a".to_string(), 1, Some(1.0)), + ] + ); + } + + #[test] + fn a_float_value_column_is_refused_by_its_type() { + use crate::accuracy::table_of; + let table = table_of( + &["key1", "value"], + vec![ + ColumnData::String(vec!["a".into()]), + ColumnData::Float64(vec![1.5]), + ], + ); + let gt = SubpopSumGT { + group_columns: vec![0], + value_column: 1, + }; + let Err(err) = gt.truth(&table).map(|_| ()) else { + panic!("an f64 value column must be refused, not summed"); + }; + let err = err.to_string(); + assert!(err.contains("f64") && err.contains("integer"), "{err}"); + } +} diff --git a/configs/datagen/hydra_columns_f64.yaml b/configs/datagen/hydra_columns_f64.yaml index 15b54921..3e738c25 100644 --- a/configs/datagen/hydra_columns_f64.yaml +++ b/configs/datagen/hydra_columns_f64.yaml @@ -1,6 +1,6 @@ -# `hydra_columns.yaml` with an `f64` value column, for the record-ingesting row -# whose item is a float: `hydra-kll`, which orders its values rather than -# counting them. +# `hydra_columns.yaml` with an `f64` value column, for the record-ingesting rows +# whose item is a float: `hydra-kll` and `hydra-kll-cdf`, which order their +# values rather than count them. # # approxbench sketchbench --variant hydra-kll --library lib \ # --spec configs/datagen/hydra_columns_f64.yaml \ diff --git a/configs/datagen/hydra_univmon_columns.yaml b/configs/datagen/hydra_univmon_columns.yaml index adad591e..19eb3160 100644 --- a/configs/datagen/hydra_univmon_columns.yaml +++ b/configs/datagen/hydra_univmon_columns.yaml @@ -1,5 +1,5 @@ # `hydra_columns.yaml` at a row count the UnivMon-celled rows can run: -# `hydra-univmon-cardinality`, `-l1-norm`, `-l2-norm` and `-entropy`. The label +# `hydra-univmon-cardinality`, `-l1-norm`, `-l2-norm`, `-entropy` and `-sum`. The label # columns and every distribution parameter match that file exactly, so the # workload is the same shape read at a smaller size. # diff --git a/docs/concepts.md b/docs/concepts.md index 0c5d31cd..e864c618 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -58,9 +58,11 @@ heavy-hitter heavy items and how heavy subpop-cardinality cardinality, within the records carrying a set of labels subpop-frequency frequency, within the records carrying a set of labels subpop-quantile quantile, within the records carrying a set of labels +subpop-cdf share at or below a value, within the records carrying a set of labels subpop-l1-norm L1-norm of values, within the records carrying a set of labels subpop-l2-norm L2-norm of values, within the records carrying a set of labels subpop-entropy entropy of values, within the records carrying a set of labels +subpop-sum sum of values, within the records carrying a set of labels keyed-cardinality how many distinct items for a specific key keyed-entropy entropy of values for a specific key keyed-l1-norm L1-norm of values for a specific key diff --git a/sketch-bench/src/registry.rs b/sketch-bench/src/registry.rs index 05bcf08a..527e8cc0 100644 --- a/sketch-bench/src/registry.rs +++ b/sketch-bench/src/registry.rs @@ -678,6 +678,38 @@ pub const REGISTRY: &[SketchId] = &[ .union(MetricsMask::MEMORY) .union(MetricsMask::ACCURACY), }, + SketchId { + algorithm: "hydra-kll", + variant: "hydra-kll-cdf", + library: "lib", + description: "asap_sketchlib::Hydra over KLL cells: Cdf(x) per subpopulation", + capability: Capability::SubpopCdf, + comparator: Some("subpop-cdf-error"), + operations: OperationMask::INSERT + .union(OperationMask::QUERY) + .union(OperationMask::MERGE), + metrics: MetricsMask::THROUGHPUT + .union(MetricsMask::LATENCY) + .union(MetricsMask::CPU) + .union(MetricsMask::MEMORY) + .union(MetricsMask::ACCURACY), + }, + SketchId { + algorithm: "hydra-kll", + variant: "hydra-kll-cdf", + library: "polars", + description: "polars exact: sorted values per label subset, share at or below x", + capability: Capability::SubpopCdf, + comparator: Some("subpop-cdf-error"), + operations: OperationMask::INSERT + .union(OperationMask::QUERY) + .union(OperationMask::PREPARE), + metrics: MetricsMask::THROUGHPUT + .union(MetricsMask::LATENCY) + .union(MetricsMask::CPU) + .union(MetricsMask::MEMORY) + .union(MetricsMask::ACCURACY), + }, SketchId { algorithm: "hydra-univmon", variant: "hydra-univmon-cardinality", @@ -806,6 +838,38 @@ pub const REGISTRY: &[SketchId] = &[ .union(MetricsMask::MEMORY) .union(MetricsMask::ACCURACY), }, + SketchId { + algorithm: "hydra-univmon", + variant: "hydra-univmon-sum", + library: "lib", + description: "asap_sketchlib::Hydra over UnivMon cells, each value inserted with itself as count: calc_l1 per subpopulation", + capability: Capability::SubpopSum, + comparator: Some("subpop-sum"), + operations: OperationMask::INSERT + .union(OperationMask::QUERY) + .union(OperationMask::MERGE), + metrics: MetricsMask::THROUGHPUT + .union(MetricsMask::LATENCY) + .union(MetricsMask::CPU) + .union(MetricsMask::MEMORY) + .union(MetricsMask::ACCURACY), + }, + SketchId { + algorithm: "hydra-univmon", + variant: "hydra-univmon-sum", + library: "polars", + description: "polars exact: group_by(subset).agg(v.sum()) over every label subset", + capability: Capability::SubpopSum, + comparator: Some("subpop-sum"), + operations: OperationMask::INSERT + .union(OperationMask::QUERY) + .union(OperationMask::PREPARE), + metrics: MetricsMask::THROUGHPUT + .union(MetricsMask::LATENCY) + .union(MetricsMask::CPU) + .union(MetricsMask::MEMORY) + .union(MetricsMask::ACCURACY), + }, SketchId { algorithm: "univmon", variant: "univmon-cardinality", @@ -1360,9 +1424,11 @@ mod tests { "subpop-cardinality", "subpop-frequency", "subpop-rank-error", + "subpop-cdf-error", "subpop-l1-norm", "subpop-l2-norm", "subpop-entropy", + "subpop-sum", "keyed-cardinality", "keyed-l1-norm", "keyed-l2-norm", diff --git a/sketch-bench/src/request.rs b/sketch-bench/src/request.rs index 421990a7..478bfc1f 100644 --- a/sketch-bench/src/request.rs +++ b/sketch-bench/src/request.rs @@ -16,9 +16,11 @@ pub enum Capability { SubpopCardinality, SubpopFrequency, SubpopQuantile, + SubpopCdf, SubpopL1Norm, SubpopL2Norm, SubpopEntropy, + SubpopSum, KeyedCardinality, KeyedL1Norm, KeyedL2Norm, @@ -44,9 +46,11 @@ impl Capability { Capability::SubpopCardinality => "subpop-cardinality", Capability::SubpopFrequency => "subpop-frequency", Capability::SubpopQuantile => "subpop-quantile", + Capability::SubpopCdf => "subpop-cdf", Capability::SubpopL1Norm => "subpop-l1-norm", Capability::SubpopL2Norm => "subpop-l2-norm", Capability::SubpopEntropy => "subpop-entropy", + Capability::SubpopSum => "subpop-sum", Capability::KeyedCardinality => "keyed-cardinality", Capability::KeyedL1Norm => "keyed-l1-norm", Capability::KeyedL2Norm => "keyed-l2-norm", diff --git a/sketch-bench/src/wrappers/hydra_kll/mod.rs b/sketch-bench/src/wrappers/hydra_kll/mod.rs index de6f74f6..35f07656 100644 --- a/sketch-bench/src/wrappers/hydra_kll/mod.rs +++ b/sketch-bench/src/wrappers/hydra_kll/mod.rs @@ -11,9 +11,11 @@ pub(crate) use super::hydra_shared::{ check_grid, grid_overhead_bytes, label_columns, labels, new_hydra, query, update, }; -pub(crate) use crate::wrappers::kll::{kll_lib_slots as kll_cell_slots, KLL_LIB_MAX_LEVELS}; +pub(crate) use crate::wrappers::kll::kll_lib_bytes as kll_cell_bytes; #[cfg(test)] -use crate::wrappers::kll::{KLL_LIB_MAX_CACHEABLE_K, KLL_LIB_MIN_LEVEL}; +use crate::wrappers::kll::{ + kll_lib_slots as kll_cell_slots, KLL_LIB_MAX_CACHEABLE_K, KLL_LIB_MAX_LEVELS, KLL_LIB_MIN_LEVEL, +}; #[cfg(test)] mod tests { @@ -103,6 +105,34 @@ mod tests { ); } + /// A cell is a whole `KLL`, merge buffer included, as the `kll-*` lib rows + /// count it. Hand-derived: at `k = 200`, 1004 retained slots (above) plus + /// 200 buffer items, 8 bytes each, plus 62 level offsets. + #[test] + fn kll_footprint_counts_each_cells_merge_buffer() { + let per_cell = (1004 + 200) * 8 + 62 * std::mem::size_of::(); + assert_eq!( + memory_hydra_kll(&built_kll()), + 3 * 64 * per_cell + grid_overhead_bytes(3, 64) + ); + } + + /// Below `k` the cell keeps every value, so `Cdf(x)` is the exact share at + /// or below `x`, inside the group only. + #[test] + fn kll_cdf_is_the_share_at_or_below_inside_the_group() { + let mut h = built_kll(); + for v in 1..=100 { + fed_kll(&mut h, &frecord("a;x", v as f64)); + } + for _ in 0..300 { + fed_kll(&mut h, &frecord("b;x", 0.5)); + } + assert_eq!(h.estimate_subpop_cdf(&[Some("a")], 25.0), 0.25); + assert_eq!(h.estimate_subpop_cdf(&[Some("a")], 100.0), 1.0); + assert_eq!(h.estimate_subpop_cdf(&[Some("a")], 0.9), 0.0); + } + #[test] fn kll_zero_cell_k_is_refused_by_name() { let bad = ParamSet::of(&HydraKllParams { @@ -119,3 +149,30 @@ mod tests { ); } } + +#[cfg(test)] +mod baseline_tests { + use super::polars::*; + use crate::params::{ParamSet, SketchParams}; + use crate::wrappers::hydra_kll::HydraKllParams; + use std::rc::Rc; + + /// `a` holds 10, 10, 20: `F_a(10)` counts both tens, `F_a(15)` too, and + /// `F_a(5)` nothing. A group never seen answers NaN, not a share. + #[test] + fn the_cdf_baseline_answers_the_share_at_or_below() { + let stream = Rc::new(vec![ + ("a;x".to_string(), 10.0), + ("a;y".to_string(), 10.0), + ("a;x".to_string(), 20.0), + ("b;x".to_string(), 30.0), + ]); + let a = || vec![Some("a".to_string())]; + let probes = Rc::new(vec![(a(), 10.0), (a(), 15.0), (a(), 5.0), (a(), 20.0)]); + let params = ParamSet::of(&HydraKllParams::canonical()); + let mut passes = + query_polars_subpop_cdf::(¶ms, stream, probes, 1).expect("baseline builds"); + let answers = passes.pop().expect("one pass")().0; + assert_eq!(answers, vec![2.0 / 3.0, 2.0 / 3.0, 0.0, 1.0]); + } +} diff --git a/sketch-bench/src/wrappers/hydra_kll/polars.rs b/sketch-bench/src/wrappers/hydra_kll/polars.rs index 11a6a268..57094c5b 100644 --- a/sketch-bench/src/wrappers/hydra_kll/polars.rs +++ b/sketch-bench/src/wrappers/hydra_kll/polars.rs @@ -56,6 +56,18 @@ impl PolarsSubpopQuantile { let idx = ((phi * values.len() as f64).floor() as usize).min(values.len() - 1); values[idx] } + + /// The share of the group at or below `x`: exact, so the CDF-error + /// comparator scores it zero. + pub fn estimate_subpop_cdf(&self, labels: &[Option<&str>], x: f64) -> f64 { + let Some(values) = self.sorted.get(&group_key(labels)) else { + return f64::NAN; + }; + if values.is_empty() { + return f64::NAN; + } + values.partition_point(|v| *v <= x) as f64 / values.len() as f64 + } } pub fn memory_polars_subpop_quantile( @@ -159,6 +171,38 @@ pub fn query_polars_subpop_quantile>, probes: Rc>, f64)>>, passes: usize, +) -> Result>, BuildError> { + asked_polars_subpop( + params, + items, + probes, + passes, + PolarsSubpopQuantile::estimate_subpop_quantile, + ) +} + +/// The `hydra-kll-cdf` baseline: the same sorted groups, asked at a value. +pub fn query_polars_subpop_cdf( + params: &ParamSet, + items: Rc>, + probes: Rc>, f64)>>, + passes: usize, +) -> Result>, BuildError> { + asked_polars_subpop( + params, + items, + probes, + passes, + PolarsSubpopQuantile::estimate_subpop_cdf, + ) +} + +fn asked_polars_subpop( + params: &ParamSet, + items: Rc>, + probes: Rc>, f64)>>, + passes: usize, + estimate: fn(&PolarsSubpopQuantile, &[Option<&str>], f64) -> f64, ) -> Result>, BuildError> { let mut out = Vec::with_capacity(passes); for _ in 0..passes { @@ -172,7 +216,7 @@ pub fn query_polars_subpop_quantile], phi: f64) -> f64 { query(&self.inner, labels, &HydraQuery::Quantile(phi), "hydra-kll") } + + /// `HydraQuery::Cdf`: the share of the group at or below `x`, the question + /// the `hydra-kll-cdf` row asks. + #[inline] + pub fn estimate_subpop_cdf(&self, labels: &[Option<&str>], x: f64) -> f64 { + query(&self.inner, labels, &HydraQuery::Cdf(x), "hydra-kll") + } } -/// Retained slots per cell times the grid area, plus the level index every -/// cell carries. Analytic because the cell allocates once — see -/// `kll_lib_slots` in this crate's `kll` module for the per-cell count. Unlike -/// the `kll-percall` lib rows (`kll_lib_bytes`), each cell's merge buffer of -/// `cell_k` items is not counted. +/// One `asap_sketchlib::KLL` per cell times the grid area: its retained +/// slots, level index and merge buffer of `cell_k` items, as the `kll-*` lib +/// rows count them (`kll_lib_bytes`). Analytic because the cell allocates once. pub fn memory_hydra_kll(sketch: &HydraKll) -> usize { let p = &sketch.params; - let per_cell = kll_cell_slots(p.cell_k) * std::mem::size_of::() - + (KLL_LIB_MAX_LEVELS + 1) * std::mem::size_of::(); - p.rows * p.cols * per_cell + grid_overhead_bytes(p.rows, p.cols) + p.rows * p.cols * kll_cell_bytes::(p.cell_k) + grid_overhead_bytes(p.rows, p.cols) } pub fn insert_hydra_kll( @@ -126,6 +129,52 @@ pub fn merge_query_hydra_kll( probes: Rc>, f64)>>, shards: usize, passes: usize, +) -> Result>, BuildError> { + asked_hydra_kll( + params, + items, + probes, + shards, + passes, + HydraKll::estimate_subpop_quantile, + ) +} + +pub fn query_hydra_kll_cdf( + params: &ParamSet, + items: Rc>, + probes: Rc>, f64)>>, + passes: usize, +) -> Result>, BuildError> { + merge_query_hydra_kll_cdf(params, items, probes, 1, passes) +} + +/// The query, asked of the sketch a fold over `shards` shards leaves. One +/// shard is the plain query. +pub fn merge_query_hydra_kll_cdf( + params: &ParamSet, + items: Rc>, + probes: Rc>, f64)>>, + shards: usize, + passes: usize, +) -> Result>, BuildError> { + asked_hydra_kll( + params, + items, + probes, + shards, + passes, + HydraKll::estimate_subpop_cdf, + ) +} + +fn asked_hydra_kll( + params: &ParamSet, + items: Rc>, + probes: Rc>, f64)>>, + shards: usize, + passes: usize, + estimate: fn(&HydraKll, &[Option<&str>], f64) -> f64, ) -> Result>, BuildError> { let mut out = Vec::with_capacity(passes); for _ in 0..passes { @@ -141,7 +190,7 @@ pub fn merge_query_hydra_kll( out.push(Box::new(move || { let mut answers = Vec::with_capacity(probes.len()); for p in probes.iter() { - answers.push(sketch.estimate_subpop_quantile(&labels(&p.0), p.1)); + answers.push(estimate(&sketch, &labels(&p.0), p.1)); } let footprint = memory_hydra_kll(&sketch); (answers, footprint) diff --git a/sketch-bench/src/wrappers/hydra_shared.rs b/sketch-bench/src/wrappers/hydra_shared.rs index 1af9d7ef..5f1c3d47 100644 --- a/sketch-bench/src/wrappers/hydra_shared.rs +++ b/sketch-bench/src/wrappers/hydra_shared.rs @@ -44,8 +44,20 @@ pub(crate) fn new_hydra( /// Records one `;`-joined record, one value per key column in place. The /// build already refused a stream of mixed widths; this is the backstop. pub(crate) fn update(h: &mut Hydra, key: &str, value: &DataInput, variant: &str) { + update_counted(h, key, value, None, variant) +} + +/// [`update`], the value counted `count` times in each cell it reaches +/// (`None` is once): the library's integer weight. +pub(crate) fn update_counted( + h: &mut Hydra, + key: &str, + value: &DataInput, + count: Option, + variant: &str, +) { let parts: Vec<&str> = key.split(';').collect(); - if let Err(e) = h.update(&parts, value, None) { + if let Err(e) = h.update(&parts, value, count) { panic!( "{variant}: record {key:?} has {} label(s), the grid {}: {e}", parts.len(), diff --git a/sketch-bench/src/wrappers/hydra_univmon/mod.rs b/sketch-bench/src/wrappers/hydra_univmon/mod.rs index ab6073c8..8ff87bc2 100644 --- a/sketch-bench/src/wrappers/hydra_univmon/mod.rs +++ b/sketch-bench/src/wrappers/hydra_univmon/mod.rs @@ -123,6 +123,52 @@ mod tests { ); } + /// Each record counted by its own value makes the group's L1 its sum. At + /// a grid far wider than the four subkeys, every cell holds one group. + #[test] + fn weighted_l1_of_a_group_is_its_sum() { + let params = ParamSet::of(&HydraUnivmonParams { + rows: 3, + cols: 1024, + cell_heap_size: 32, + cell_sketch_row: 5, + cell_sketch_col: 256, + cell_layer_size: 4, + }); + let stream = std::rc::Rc::new(vec![ + record("a;x", 10), + record("a;y", 10), + record("a;x", 25), + record("b;x", 7), + ]); + let probes = std::rc::Rc::new(vec![ + vec![Some("a".to_string())], + vec![Some("b".to_string())], + vec![None, Some("x".to_string())], + ]); + let mut passes = query_hydra_univmon_sum(¶ms, stream, probes, 1).expect("builds"); + let answers = passes.pop().expect("one pass")().0; + assert_eq!(answers, vec![45.0, 7.0, 42.0]); + } + + /// The library's weight is an `i32` count: a value it cannot carry is + /// refused by name, not truncated into a different sum. + #[test] + fn a_value_past_an_i32_count_is_refused_by_name() { + let params = ParamSet::of(&HydraUnivmonParams::canonical()); + for bad in [i64::from(i32::MAX) + 1, -1] { + let stream = std::rc::Rc::new(vec![record("a;x", 1), record("a;x", bad)]); + let Err(err) = insert_hydra_univmon_sum(¶ms, stream, 1) else { + panic!("{bad} must be refused, not inserted"); + }; + let err = err.to_string(); + assert!( + err.contains("hydra-univmon-sum") && err.contains(&bad.to_string()), + "{err}" + ); + } + } + #[test] fn zero_dimensions_are_refused_by_name() { let bad = ParamSet::of(&HydraUnivmonParams { @@ -198,6 +244,15 @@ mod baseline_tests { ); } + /// `a` holds 10, 10, 20; its L1 sibling answers 3 for it. + #[test] + fn the_sum_baseline_answers_the_group_sum() { + assert_eq!( + answers_for(query_polars_subpop_sum::), + vec![40.0, 30.0] + ); + } + #[test] fn the_cardinality_baseline_answers_distinct_values() { assert_eq!( diff --git a/sketch-bench/src/wrappers/hydra_univmon/polars.rs b/sketch-bench/src/wrappers/hydra_univmon/polars.rs index 0e9dd152..a4f32f46 100644 --- a/sketch-bench/src/wrappers/hydra_univmon/polars.rs +++ b/sketch-bench/src/wrappers/hydra_univmon/polars.rs @@ -8,8 +8,10 @@ use crate::params::ParamSet; use crate::wrappers::hll::CardinalityValue; use crate::wrappers::polars_shared::*; use crate::wrappers::{BuildError, Pass, QueryPass, Shared, StepPass}; +use ::polars::prelude::*; use aqpbm_core::accuracy::subpopulation::{entropy, l1_norm, l2_norm}; use std::cell::RefCell; +use std::collections::HashMap; use std::rc::Rc; pub struct PolarsSubpopUnivmon(PolarsSubpopVectorCore); @@ -353,3 +355,145 @@ pub fn prepare_polars_subpop_entropy( ) -> Result, BuildError> { prepare_polars_subpop_vector(params, items, passes, entropy) } + +/// `hydra-univmon-sum/polars` — the exact sum of each label subset's values. +pub struct PolarsSubpopSum { + buf: Vec<(String, T)>, + exact: HashMap, +} + +// Hand-written rather than derived, for the reason the cardinality baseline's is. +impl Default for PolarsSubpopSum { + fn default() -> Self { + Self { + buf: Vec::new(), + exact: HashMap::new(), + } + } +} + +pub fn build_polars_subpop_sum( + config: &ParamSet, +) -> Result, BuildError> { + // Parsed and discarded, as the other baselines here do. + let _p: HydraUnivmonParams = config.parse()?; + Ok(PolarsSubpopSum::default()) +} + +impl PolarsSubpopSum { + /// The exact answer, over the stream as buffered. Its own step because the + /// runner times it as `prepare`: this row's cost is the pass, not the ask. + fn finalize(&mut self) { + let Some(result) = grouped_values(&self.buf, |lazy| { + lazy.group_by([col("g")]).agg([col("v").sum().alias("sum")]) + }) else { + return; + }; + let groups = result.column("g").expect("g column"); + let groups = groups.str().expect("str groups"); + let sums = result + .column("sum") + .expect("sum column") + .cast(&DataType::Float64) + .expect("cast to f64"); + let sums = sums.f64().expect("f64 sums"); + self.exact = groups + .into_iter() + .zip(sums) + .filter_map(|(g, s)| Some((g?.to_string(), s?))) + .collect(); + } + + pub fn estimate_subpop_sum(&self, labels: &[Option<&str>]) -> f64 { + self.exact.get(&group_key(labels)).copied().unwrap_or(0.0) + } +} + +pub fn memory_polars_subpop_sum(sketch: &PolarsSubpopSum) -> usize { + let held: usize = sketch.buf.iter().map(|(g, _)| g.capacity()).sum(); + sketch.buf.capacity() * std::mem::size_of::<(String, T)>() + + held + + sketch.exact.capacity() * (std::mem::size_of::() + std::mem::size_of::()) +} + +pub fn insert_polars_subpop_sum( + params: &ParamSet, + items: Rc>, + passes: usize, +) -> Result, BuildError> { + let mut out = Vec::with_capacity(passes); + for _ in 0..passes { + // Built here, so calling the closure is the insert and nothing else. + let mut sketch = build_polars_subpop_sum::(params)?; + let items = items.clone(); + out.push(Box::new(move || { + for v in items.iter() { + sketch.buf.push(v.clone()); + } + memory_polars_subpop_sum(&sketch) + }) as Pass); + } + Ok(out) +} + +pub fn insert_step_polars_subpop_sum( + params: &ParamSet, + items: Rc>, + passes: usize, +) -> Result, BuildError> { + let mut out = Vec::with_capacity(passes); + for _ in 0..passes { + let sketch: Shared<_> = Rc::new(RefCell::new(build_polars_subpop_sum::(params)?)); + let (driven, read) = (sketch.clone(), sketch); + let stream = items.clone(); + out.push(StepPass { + steps: items.len(), + step: Box::new(move |i| driven.borrow_mut().buf.push(stream[i].clone())), + footprint: Box::new(move || memory_polars_subpop_sum(&read.borrow())), + }); + } + Ok(out) +} + +pub fn query_polars_subpop_sum( + params: &ParamSet, + items: Rc>, + probes: Rc>>>, + passes: usize, +) -> Result>, BuildError> { + let mut out = Vec::with_capacity(passes); + for _ in 0..passes { + // Built and fed here: the closure below asks, and only asks. + let mut sketch = build_polars_subpop_sum::(params)?; + sketch.buf.extend(items.iter().cloned()); + sketch.finalize(); + let probes = probes.clone(); + out.push(Box::new(move || { + let mut answers = Vec::with_capacity(probes.len()); + for p in probes.iter() { + answers.push(sketch.estimate_subpop_sum(&labels(p))); + } + let footprint = memory_polars_subpop_sum(&sketch); + (answers, footprint) + }) as QueryPass); + } + Ok(out) +} + +pub fn prepare_polars_subpop_sum( + params: &ParamSet, + items: Rc>, + passes: usize, +) -> Result, BuildError> { + let mut out = Vec::with_capacity(passes); + for _ in 0..passes { + // Fed here: the closure below is the step that makes it ready to answer. + let mut sketch = build_polars_subpop_sum::(params)?; + sketch.buf.extend(items.iter().cloned()); + out.push(Box::new(move || { + sketch.finalize(); + memory_polars_subpop_sum(&sketch) + }) as Pass); + } + Ok(out) +} diff --git a/sketch-bench/src/wrappers/hydra_univmon/sketchlib.rs b/sketch-bench/src/wrappers/hydra_univmon/sketchlib.rs index e7465088..d9760048 100644 --- a/sketch-bench/src/wrappers/hydra_univmon/sketchlib.rs +++ b/sketch-bench/src/wrappers/hydra_univmon/sketchlib.rs @@ -6,6 +6,7 @@ use super::*; use crate::params::ParamSet; use crate::wrappers::hll::CardinalityValue; +use crate::wrappers::hydra_shared::update_counted; use crate::wrappers::partition; use crate::wrappers::{BuildError, Pass, QueryPass, Shared, StepPass}; use asap_sketchlib::input::{HHItem, HydraCounter, HydraQuery}; @@ -82,10 +83,80 @@ pub fn memory_hydra_univmon(sketch: &HydraUnivmon) -> usize { p.rows * p.cols * p.cell_layer_size * (counters + heap) + grid_overhead_bytes(p.rows, p.cols) } +/// A value the `-sum` row inserts as its own count, the library's integer +/// weight: `None` for one an `i32` weight cannot carry, or a negative one. +pub trait WeightValue: CardinalityValue { + fn weight(&self) -> Option; +} + +impl WeightValue for i64 { + fn weight(&self) -> Option { + i32::try_from(*self).ok().filter(|w| *w >= 0) + } +} + +impl WeightValue for u64 { + fn weight(&self) -> Option { + i32::try_from(*self).ok() + } +} + +/// How one record enters the grid: its value as the key, counted once +/// ([`counted`]) or counted by itself ([`weighted`]). +type Feed = fn(&mut HydraUnivmon, &(String, V)); + +fn counted(sketch: &mut HydraUnivmon, r: &(String, V)) { + update(&mut sketch.inner, &r.0, &r.1.data_input(), "hydra-univmon"); +} + +/// Weighted L1 is the group's sum. Every weight was checked by [`weights`] +/// before the grid was built. +fn weighted(sketch: &mut HydraUnivmon, r: &(String, V)) { + let count = r.1.weight().expect("weights() checked every value"); + update_counted( + &mut sketch.inner, + &r.0, + &r.1.data_input(), + Some(count), + "hydra-univmon-sum", + ); +} + +/// Refuses a stream with a value [`WeightValue::weight`] cannot carry, naming +/// it, so no sum is built over a truncated weight. +fn weights(items: &[(String, V)]) -> Result<(), BuildError> { + match items.iter().find(|r| r.1.weight().is_none()) { + Some(r) => Err(BuildError(format!( + "hydra-univmon-sum: value {:?} is not a count in [0, {}], the library's i32 weight", + r.1, + i32::MAX + ))), + None => Ok(()), + } +} + pub fn insert_hydra_univmon( params: &ParamSet, items: Rc>, passes: usize, +) -> Result, BuildError> { + inserted(params, items, passes, counted::) +} + +pub fn insert_hydra_univmon_sum( + params: &ParamSet, + items: Rc>, + passes: usize, +) -> Result, BuildError> { + weights(&items)?; + inserted(params, items, passes, weighted::) +} + +fn inserted( + params: &ParamSet, + items: Rc>, + passes: usize, + feed: Feed, ) -> Result, BuildError> { let mut out = Vec::with_capacity(passes); for _ in 0..passes { @@ -94,7 +165,7 @@ pub fn insert_hydra_univmon( let items = items.clone(); out.push(Box::new(move || { for v in items.iter() { - update(&mut sketch.inner, &v.0, &v.1.data_input(), "hydra-univmon"); + feed(&mut sketch, v); } memory_hydra_univmon(&sketch) }) as Pass); @@ -106,6 +177,24 @@ pub fn insert_step_hydra_univmon( params: &ParamSet, items: Rc>, passes: usize, +) -> Result, BuildError> { + inserted_step(params, items, passes, counted::) +} + +pub fn insert_step_hydra_univmon_sum( + params: &ParamSet, + items: Rc>, + passes: usize, +) -> Result, BuildError> { + weights(&items)?; + inserted_step(params, items, passes, weighted::) +} + +fn inserted_step( + params: &ParamSet, + items: Rc>, + passes: usize, + feed: Feed, ) -> Result, BuildError> { let mut out = Vec::with_capacity(passes); for _ in 0..passes { @@ -118,9 +207,7 @@ pub fn insert_step_hydra_univmon( out.push(StepPass { steps: items.len(), step: Box::new(move |i| { - let sketch = &mut *driven.borrow_mut(); - let v = &stream[i]; - update(&mut sketch.inner, &v.0, &v.1.data_input(), "hydra-univmon"); + feed(&mut driven.borrow_mut(), &stream[i]); }), footprint: Box::new(move || memory_hydra_univmon(&read.borrow())), }); @@ -134,12 +221,13 @@ fn asked_hydra_univmon( probes: Rc>>>, shards: usize, passes: usize, + feed: Feed, estimate: fn(&HydraUnivmon, &[Option<&str>]) -> f64, ) -> Result>, BuildError> { let mut out = Vec::with_capacity(passes); for _ in 0..passes { // Built, fed and folded here: the closure below asks, and only asks. - let (mut sketch, rest) = hydra_univmon_shards(params, &items, shards)?; + let (mut sketch, rest) = hydra_univmon_shards(params, &items, shards, feed)?; for other in rest.iter() { sketch .inner @@ -171,6 +259,7 @@ pub fn query_hydra_univmon_cardinality( probes, 1, passes, + counted::, HydraUnivmon::estimate_subpop_cardinality, ) } @@ -190,6 +279,7 @@ pub fn merge_query_hydra_univmon_cardinality( probes, shards, passes, + counted::, HydraUnivmon::estimate_subpop_cardinality, ) } @@ -206,6 +296,7 @@ pub fn query_hydra_univmon_l1_norm( probes, 1, passes, + counted::, HydraUnivmon::estimate_subpop_l1_norm, ) } @@ -225,6 +316,7 @@ pub fn merge_query_hydra_univmon_l1_norm( probes, shards, passes, + counted::, HydraUnivmon::estimate_subpop_l1_norm, ) } @@ -241,6 +333,7 @@ pub fn query_hydra_univmon_l2_norm( probes, 1, passes, + counted::, HydraUnivmon::estimate_subpop_l2_norm, ) } @@ -260,6 +353,7 @@ pub fn merge_query_hydra_univmon_l2_norm( probes, shards, passes, + counted::, HydraUnivmon::estimate_subpop_l2_norm, ) } @@ -276,6 +370,7 @@ pub fn query_hydra_univmon_entropy( probes, 1, passes, + counted::, HydraUnivmon::estimate_subpop_entropy, ) } @@ -295,19 +390,72 @@ pub fn merge_query_hydra_univmon_entropy( probes, shards, passes, + counted::, HydraUnivmon::estimate_subpop_entropy, ) } +/// The group's sum: the L1 norm of a grid whose records were each counted by +/// their own value. +pub fn query_hydra_univmon_sum( + params: &ParamSet, + items: Rc>, + probes: Rc>>>, + passes: usize, +) -> Result>, BuildError> { + merge_query_hydra_univmon_sum(params, items, probes, 1, passes) +} + +/// The query, asked of the sketch a fold over `shards` shards leaves. One +/// shard is the plain query. +pub fn merge_query_hydra_univmon_sum( + params: &ParamSet, + items: Rc>, + probes: Rc>>>, + shards: usize, + passes: usize, +) -> Result>, BuildError> { + weights(&items)?; + asked_hydra_univmon( + params, + items, + probes, + shards, + passes, + weighted::, + HydraUnivmon::estimate_subpop_l1_norm, + ) +} + pub fn merge_hydra_univmon( params: &ParamSet, items: Rc>, shards: usize, passes: usize, +) -> Result, BuildError> { + merged(params, items, shards, passes, counted::) +} + +pub fn merge_hydra_univmon_sum( + params: &ParamSet, + items: Rc>, + shards: usize, + passes: usize, +) -> Result, BuildError> { + weights(&items)?; + merged(params, items, shards, passes, weighted::) +} + +fn merged( + params: &ParamSet, + items: Rc>, + shards: usize, + passes: usize, + feed: Feed, ) -> Result, BuildError> { let mut out = Vec::with_capacity(passes); for _ in 0..passes { - let (mut acc, rest) = hydra_univmon_shards(params, &items, shards)?; + let (mut acc, rest) = hydra_univmon_shards(params, &items, shards, feed)?; out.push(Box::new(move || { for other in rest.iter() { acc.inner @@ -325,10 +473,30 @@ pub fn merge_step_hydra_univmon( items: Rc>, shards: usize, passes: usize, +) -> Result, BuildError> { + merged_step(params, items, shards, passes, counted::) +} + +pub fn merge_step_hydra_univmon_sum( + params: &ParamSet, + items: Rc>, + shards: usize, + passes: usize, +) -> Result, BuildError> { + weights(&items)?; + merged_step(params, items, shards, passes, weighted::) +} + +fn merged_step( + params: &ParamSet, + items: Rc>, + shards: usize, + passes: usize, + feed: Feed, ) -> Result, BuildError> { let mut out = Vec::with_capacity(passes); for _ in 0..passes { - let (acc, rest) = hydra_univmon_shards(params, &items, shards)?; + let (acc, rest) = hydra_univmon_shards(params, &items, shards, feed)?; let acc: Shared<_> = Rc::new(RefCell::new(acc)); let (driven, read) = (acc.clone(), acc); out.push(StepPass { @@ -353,12 +521,13 @@ fn hydra_univmon_shards( params: &ParamSet, items: &[(String, V)], shards: usize, + feed: Feed, ) -> Result<(HydraUnivmon, Vec), BuildError> { let mut parts: Vec = Vec::new(); for shard in partition(items, shards) { let mut sketch = build_hydra_univmon(params, label_columns(items, "hydra-univmon")?)?; for v in shard { - update(&mut sketch.inner, &v.0, &v.1.data_input(), "hydra-univmon"); + feed(&mut sketch, v); } parts.push(sketch); } diff --git a/sketch-bench/src/wrappers/kll/mod.rs b/sketch-bench/src/wrappers/kll/mod.rs index 92521c08..00e2d9b0 100644 --- a/sketch-bench/src/wrappers/kll/mod.rs +++ b/sketch-bench/src/wrappers/kll/mod.rs @@ -65,8 +65,9 @@ pub(crate) fn kll_lib_slots(k: u32) -> usize { /// and the merge buffer's capacity of `k` items. Independent of the item count, /// so a window's sketch costs the same however many values it sees. The /// sketch's own allocation only: the cdf row's prepared CDF table, built for -/// queries, is not counted (nor is the oxide cdf row's). -fn kll_lib_bytes(k: u32) -> usize { +/// queries, is not counted (nor is the oxide cdf row's). Each `hydra-kll` +/// cell is one of these. +pub(crate) fn kll_lib_bytes(k: u32) -> usize { kll_lib_slots(k) * std::mem::size_of::() + (KLL_LIB_MAX_LEVELS + 1) * std::mem::size_of::() + kll_lib_k(k) * std::mem::size_of::() diff --git a/sketch-bench/src/wrappers/polars_shared.rs b/sketch-bench/src/wrappers/polars_shared.rs index 7e9e3c52..7cc7f02b 100644 --- a/sketch-bench/src/wrappers/polars_shared.rs +++ b/sketch-bench/src/wrappers/polars_shared.rs @@ -519,7 +519,7 @@ impl PolarsSubpopVectorCore { } } -fn grouped_values( +pub(crate) fn grouped_values( buf: &[(String, T)], aggregate: impl Fn(LazyFrame) -> LazyFrame, ) -> Option { From e661b13b7cdab8eaeccbf4af4a47f5ffc3dce45d Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 10 Oct 2026 05:04:20 +0000 Subject: [PATCH 10/18] hydra-kll: the footprint's merge buffer is the steady state; tie tests The grid clones its cells from one template, and a cloned Vec keeps no capacity, so a cell's merge buffer is empty at build and grows to about cell_k items at its first compaction. Say the footprint counts it at that bound. The CDF test now asks a group of ties (at or below) and a mixed-tie group on a grid wide enough that it shares no cell. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FhcWJExZcmVqS6r6rEtjis --- sketch-bench/src/wrappers/hydra_kll/mod.rs | 19 +++++++++++++++++++ .../src/wrappers/hydra_kll/sketchlib.rs | 7 +++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/sketch-bench/src/wrappers/hydra_kll/mod.rs b/sketch-bench/src/wrappers/hydra_kll/mod.rs index 35f07656..4b92fc5e 100644 --- a/sketch-bench/src/wrappers/hydra_kll/mod.rs +++ b/sketch-bench/src/wrappers/hydra_kll/mod.rs @@ -131,6 +131,25 @@ mod tests { assert_eq!(h.estimate_subpop_cdf(&[Some("a")], 25.0), 0.25); assert_eq!(h.estimate_subpop_cdf(&[Some("a")], 100.0), 1.0); assert_eq!(h.estimate_subpop_cdf(&[Some("a")], 0.9), 0.0); + // Ties count as at or below: every one of b's values is 0.5. + assert_eq!(h.estimate_subpop_cdf(&[Some("b")], 0.5), 1.0); + + // Mixed ties, on a grid wide enough that `c` shares no cell: its 2s + // are three of its five values. + let mut wide = build_hydra_kll( + &ParamSet::of(&HydraKllParams { + rows: 3, + cols: 4096, + cell_k: 200, + }), + 2, + ) + .expect("canonical dimensions build"); + for v in [1.0, 2.0, 2.0, 2.0, 3.0] { + fed_kll(&mut wide, &frecord("c;y", v)); + } + assert_eq!(wide.estimate_subpop_cdf(&[Some("c")], 2.0), 0.8); + assert_eq!(wide.estimate_subpop_cdf(&[Some("c")], 1.0), 0.2); } #[test] diff --git a/sketch-bench/src/wrappers/hydra_kll/sketchlib.rs b/sketch-bench/src/wrappers/hydra_kll/sketchlib.rs index 97a5bfd1..c1af9097 100644 --- a/sketch-bench/src/wrappers/hydra_kll/sketchlib.rs +++ b/sketch-bench/src/wrappers/hydra_kll/sketchlib.rs @@ -59,8 +59,11 @@ impl HydraKll { } /// One `asap_sketchlib::KLL` per cell times the grid area: its retained -/// slots, level index and merge buffer of `cell_k` items, as the `kll-*` lib -/// rows count them (`kll_lib_bytes`). Analytic because the cell allocates once. +/// slots and level index, allocated with the grid, plus its merge buffer at +/// its `cell_k`-item bound (`kll_lib_bytes`). The grid clones its cells from +/// one template and a cloned `Vec` keeps no capacity, so a cell's buffer is +/// empty at build and grows to about `cell_k` items once its level 0 first +/// compacts: this is the steady-state footprint, not the one at build. pub fn memory_hydra_kll(sketch: &HydraKll) -> usize { let p = &sketch.params; p.rows * p.cols * kll_cell_bytes::(p.cell_k) + grid_overhead_bytes(p.rows, p.cols) From 6e649de5ea1a76a919237beb1cb41bc49ab3f24b Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 10 Oct 2026 05:09:40 +0000 Subject: [PATCH 11/18] hydra: record merge_split; review fixes - `merge_split` (contiguous|interleaved) is a record field next to `merge_shards` (BenchSection and MergeMetrics; schema regenerated). Merge records stamp it; a record without it split contiguously, so existing results parse, and flatten refuses two different splits in one row as it does two shard counts. - `MergeSplit` moves to aqpbm-core (it is a record type) and is parsed by clap directly instead of matching strings. - The interleaved copy of the stream is built only when a merge runs. - README states the round-robin exactly (contiguous shard sizes, so a short last shard shifts the pattern); MergeMetrics docs name the split. - hydra-cs bound test: note its margin (passes from beta ~4.8). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FhcWJExZcmVqS6r6rEtjis --- README.md | 10 ++++-- aqpbm-cli/src/cli.rs | 7 ++-- aqpbm-cli/src/flatten_record.rs | 36 ++++++++++++++++++-- aqpbm-cli/src/main.rs | 7 ++-- aqpbm-cli/src/repeat.rs | 1 + aqpbm-cli/src/rows/statistic.rs | 9 +++-- aqpbm-core/schema/merged_record.schema.json | 30 ++++++++++++++++- aqpbm-core/src/benchmark_result/mod.rs | 2 +- aqpbm-core/src/benchmark_result/schema.rs | 37 +++++++++++++++++++-- aqpbm-core/src/lib.rs | 4 +-- sketch-bench/src/wrappers/hydra_shared.rs | 5 +++ sketch-bench/src/wrappers/mod.rs | 13 ++------ 12 files changed, 130 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 5d2925cd..b5dd7006 100644 --- a/README.md +++ b/README.md @@ -392,6 +392,7 @@ approxbench: hydra-cms/lib config={"cell_cols":1024,"cell_rows":3,"cols":1024,"r ] }, "merge_shards": 8, + "merge_split": "contiguous", "merge_supported": true, "merge_cpu_time_ms": { "user_ms": { @@ -452,9 +453,12 @@ many ways, one sketch is fed per shard, and the sketches are folded into one. **`--merge-split`**: how the stream is split into those shards. `contiguous` (the default) cuts it into consecutive stretches; `interleaved` deals it -round-robin by record, so each shard is a sample of the whole stream (records -`0, k, 2k, ...` go to the first of `k` shards). The shard sizes are the same -either way. CMS, CountSketch and HLL cells (and their Hydra grids) merge +round-robin by record, so each shard is a sample of the whole stream. The +round-robin runs over the contiguous split's shard sizes (`ceil(N/k)` each, +the last one short), skipping a shard once it is full. So when `k` divides +`N`, shard 0 gets records `0, k, 2k, ...`; with a short last shard the +pattern shifts once that shard fills (N = 10, k = 4: shard 0 gets `0, 4, 7`). +The record's `merge_split` says which split ran. CMS, CountSketch and HLL cells (and their Hydra grids) merge exactly, so they answer identically under both; KLL and UnivMon merges are lossy and can differ. diff --git a/aqpbm-cli/src/cli.rs b/aqpbm-cli/src/cli.rs index 6b911613..cff64dad 100644 --- a/aqpbm-cli/src/cli.rs +++ b/aqpbm-cli/src/cli.rs @@ -2,6 +2,8 @@ //! `main.rs`. Option help and grouping track `docs/aqpbm-cli-reference.md`, which //! `scripts/dump_cli_reference.sh` diffs the built binary against. +use aqpbm_core::MergeSplit; +use clap::builder::TypedValueParser; use clap::{Parser, Subcommand}; use crate::atomic_costs_cmd; @@ -241,10 +243,11 @@ pub struct SketchbenchArgs { #[arg( long, default_value = "contiguous", - value_parser = ["contiguous", "interleaved"], + value_parser = clap::builder::PossibleValuesParser::new(["contiguous", "interleaved"]) + .try_map(|s| s.parse::()), help_heading = "Measurement content" )] - pub merge_split: String, + pub merge_split: MergeSplit, /// Path to append JSONL records to. `-` or omitted sends them to stdout. #[arg(long, help_heading = "Output")] diff --git a/aqpbm-cli/src/flatten_record.rs b/aqpbm-cli/src/flatten_record.rs index c69b1004..225b01b0 100644 --- a/aqpbm-cli/src/flatten_record.rs +++ b/aqpbm-cli/src/flatten_record.rs @@ -4,7 +4,8 @@ //! named by both. Grouping records by identity is the caller's job. use aqpbm_core::{ - BenchSection, InsertMetrics, MergeMetrics, MergedRecord, PrepareMetrics, QueryMetrics, Record, + BenchSection, InsertMetrics, MergeMetrics, MergeSplit, MergedRecord, PrepareMetrics, + QueryMetrics, Record, }; /// Which slot a `Record` lands in, per its own `bench.operation` field. Every @@ -84,6 +85,7 @@ pub fn flatten_record(records: &[Record]) -> Result { accuracy, merge_folds_per_sec, merge_shards, + merge_split, merge_supported, } = bench; @@ -113,7 +115,7 @@ pub fn flatten_record(records: &[Record]) -> Result { } }; } - // `merge_shards`/`merge_supported` describe the merge setup, which + // `merge_shards`/`merge_split`/`merge_supported` describe the merge setup, which // every merge square of one invocation stamps — equal values agree, // different ones mean two setups were folded together. macro_rules! keep_same { @@ -183,6 +185,12 @@ pub fn flatten_record(records: &[Record]) -> Result { "merge_folds_per_sec" ); keep_same!(out.merge.merge_shards, *merge_shards, "merge_shards"); + // A record from before `merge_split` existed split contiguously. + keep_same!( + out.merge.merge_split, + merge_split.or(Some(MergeSplit::Contiguous)), + "merge_split" + ); keep_same!( out.merge.merge_supported, *merge_supported, @@ -422,6 +430,30 @@ mod tests { assert!(err.contains("merge_shards"), "{err}"); } + /// Two splits are two setups; a record without the field split + /// contiguously, so it agrees with a contiguous one. + #[test] + fn merge_split_on_two_squares_must_agree() { + let merge = |metric, split| { + record( + "merge", + metric, + BenchSection { + merge_split: split, + ..Default::default() + }, + ) + }; + let out = flatten_record(&[merge("throughput", None)]).expect("one setup"); + assert_eq!(out.merge.merge_split, Some(MergeSplit::Contiguous)); + let err = flatten_record(&[ + merge("throughput", Some(MergeSplit::Interleaved)), + merge("accuracy", None), + ]) + .expect_err("two splits in one row"); + assert!(err.contains("merge_split"), "{err}"); + } + /// `throughput_items_per_sec` is owned by exactly one metric-pass per /// operation — nothing legitimately produces it twice for `insert`. A /// second value here is a genuine duplicate, refused by field name diff --git a/aqpbm-cli/src/main.rs b/aqpbm-cli/src/main.rs index 6ccef32e..84886741 100644 --- a/aqpbm-cli/src/main.rs +++ b/aqpbm-cli/src/main.rs @@ -50,7 +50,6 @@ use report_sink::ReportSink; // not know the set; it asks. use sketch_bench::registry; use sketch_bench::request::Requirement; -use sketch_bench::wrappers::MergeSplit; /// What is measured. No default and no `all`: a request names the measurements /// it wants, and a shorthand that swept every operation against every metric @@ -358,10 +357,7 @@ fn run_sketchbench(args: SketchbenchArgs) -> Result<()> { width, workers: args.workers.max(1), merge_shards: args.merge_shards, - merge_split: match args.merge_split.as_str() { - "interleaved" => MergeSplit::Interleaved, - _ => MergeSplit::Contiguous, - }, + merge_split: args.merge_split, comparator: args.comparator.clone(), runs: args.runs, warmup_runs: args.warmup_runs, @@ -444,6 +440,7 @@ fn run_sketchbench(args: SketchbenchArgs) -> Result<()> { // and a record states what ran rather than what was asked for. if operation == Operation::Merge { report.bench.merge_shards = Some(args.merge_shards.max(MIN_MERGE_SHARDS)); + report.bench.merge_split = Some(args.merge_split); } reports.push(report); } diff --git a/aqpbm-cli/src/repeat.rs b/aqpbm-cli/src/repeat.rs index 19ae79e9..8a37f7c0 100644 --- a/aqpbm-cli/src/repeat.rs +++ b/aqpbm-cli/src/repeat.rs @@ -220,6 +220,7 @@ mod tests { accuracy: Some(serde_json::json!({"are_all": 0.01, "accuracy_runs": 5})), merge_folds_per_sec: Some(stats(mean * 0.5)), merge_shards: Some(4), + merge_split: None, merge_supported: Some(true), }); rec diff --git a/aqpbm-cli/src/rows/statistic.rs b/aqpbm-cli/src/rows/statistic.rs index cea3cbf4..ce35a7ef 100644 --- a/aqpbm-cli/src/rows/statistic.rs +++ b/aqpbm-cli/src/rows/statistic.rs @@ -352,10 +352,13 @@ where let (probes, score) = questions(ground_truth, &table)?; let items = materialise(description, table)?; // What the folds cut into shards: the stream itself, or reordered so the - // same cut deals it round-robin. + // same cut deals it round-robin. Copied only when a merge will use it. + let merging = want + .iter() + .any(|&(operation, _)| operation == Operation::Merge); let merged = match req.merge_split { - MergeSplit::Contiguous => items.clone(), - MergeSplit::Interleaved => Rc::new(interleave(&items, shards(req))), + MergeSplit::Interleaved if merging => Rc::new(interleave(&items, shards(req))), + _ => items.clone(), }; let mut bodies = Vec::with_capacity(want.len()); for &(operation, metric) in want { diff --git a/aqpbm-core/schema/merged_record.schema.json b/aqpbm-core/schema/merged_record.schema.json index 4b858a32..251d771e 100644 --- a/aqpbm-core/schema/merged_record.schema.json +++ b/aqpbm-core/schema/merged_record.schema.json @@ -100,7 +100,7 @@ "minimum": 0 }, "merge_accuracy": { - "description": "The same comparator as `query_accuracy`, against the same whole-stream\ntruth, scoring the sketch left by folding `merge_shards` contiguous\nshards. A linear sketch should match `query_accuracy` exactly." + "description": "The same comparator as `query_accuracy`, against the same whole-stream\ntruth, scoring the sketch left by folding `merge_shards` shards split\nas `merge_split` says. A linear sketch should match `query_accuracy`\nexactly under either split." }, "merge_cpu_time_ms": { "anyOf": [ @@ -147,6 +147,18 @@ "format": "uint", "minimum": 0 }, + "merge_split": { + "description": "How the stream was split into the `merge_shards` shards.", + "anyOf": [ + { + "$ref": "#/$defs/MergeSplit" + }, + { + "type": "null" + } + ], + "default": null + }, "merge_supported": { "type": [ "boolean", @@ -322,6 +334,7 @@ "merge_heap_allocated_kb", "merge_rss_peak_kb", "merge_shards", + "merge_split", "merge_supported", "merge_timestamp", "merge_wall_time_ms", @@ -694,6 +707,21 @@ "count" ] }, + "MergeSplit": { + "description": "How a merge run split the stream into its shards.", + "oneOf": [ + { + "description": "Each shard a consecutive stretch of the stream.", + "type": "string", + "const": "contiguous" + }, + { + "description": "Round-robin by record over the contiguous split's shard sizes, so each\nshard is a sample of the whole stream.", + "type": "string", + "const": "interleaved" + } + ] + }, "Mode": { "type": "string", "enum": [ diff --git a/aqpbm-core/src/benchmark_result/mod.rs b/aqpbm-core/src/benchmark_result/mod.rs index f9bcac0b..3fcf7932 100644 --- a/aqpbm-core/src/benchmark_result/mod.rs +++ b/aqpbm-core/src/benchmark_result/mod.rs @@ -11,7 +11,7 @@ pub use bench_report::BenchReport; pub use fold::MemoryMaxima; pub use schema::{ BenchSection, CpuTime, ExternalWorkload, InsertMetrics, Language, LatencySummary, MergeMetrics, - MergedRecord, Mode, PrepareMetrics, QueryMetrics, Record, RunStats, Source, + MergeSplit, MergedRecord, Mode, PrepareMetrics, QueryMetrics, Record, RunStats, Source, WorkloadDescription, SCHEMA_VERSION, }; pub use welford::Welford; diff --git a/aqpbm-core/src/benchmark_result/schema.rs b/aqpbm-core/src/benchmark_result/schema.rs index 6b644160..43bd4cd2 100644 --- a/aqpbm-core/src/benchmark_result/schema.rs +++ b/aqpbm-core/src/benchmark_result/schema.rs @@ -129,6 +129,30 @@ pub enum Source { CppBench, } +/// How a merge run split the stream into its shards. +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum MergeSplit { + /// Each shard a consecutive stretch of the stream. + #[default] + Contiguous, + /// Round-robin by record over the contiguous split's shard sizes, so each + /// shard is a sample of the whole stream. + Interleaved, +} + +impl std::str::FromStr for MergeSplit { + type Err = String; + fn from_str(s: &str) -> Result { + match s { + "contiguous" => Ok(Self::Contiguous), + "interleaved" => Ok(Self::Interleaved), + other => Err(format!("unknown merge split '{other}'")), + } + } +} + /// MACRO (benchmark) section of a record. Every sub-field is /// `Option`al so N/A metrics don't pollute the JSON with zero /// placeholders. @@ -179,6 +203,11 @@ pub struct BenchSection { /// if the implementation turned out not to support merging. #[serde(skip_serializing_if = "Option::is_none")] pub merge_shards: Option, + /// How the stream was split into those shards. Present whenever merge was + /// measured; absent on records written before the field existed, which + /// all split contiguously. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub merge_split: Option, /// Present, and `true`, whenever a merge was measured. Never `false`: a row /// that provides no merge is refused where the closures are built, before anything /// is timed, so the gap shows up as an error naming the row rather than as a @@ -323,10 +352,14 @@ pub struct MergeMetrics { #[serde(default, skip_serializing_if = "Option::is_none")] pub merge_folds_per_sec: Option, pub merge_shards: Option, + /// How the stream was split into the `merge_shards` shards. + #[serde(default)] + pub merge_split: Option, pub merge_supported: Option, /// The same comparator as `query_accuracy`, against the same whole-stream - /// truth, scoring the sketch left by folding `merge_shards` contiguous - /// shards. A linear sketch should match `query_accuracy` exactly. + /// truth, scoring the sketch left by folding `merge_shards` shards split + /// as `merge_split` says. A linear sketch should match `query_accuracy` + /// exactly under either split. #[serde(rename = "merge_accuracy")] pub accuracy: Option, #[serde(rename = "merge_cpu_time_ms")] diff --git a/aqpbm-core/src/lib.rs b/aqpbm-core/src/lib.rs index b104f323..053a0ccf 100644 --- a/aqpbm-core/src/lib.rs +++ b/aqpbm-core/src/lib.rs @@ -25,8 +25,8 @@ pub use atomic_costs::{ }; pub use benchmark_result::{ BenchReport, BenchSection, CpuTime, ExternalWorkload, InsertMetrics, LatencySummary, - MergeMetrics, MergedRecord, Mode, PrepareMetrics, QueryMetrics, Record, RunStats, Source, - WorkloadDescription, SCHEMA_VERSION, + MergeMetrics, MergeSplit, MergedRecord, Mode, PrepareMetrics, QueryMetrics, Record, RunStats, + Source, WorkloadDescription, SCHEMA_VERSION, }; pub use erp::{ erp_artifact, erp_record, ErpArtifact, ErpRecord, ErpResourceProfile, ERP_SCHEMA_VERSION, diff --git a/sketch-bench/src/wrappers/hydra_shared.rs b/sketch-bench/src/wrappers/hydra_shared.rs index 2ccd7250..7cd0db7d 100644 --- a/sketch-bench/src/wrappers/hydra_shared.rs +++ b/sketch-bench/src/wrappers/hydra_shared.rs @@ -294,6 +294,11 @@ mod tests { } } + /// The tightest of the three: on this stream it passes from `BETA` about + /// 4.8, so 8 leaves less than 2x, and it fails with `BETA = 0` or `eps = 0`. + /// A wider cell does not buy slack: at `cell_cols` 1024 the worst group + /// needs `BETA` about 14, because `eps` shrinks while the colliders' share + /// of the error does not. #[test] fn hydra_cs_within_its_bound() { let items = records(N); diff --git a/sketch-bench/src/wrappers/mod.rs b/sketch-bench/src/wrappers/mod.rs index 343730ec..49e2049c 100644 --- a/sketch-bench/src/wrappers/mod.rs +++ b/sketch-bench/src/wrappers/mod.rs @@ -148,16 +148,9 @@ pub fn partition(items: &[T], n: usize) -> Vec<&[T]> { items.chunks(chunk).collect() } -/// How a merge run splits the stream into its shards. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub enum MergeSplit { - /// Each shard a stretch of the stream, as [`partition`] cuts it. - #[default] - Contiguous, - /// Round-robin by record, so each shard is a sample of the whole stream: - /// the stream is first reordered by [`interleave`]. - Interleaved, -} +/// How a merge run splits the stream into its shards: `Contiguous` as +/// [`partition`] cuts it, `Interleaved` reordered first by [`interleave`]. +pub use aqpbm_core::MergeSplit; /// The stream reordered so that [`partition`] into `n` hands out a round-robin /// split: record `j` goes to the next shard with room, so each shard holds From e453ce1057a48bb9cf89053b3b1b748029600e02 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 10 Oct 2026 05:23:37 +0000 Subject: [PATCH 12/18] study: --phase hydra on the eval's datasets; Hydra cost rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit configs/datagen/hydra_http.yaml, hydra_http_latency.yaml and hydra_flows.yaml follow the §6.3 eval's http and flows schemas (labels in schema order, cardinalities, child_of, skews; user_id Zipf 0.8, latency Pareto a = 2, src_ip uniform). The flows DDoS burst is not generated: datagen draws each column independently. --phase hydra runs HYDRA_SWEEPS: the eval's variants on those datasets (every grouping, W 1024/4096/16384, N 1e5..1e7, 1/4/16 interleaved shards, 3 seeds) and every statistic on hydra_hier{_d2,_d3,}.yaml, one run per grouping with --per-group-out, into hydra_saturation.csv with the coverage columns err_max_cov_0.01/0.05 (share >= tau plus the largest group). --jobs under a memory budget, --resume, and --hydra-shard i/n to split it across nodes. --phase optimizer-cost adds Hydra rows (eval variants x W x dataset); MeasuredAt gains dataset and schema_width, the reducer reading the width off the row's scores. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FhcWJExZcmVqS6r6rEtjis --- aqpbm-core/src/atomic_costs.rs | 37 ++- configs/datagen/hydra_flows.yaml | 63 ++++ configs/datagen/hydra_http.yaml | 69 +++++ configs/datagen/hydra_http_latency.yaml | 65 +++++ docs/saturation_study.md | 58 ++++ rqe-optimizer/src/lib.rs | 2 + rqe-optimizer/src/saturation.rs | 2 + rqe-optimizer/tests/mixed_grouping.rs | 2 + scripts/study_saturation.py | 373 +++++++++++++++++++++++- scripts/test_study_saturation.py | 174 ++++++++++- 10 files changed, 835 insertions(+), 10 deletions(-) create mode 100644 configs/datagen/hydra_flows.yaml create mode 100644 configs/datagen/hydra_http.yaml create mode 100644 configs/datagen/hydra_http_latency.yaml diff --git a/aqpbm-core/src/atomic_costs.rs b/aqpbm-core/src/atomic_costs.rs index 449b0d2f..3587632e 100644 --- a/aqpbm-core/src/atomic_costs.rs +++ b/aqpbm-core/src/atomic_costs.rs @@ -74,6 +74,15 @@ pub struct MeasuredAt { pub merge_operand_items: Option, /// The value column's distribution as generated, e.g. Zipf with its skew. pub distribution: Option, + /// The datagen spec (yaml stem) a `hydra-*` row was measured on, named by + /// the driver: the record carries the description, not its name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dataset: Option, + /// A `hydra-*` row's label columns `d`: each record is inserted into + /// `2^d - 1` subsets, so its insert cost holds at this width only. Absent + /// for every other row. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schema_width: Option, } impl MeasuredAt { @@ -275,7 +284,10 @@ pub(crate) fn reduce_scores(record: &MergedRecord) -> Result) -> MeasuredA .filter(|&m| m > 0) .map(|m| items / m as u64), distribution: value.map(|c| c.distribution.clone()), + dataset: None, + schema_width: None, } } @@ -483,6 +497,25 @@ mod tests { /// A one-column table: its column gives the keys and the values, and the /// merge operand is the stream split `merge_shards` ways. + /// A Hydra row's insert is per record at its schema width, which the + /// row names; a row without the score names none. + #[test] + fn measured_at_carries_a_hydra_rows_schema_width() { + let mut record = full_record(); + assert_eq!( + reduce_one(&record, METRIC) + .unwrap() + .measured_at + .schema_width, + None + ); + record.query.accuracy = + Some(serde_json::json!({"relative_error_mean": 0.1, "schema_width": 4.0})); + let entry = reduce_one(&record, METRIC).expect("fully populated record"); + assert_eq!(entry.measured_at.schema_width, Some(4)); + assert!((entry.insert_cpu_secs - 0.5e-6).abs() < 1e-12); + } + #[test] fn measured_at_reads_a_single_column_workload() { let mut record = full_record(); @@ -721,6 +754,8 @@ mod tests { value_range: None, merge_operand_items: None, distribution: None, + dataset: None, + schema_width: None, }, }; let json = serde_json::to_string(&entry).unwrap(); diff --git a/configs/datagen/hydra_flows.yaml b/configs/datagen/hydra_flows.yaml new file mode 100644 index 00000000..5a365913 --- /dev/null +++ b/configs/datagen/hydra_flows.yaml @@ -0,0 +1,63 @@ +# The §6.3 eval's `flows` metric (export_autosketch_eval_table.py SCHEMAS, +# #196), for `study_saturation.py --phase hydra`. Labels in schema order, then +# the value: +# +# dst_subnet 1000 values, Zipf 1.1 +# dst_port 1000 values, Zipf 1.2 +# proto 3 values, equally likely (#196's skew 0.0) +# src_ip uniform over 1e6 (distinct sources) +# +# Not generated: #196's DDoS burst, where the least popular subnet (rank 999) +# gets 5% of the traffic from 1e4 distinct sources. datagen draws each column +# independently, so it cannot tie a value's sources to one label value. The +# burst subnet is one group of share >= 5% with few distinct sources; the +# study's 5%-coverage column here reads the top Zipf subnets instead. +# +# approxbench sketchbench --variant hydra-hll --library lib \ +# --spec configs/datagen/hydra_flows.yaml --dtype i64 \ +# --config "rows=3 cols=1024" --group-columns 0 \ +# --operations query --metrics accuracy +# +# Three labels cost `2^3 - 1 = 7` cell insertions per record. + +column_num: 4 +column_label: [dst_subnet, dst_port, proto, src_ip] +row_num: 1000000 + +column_spec: + - data_type: string + string: + min_len: 4 + max_len: 4 + distribution: + kind: zipf + skewness: 1.1 + population_size: 1000 + seed: 1 + + - data_type: string + string: + min_len: 4 + max_len: 4 + distribution: + kind: zipf + skewness: 1.2 + population_size: 1000 + seed: 2 + + - data_type: string + string: + min_len: 2 + max_len: 2 + distribution: + kind: uniform + lower_bound: 0.0 + upper_bound: 3.0 + seed: 3 + + - data_type: i64 + distribution: + kind: uniform + lower_bound: 0.0 + upper_bound: 1000000.0 + seed: 4 diff --git a/configs/datagen/hydra_http.yaml b/configs/datagen/hydra_http.yaml new file mode 100644 index 00000000..d0ed960a --- /dev/null +++ b/configs/datagen/hydra_http.yaml @@ -0,0 +1,69 @@ +# The §6.3 eval's `http` metric (export_autosketch_eval_table.py SCHEMAS, #196), +# for `study_saturation.py --phase hydra`: the eval reads Hydra's accuracy on +# data with its own label schema and skews, since a group's error depends on +# its share of the fanned-out mass. Labels in schema order, then the value: +# +# region 4 values, Zipf 0.5 +# service 25 values, Zipf 1.1 (independent of region, as #196 has it) +# endpoint 25 per service (child_of service), Zipf 1.1 +# status 4 values, Zipf 2.0 +# user_id Zipf 0.8 over 1e6 (distinct users) +# +# approxbench sketchbench --variant hydra-hll --library lib \ +# --spec configs/datagen/hydra_http.yaml --dtype i64 \ +# --config "rows=3 cols=1024" --group-columns 0,1 \ +# --operations query --metrics accuracy +# +# Four labels cost `2^4 - 1 = 15` cell insertions per record. The study sets +# `row_num` per N and offsets every seed per study seed. + +column_num: 5 +column_label: [region, service, endpoint, status, user_id] +row_num: 1000000 + +column_spec: + - data_type: string + string: + min_len: 4 + max_len: 4 + distribution: + kind: zipf + skewness: 0.5 + population_size: 4 + seed: 1 + + - data_type: string + string: + min_len: 4 + max_len: 4 + distribution: + kind: zipf + skewness: 1.1 + population_size: 25 + seed: 2 + + - data_type: string + child_of: 1 + fan_out: 25 + distribution: + kind: zipf + skewness: 1.1 + population_size: 25 + seed: 3 + + - data_type: string + string: + min_len: 3 + max_len: 3 + distribution: + kind: zipf + skewness: 2.0 + population_size: 4 + seed: 4 + + - data_type: i64 + distribution: + kind: zipf + skewness: 0.8 + population_size: 1000000 + seed: 5 diff --git a/configs/datagen/hydra_http_latency.yaml b/configs/datagen/hydra_http_latency.yaml new file mode 100644 index 00000000..adfb3723 --- /dev/null +++ b/configs/datagen/hydra_http_latency.yaml @@ -0,0 +1,65 @@ +# `hydra_http.yaml` with the latency value instead of user_id, for the eval's +# quantile RQEs (template 16, p99 latency): the same labels, skews and seeds, +# and an f64 value drawn Pareto(a = 2, scale 1000), the classic set's tail +# (COST_PARETO_ALPHA). +# +# Not modeled, as #196's runner does not model it either: #196 scales each +# service's latencies by its own factor and makes one endpoint 10x slower. +# Scaling keeps the tail index, and datagen draws a column independently of +# the labels, so here every group shares one latency distribution. +# +# approxbench sketchbench --variant hydra-kll --library lib \ +# --spec configs/datagen/hydra_http_latency.yaml --dtype f64 \ +# --config "rows=3 cols=1024 cell_k=200" --group-columns 1 \ +# --operations query --metrics accuracy + +column_num: 5 +column_label: [region, service, endpoint, status, latency] +row_num: 1000000 + +column_spec: + - data_type: string + string: + min_len: 4 + max_len: 4 + distribution: + kind: zipf + skewness: 0.5 + population_size: 4 + seed: 1 + + - data_type: string + string: + min_len: 4 + max_len: 4 + distribution: + kind: zipf + skewness: 1.1 + population_size: 25 + seed: 2 + + - data_type: string + child_of: 1 + fan_out: 25 + distribution: + kind: zipf + skewness: 1.1 + population_size: 25 + seed: 3 + + - data_type: string + string: + min_len: 3 + max_len: 3 + distribution: + kind: zipf + skewness: 2.0 + population_size: 4 + seed: 4 + + - data_type: f64 + distribution: + kind: pareto + alpha: 2.0 + scale: 1000.0 + seed: 5 diff --git a/docs/saturation_study.md b/docs/saturation_study.md index a2ad6f7d..f60b69de 100644 --- a/docs/saturation_study.md +++ b/docs/saturation_study.md @@ -264,6 +264,64 @@ point in `crossover.csv`. The plotting scripts draw one config per sketch (rows=3 cols=1024, lg_k=12, k=200, alpha=0.01, the configs their bounds are written for). +## Hydra (`--phase hydra`) + +A Hydra grid's error for a group depends on the group's share of the +fanned-out mass `N·(2^d − 1)`, not on a per-key shape, so it is measured on +data with the §6.3 eval's own schemas rather than on the Zipf grid above +(docs/rqe_optimizer_hydra.md). `HYDRA_SWEEPS` holds two sweeps: + +- `eval`: `hydra-hll` and `hydra-univmon-cardinality` on + `configs/datagen/hydra_http.yaml` and `hydra_flows.yaml`, `hydra-kll` on + `hydra_http_latency.yaml` (the specs follow `export_autosketch_eval_table.py`'s + `SCHEMAS`; their headers say what is not modelled). Grid `R = 3`, + `W ∈ {1024, 4096, 16384}`, every non-empty label subset (15 for http, 7 for + flows), `N ∈ {1e5, 1e6, 1e7}`, merged from `{1, 4, 16}` interleaved shards, + seeds 1..3. +- `generic`: every statistic (`HYDRA_CELLS`) on `hydra_hier_d2.yaml`, + `hydra_hier_d3.yaml`, `hydra_hier.yaml` (`d` = 2, 3, 4), every subset, + `W ∈ {1024, 4096}`, `N = 1e6`, shards `{1, 4}`, seeds 1..2. + +Inner cells are the canonical ones, except UnivMon's (`HYDRA_UNIVMON_CELL`, +3 × 256 × 8 layers, heap 64): the canonical 5 × 2048 cell makes a +3 × 16384 grid 32 GB. A study seed `s` rewrites the spec's `row_num` and adds +`1000·(s − 1)` to every seed, so seed 1 is the file as written. + +Each run asks one grouping with `--group-columns` and `--per-group-out`, and +adds one row to `hydra_saturation.csv`: `variant, statistic, config, R, W, +dataset, group_columns` (label names joined by `,`, in schema order; quoted), +`schema_width, records, fanned_mass, merge_shards, merge_split` (empty for +the plain query), `seed`, the record's `err_mean, err_p50, err_p90, err_max, +groups_scored` (in the family's own metric), and `err_max_cov_0.01`, +`err_max_cov_0.05`: the worst error over the groups holding at least that +share `n_q / records` of the records, plus always the largest group (an +unscored group has no error but can be the largest). Rows are written as runs +finish; `--resume` keeps them and runs the rest, and a failed run is reported +and rerun by the next `--resume`. + +`--jobs` runs in parallel, admitted while the running runs' estimated memory +(`m` shard grids plus the data, `hydra_mem_bytes`) fits `--hydra-mem-gb`. +`--hydra-shard i/n` takes every `n`-th run of the fixed run list from the +`i`-th, so `n` machines split a sweep; concatenate their tables afterwards. +`--hydra-ns`, `--hydra-ws`, `--hydra-seeds`, `--hydra-variants`, +`--hydra-datasets` and `--hydra-groupings` narrow the sweeps. + +```sh +# Node i of 5 (0-based), 48 parallel runs: +python3 scripts/study_saturation.py --phase hydra --jobs 48 \ + --hydra-shard i/5 --out out_hydra_i # add --resume after an interruption +# Then, in one place: +awk 'FNR > 1 || NR == 1' out_hydra_*/hydra_saturation.csv > hydra_saturation.csv +``` + +`--phase optimizer-cost` also measures the eval variants at each eval `W` on +their datasets (full schema, `COST_N` records, `--families` includes `hydra`): +`insert_cpu_secs` per record at that schema width, `merge_cpu_secs` per grid +merge, `query_cpu_secs` per probe (a group, or a group and quantile), memory +per grid; `measured_at` names the `dataset` and `schema_width`. Their accuracy +column is one pass's `err_max` at the default grouping; the optimizer reads +Hydra accuracy from `hydra_saturation.csv`. + ## Results (config grid, N up to 1e7) `--phase accuracy --n-max 1e7 --seeds 3 --jobs 16 --out out_grid_1e7`: 595 diff --git a/rqe-optimizer/src/lib.rs b/rqe-optimizer/src/lib.rs index e7868d5d..a3b59131 100644 --- a/rqe-optimizer/src/lib.rs +++ b/rqe-optimizer/src/lib.rs @@ -628,6 +628,8 @@ pub(crate) mod test_support { value_range: None, merge_operand_items: None, distribution: None, + dataset: None, + schema_width: None, } } diff --git a/rqe-optimizer/src/saturation.rs b/rqe-optimizer/src/saturation.rs index bbf623ee..cea60c71 100644 --- a/rqe-optimizer/src/saturation.rs +++ b/rqe-optimizer/src/saturation.rs @@ -1320,6 +1320,8 @@ mod tests { population_size: 1_000, seed: 1, })), + dataset: None, + schema_width: None, }, ..deployment(TOPK, 1).config } diff --git a/rqe-optimizer/tests/mixed_grouping.rs b/rqe-optimizer/tests/mixed_grouping.rs index b33c564d..59c5f3d5 100644 --- a/rqe-optimizer/tests/mixed_grouping.rs +++ b/rqe-optimizer/tests/mixed_grouping.rs @@ -92,6 +92,8 @@ fn cost_row(sketch: &str, params: serde_json::Value) -> AtomicCostEntry { value_range: None, merge_operand_items: None, distribution: None, + dataset: None, + schema_width: None, }, } } diff --git a/scripts/study_saturation.py b/scripts/study_saturation.py index 82ba8fad..b5954bcb 100644 --- a/scripts/study_saturation.py +++ b/scripts/study_saturation.py @@ -30,6 +30,8 @@ The grid also holds every config at the optimizer-cost shape unless --no-cost-shape: θ = 1.1 and K = 1e4 join --thetas and --cardinalities (a full row and column, so the grid stays a full cross) and a = 2 joins --alphas. +--phase hydra measures Hydra's per-group error on the eval's datasets and a +generic hierarchy (HYDRA_SWEEPS) into hydra_saturation.csv; see `hydra`. --resume keeps the complete curves of an interrupted accuracy run; --cost-rows 3 times only the rows=3 Vector2D configs. @@ -46,16 +48,21 @@ seed-mean error per N of the sketch merged from m contiguous shards (m=1 is the plain single-sketch query) rqe_atomic_costs.json (+ _raw, _grid .jsonl) --phase optimizer-cost only + hydra_saturation.csv (+ hydra_accuracy.jsonl) --phase hydra only: one row + per run, HYDRA_COLUMNS """ import argparse import csv +import itertools import json import math import os +import re import subprocess import sys -from concurrent.futures import ThreadPoolExecutor +import threading +from concurrent.futures import ThreadPoolExecutor, as_completed FREQ_CONFIGS = [f"rows={r} cols={c}" for r in (3, 5) for c in (256, 1024, 4096, 16384)] @@ -122,6 +129,75 @@ ] EXACT_METRIC = "relative_error" +# --phase hydra (docs/rqe_optimizer_hydra.md; ASAPQuery#777 §6.3): Hydra's +# per-group error depends on the group's share of the fanned-out mass, so the +# eval reads accuracy measured on datasets with its own schemas +# (configs/datagen/hydra_http*.yaml, hydra_flows.yaml, matching +# export_autosketch_eval_table.py's SCHEMAS). Each run asks one grouping (by +# label name, every non-empty subset of the schema) at grid R x W, N records, +# merged from m interleaved shards (1 = the plain query), one seed, and writes +# its per-group errors (--per-group-out), from which the coverage columns are +# read. The other statistics run on the generic hydra_hier*.yaml sweep +# (schema width 2, 3, 4). +HYDRA_R = 3 +# Inner cells: the canonical ones (docs/rqe_optimizer_hydra.md §1), except +# UnivMon's, whose 5 x 2048 x 8-layer cell (~650 KB) makes a 3 x 16384 grid +# 32 GB; this one is ~100 KB. +HYDRA_UNIVMON_CELL = "cell_heap_size=64 cell_sketch_row=3 cell_sketch_col=256 cell_layer_size=8" +HYDRA_CELLS = { + "hydra-cms": "cell_rows=3 cell_cols=512", + "hydra-cs": "cell_rows=3 cell_cols=512", + "hydra-hll": "", + "hydra-kll": "cell_k=200", + "hydra-kll-cdf": "cell_k=200", + **{f"hydra-univmon-{s}": HYDRA_UNIVMON_CELL + for s in ("cardinality", "l1-norm", "l2-norm", "entropy", "sum")}, +} +# The statistic each variant answers, a column of hydra_saturation.csv. +HYDRA_STATISTICS = { + "hydra-cms": "frequency", "hydra-cs": "frequency", "hydra-hll": "cardinality", + "hydra-kll": "quantile", "hydra-kll-cdf": "cdf", + "hydra-univmon-cardinality": "cardinality", "hydra-univmon-l1-norm": "l1_norm", + "hydra-univmon-l2-norm": "l2_norm", "hydra-univmon-entropy": "entropy", + "hydra-univmon-sum": "sum", +} +# Bytes per grid cell, measured at N = 1e6 (data and truth excluded), and per +# record of generated data plus its exact per-group truth: --hydra-mem-gb +# admits a run only while the running ones' estimates fit. A merge holds all +# m shard grids at once. +HYDRA_CELL_BYTES = {"hydra-cms": 6_500, "hydra-cs": 6_500, "hydra-hll": 17_000, + "hydra-kll": 10_000, "hydra-kll-cdf": 10_000} +HYDRA_UNIVMON_CELL_BYTES = 100_000 +HYDRA_RECORD_BYTES = 250 +# (sweep, datasets as (yaml stem, value dtype, variants), W, N, merge shards, +# seeds). +HYDRA_SWEEPS = { + "eval": ([("hydra_http", "i64", ["hydra-hll", "hydra-univmon-cardinality"]), + ("hydra_flows", "i64", ["hydra-hll", "hydra-univmon-cardinality"]), + ("hydra_http_latency", "f64", ["hydra-kll"])], + (1024, 4096, 16384), (100_000, 1_000_000, 10_000_000), (1, 4, 16), 3), + "generic": ([(stem, "i64", list(HYDRA_CELLS)) + for stem in ("hydra_hier_d2", "hydra_hier_d3", "hydra_hier")], + (1024, 4096), (1_000_000,), (1, 4), 2), +} +HYDRA_DATAGEN = "configs/datagen" +HYDRA_TABLE = "hydra_saturation.csv" +# Coverage thresholds τ: err_max_cov_τ is the worst error over the groups of +# share >= τ plus the largest group (the eval's covers_share rule). +HYDRA_COVERS = (0.01, 0.05) +HYDRA_COLUMNS = [ + "variant", "statistic", "config", "R", "W", "dataset", "group_columns", "schema_width", + "records", "fanned_mass", "merge_shards", "merge_split", "seed", "err_mean", "err_p50", + "err_p90", "err_max", "groups_scored", +] + [f"err_max_cov_{tau}" for tau in HYDRA_COVERS] +# What makes a row the same measurement, for --resume. +HYDRA_KEY = ["variant", "config", "dataset", "group_columns", "records", "merge_shards", + "seed"] +# The optimizer-cost table's Hydra rows: the eval's variants on its datasets +# at each eval W. Their accuracy column is the default grouping's err_max; the +# optimizer reads Hydra accuracy from hydra_saturation.csv. +HYDRA_COST_METRIC = "err_max" + SUMMARY_COLUMNS = [ "family", "sketch", "config", "dist", "param", "cardinality", "n_sat", "final_error", "error_metric", "insert_cpu_secs", "merge_cpu_secs", @@ -333,6 +409,32 @@ def optimizer_cost(args, families): for variant, comparator, spec in EXACT_COST_ROWS: metrics[variant] = EXACT_METRIC rows.append((variant, "exact", [], comparator, ["--spec", spec, "--dtype", "i64"])) + # Hydra: each eval variant on its datasets at the full schema, at each + # eval W, COST_N records (the spec as written, seed 1). Insert is per + # record at that schema width, merge per grid, query per probe, memory + # per grid; measured_at names the dataset. + hydra_datasets = {} + if "hydra" in families: + datasets, ws = HYDRA_SWEEPS["eval"][0], HYDRA_SWEEPS["eval"][1] + for stem, dtype, variants in datasets: + with open(os.path.join(args.datagen, f"{stem}.yaml")) as f: + text = f.read() + spec = os.path.join(args.out, f"{stem}_n{COST_N}.yaml") + with open(spec, "w") as f: + f.write(hydra_spec(text, COST_N, 1)) + for variant in variants: + metrics[variant] = HYDRA_COST_METRIC + for w in ws[:1] if args.one_config else ws: + config = hydra_config(variant, w) + rows.append((variant, "lib", ["--config", config], None, + ["--spec", spec, "--dtype", dtype])) + # The reducer reads the schema width off the scores; the + # dataset is the one thing the record cannot name. + key = (variant, w, len(hydra_labels(text))) + if key in hydra_datasets: + sys.exit(f"Hydra cost rows {key} on {hydra_datasets[key]} and {stem} " + "cannot be told apart") + hydra_datasets[key] = stem def sketchbench(variant, library, config, data, passes): subprocess.run( @@ -348,8 +450,8 @@ def sketchbench(variant, library, config, data, passes): "--merge-shards", str(MERGE_SHARDS), "--runs", str(COST_RUNS), "--warmup-runs", str(COST_WARMUP)]) sketchbench(variant, library, config, data, [ - "--operations", "query", "--metrics", "accuracy", "--comparator", comparator, - "--runs", "1", "--warmup-runs", "0"]) + "--operations", "query", "--metrics", "accuracy", "--runs", "1", + "--warmup-runs", "0"] + (["--comparator", comparator] if comparator else [])) subprocess.run([args.binary, "flatten", raw, "--output", grid], check=True) reduce = subprocess.run( @@ -367,6 +469,17 @@ def sketchbench(variant, library, config, data, passes): if summary != expected: sys.exit(f"atomic-costs did not keep all {len(rows)} rows: {summary!r}") seed_mean_accuracy(args, table, rows, metrics) + if hydra_datasets: + with open(table) as f: + entries = json.load(f) + for entry in entries: + at = entry["measured_at"] + if entry["sketch"].startswith("hydra-"): + at["dataset"] = hydra_datasets[( + entry["sketch"], int(entry["sketch_config"]["params"]["cols"]), + at["schema_width"])] + with open(table, "w") as f: + json.dump(entries, f, indent=2) print(f"Done. {table}", file=sys.stderr) return 0 @@ -380,8 +493,10 @@ def params(config): means = {} for variant, library, config, comparator, data in rows: - if library == "exact" or float(params(config[1]).get("heap", TOPK_HEAPS[0])) != \ - TOPK_HEAPS[0]: + # Hydra rows keep their one pass: the optimizer reads Hydra's accuracy + # from hydra_saturation.csv. + if library == "exact" or variant.startswith("hydra-") or \ + float(params(config[1]).get("heap", TOPK_HEAPS[0])) != TOPK_HEAPS[0]: continue errors = [error(run(args.binary, [ "--variant", variant, "--library", library, *config, @@ -401,13 +516,232 @@ def params(config): json.dump(entries, f, indent=2) +def hydra_labels(spec_text): + """The label names of a Hydra datagen spec: its column_label before the + value column, in schema order.""" + match = re.search(r"^column_label:\s*\[(.*)\]", spec_text, re.M) + return [name.strip() for name in match.group(1).split(",")][:-1] + + +def hydra_spec(spec_text, records, seed): + """`spec_text` at `records` rows, every distribution seed offset by + 1000 · (seed − 1), so seed 1 is the file as written and each study seed + draws every column afresh. Plain text edits: the specs are this repo's.""" + text = re.sub(r"^row_num:.*$", f"row_num: {records}", spec_text, flags=re.M) + return re.sub(r"^(\s*seed:\s*)(\d+)\s*$", + lambda m: f"{m.group(1)}{int(m.group(2)) + 1000 * (seed - 1)}", text, + flags=re.M) + + +def hydra_groupings(labels): + """Every non-empty subset of `labels`, in schema order, smallest first.""" + return [list(c) for n in range(1, len(labels) + 1) + for c in itertools.combinations(labels, n)] + + +def hydra_config(variant, w): + return " ".join(filter(None, [f"rows={HYDRA_R} cols={w}", HYDRA_CELLS[variant]])) + + +def hydra_jobs(sweeps, specs, records=None): + """Every run of `sweeps`, in one fixed order (so `--hydra-shard i/n` picks + the same runs on every node): (dataset, dtype, variant, config, W, + grouping, records, merge shards, seed). `specs` maps a yaml stem to its + text; `records`, if given, replaces each sweep's record counts.""" + jobs = [] + for sweep in sweeps: + datasets, ws, ns, shards, seeds = HYDRA_SWEEPS[sweep] + ns = records or ns + for stem, dtype, variants in datasets: + groupings = hydra_groupings(hydra_labels(specs[stem])) + for variant, w, grouping, n, m, seed in itertools.product( + variants, ws, groupings, ns, shards, range(1, seeds + 1)): + jobs.append((stem, dtype, variant, hydra_config(variant, w), w, grouping, n, + m, seed)) + return jobs + + +def hydra_mem_bytes(job): + """A run's estimated peak memory: its m shard grids plus the data.""" + _, _, variant, _, w, _, n, m, _ = job + cell = HYDRA_CELL_BYTES.get(variant, HYDRA_UNIVMON_CELL_BYTES) + return m * HYDRA_R * w * cell + n * HYDRA_RECORD_BYTES + + +def read_per_group(path): + """[(n_q, error or None)] from a --per-group-out CSV; an empty error is a + group the comparator could not score.""" + with open(path, newline="") as f: + return [(int(r["n_q"]), float(r["error"]) if r["error"] else None) + for r in csv.DictReader(f)] + + +def covered_max(groups, records, tau): + """The worst error over the groups holding >= `tau` of the `records`, + plus always the largest group (so no coverage is empty): the eval's + covers_share rule. Unscored groups count for "largest" but have no + error; "" when no covered group was scored.""" + if not groups: + return "" + largest = max(range(len(groups)), key=lambda i: groups[i][0]) + errors = [e for i, (n_q, e) in enumerate(groups) + if e is not None and (i == largest or n_q / records >= tau)] + return max(errors) if errors else "" + + +def hydra_row(job, labels, record, groups): + """One hydra_saturation.csv row from a run's record and per-group file.""" + stem, _, variant, config, w, grouping, n, m, seed = job + acc = record["bench"]["accuracy"] + records = int(acc["records"]) + return { + "variant": variant, "statistic": HYDRA_STATISTICS[variant], "config": config, + "R": HYDRA_R, "W": w, "dataset": stem, "group_columns": ",".join(grouping), + "schema_width": int(acc["schema_width"]), "records": records, + "fanned_mass": int(acc["fanned_mass"]), "merge_shards": m, + "merge_split": record["bench"].get("merge_split", ""), "seed": seed, + **{k: acc.get(k, "") for k in ("err_mean", "err_p50", "err_p90", "err_max")}, + "groups_scored": int(acc["groups_scored"]), + **{f"err_max_cov_{tau}": covered_max(groups, records, tau) for tau in HYDRA_COVERS}, + } + + +def hydra_key(row): + """A hydra_saturation.csv row's identity, from a row dict (values as the + CSV holds them).""" + return tuple(str(row[c]) for c in HYDRA_KEY) + + +class MemoryBudget: + """Admits a run while the running runs' estimates fit in `total` bytes; + a run larger than the whole budget runs alone.""" + + def __init__(self, total): + self.total, self.used, self.cond = total, 0, threading.Condition() + + def acquire(self, n): + with self.cond: + self.cond.wait_for(lambda: self.used == 0 or self.used + n <= self.total) + self.used += n + + def release(self, n): + with self.cond: + self.used -= n + self.cond.notify_all() + + +def hydra(args): + """--phase hydra: every run of --hydra-sweeps (this node's share with + --hydra-shard i/n), one row each in --out/hydra_saturation.csv, rows + written as runs finish. --resume keeps the rows already there and runs + the rest; a failed run is reported and the phase exits 1 at the end.""" + os.environ.setdefault("BENCH_WARMUP_SECS", "0") + # --jobs runs share the cores; polars' own pool would oversubscribe them. + os.environ.setdefault("POLARS_MAX_THREADS", "1") + os.makedirs(args.out, exist_ok=True) + specs = {} + for sweep in args.hydra_sweeps.split(","): + for stem, _, _ in HYDRA_SWEEPS[sweep][0]: + with open(os.path.join(args.datagen, f"{stem}.yaml")) as f: + specs[stem] = f.read() + ns = [int(float(n)) for n in args.hydra_ns.split(",")] if args.hydra_ns else None + jobs = hydra_jobs(args.hydra_sweeps.split(","), specs, ns) + ws = {int(w) for w in args.hydra_ws.split(",")} if args.hydra_ws else None + groupings = ({tuple(g.split(",")) for g in args.hydra_groupings.split(";")} + if args.hydra_groupings else None) + jobs = [j for j in jobs + if (ws is None or j[4] in ws) + and (groupings is None or tuple(j[5]) in groupings) + and j[8] <= (args.hydra_seeds or j[8]) + and (not args.hydra_variants or j[2] in args.hydra_variants.split(",")) + and (not args.hydra_datasets or j[0] in args.hydra_datasets.split(","))] + shard, n_shards = (int(x) for x in args.hydra_shard.split("/")) + jobs = jobs[shard::n_shards] + + table = os.path.join(args.out, HYDRA_TABLE) + kept = [] + if args.resume and os.path.exists(table): + with open(table, newline="") as f: + # A row cut off mid-write has empty trailing fields; it reruns. + kept = [r for r in csv.DictReader(f) if r.get(HYDRA_COLUMNS[-1]) is not None] + done = {hydra_key(r) for r in kept} + todo = [j for j in jobs if hydra_key({ + "variant": j[2], "config": j[3], "dataset": j[0], "group_columns": ",".join(j[5]), + "records": j[6], "merge_shards": j[7], "seed": j[8]}) not in done] + # Largest first, so the long runs do not trail the pool. + todo.sort(key=lambda j: (j[6], j[7], j[4]), reverse=True) + print(f"{len(jobs)} runs in shard {shard}/{n_shards}, {len(jobs) - len(todo)} kept, " + f"{len(todo)} to run", file=sys.stderr) + + spec_dir = os.path.join(args.out, "hydra_specs") + group_dir = os.path.join(args.out, "hydra_groups") + os.makedirs(spec_dir, exist_ok=True) + os.makedirs(group_dir, exist_ok=True) + for stem, n, seed in {(j[0], j[6], j[8]) for j in todo}: + with open(os.path.join(spec_dir, f"{stem}_n{n}_s{seed}.yaml"), "w") as f: + f.write(hydra_spec(specs[stem], n, seed)) + budget = MemoryBudget(args.hydra_mem_gb * 1e9) + + def measure(i, job): + stem, dtype, variant, config, _, grouping, n, m, seed = job + labels = hydra_labels(specs[stem]) + per_group = os.path.join(group_dir, f"{shard}_{i}.csv") + operation = ["query"] if m == 1 else [ + "merge", "--merge-shards", str(m), "--merge-split", "interleaved"] + need = hydra_mem_bytes(job) + budget.acquire(need) + try: + record = run(args.binary, [ + "--variant", variant, "--library", "lib", "--config", config, + "--spec", os.path.join(spec_dir, f"{stem}_n{n}_s{seed}.yaml"), + "--dtype", dtype, + "--group-columns", ",".join(str(labels.index(g)) for g in grouping), + "--per-group-out", per_group, "--operations", *operation, + "--metrics", "accuracy", "--runs", "1", "--warmup-runs", "0"]) + finally: + budget.release(need) + groups = read_per_group(per_group) + os.remove(per_group) + return record, hydra_row(job, labels, record, groups) + + with open(table, "w", newline="") as f, \ + open(os.path.join(args.out, "hydra_accuracy.jsonl"), "a") as raw: + writer = csv.DictWriter(f, HYDRA_COLUMNS) + writer.writeheader() + writer.writerows(kept) + f.flush() + failed = 0 + with ThreadPoolExecutor(max_workers=args.jobs) as pool: + futures = {pool.submit(measure, i, j): j for i, j in enumerate(todo)} + for k, future in enumerate(as_completed(futures), 1): + job = futures[future] + try: + record, row = future.result() + except (subprocess.CalledProcessError, OSError, KeyError, ValueError) as e: + failed += 1 + detail = getattr(e, "stderr", "") or e + print(f" FAILED {job}: {str(detail).strip()[-500:]}", file=sys.stderr) + continue + raw.write(json.dumps(record) + "\n") + writer.writerow(row) + f.flush() + if k % 50 == 0 or k == len(todo): + print(f" {k}/{len(todo)} runs", file=sys.stderr) + print(f"Done. {table}", file=sys.stderr) + if failed: + print(f"{failed} run(s) failed; --resume reruns them", file=sys.stderr) + return 1 + return 0 + + def main(): parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) parser.add_argument("--binary", default="./target/release/approxbench") parser.add_argument("--out", default="out") - parser.add_argument("--families", default="frequency,topk,cardinality,quantile") + parser.add_argument("--families", default="frequency,topk,cardinality,quantile,hydra", + help="hydra: --phase optimizer-cost's Hydra rows") parser.add_argument("--one-config", action="store_true", help="only the first config of each sketch") parser.add_argument("--thetas", default="0,0.5,0.8,1.0,1.2,1.5,2.0") @@ -427,12 +761,14 @@ def main(): parser.add_argument("--jobs", type=int, default=1, help="parallel accuracy runs (cost runs are always serial)") parser.add_argument("--phase", - choices=["accuracy", "cost", "crossover", "all", "optimizer-cost"], + choices=["accuracy", "cost", "crossover", "all", "optimizer-cost", + "hydra"], default="all", help="cost reads saturation_curve.csv from --out; crossover " "recomputes crossover.csv from the cost phase's JSONL in --out; " "optimizer-cost writes the optimizer's cost table to --out " - "(the grid's configs at one shape, no curves)") + "(the grid's configs at one shape, no curves); hydra writes " + "hydra_saturation.csv (HYDRA_SWEEPS, its own grid)") parser.add_argument("--points-from", help="CSV with sketch,config,dist,param,cardinality columns " "(e.g. a filtered saturation.csv): run only those points") @@ -445,7 +781,28 @@ def main(): parser.add_argument("--merge-shards-list", default="", help="accuracy: e.g. 1,4,16,64: also score the sketch merged from " "m shards, for each m>1, into saturation_merge_curve.csv") + parser.add_argument("--datagen", default=HYDRA_DATAGEN, + help="hydra: where the sweeps' datagen specs are") + parser.add_argument("--hydra-sweeps", default="eval,generic", + help="hydra: which of HYDRA_SWEEPS to run") + parser.add_argument("--hydra-shard", default="0/1", + help="hydra: i/n runs every n-th run from the i-th (0-based), " + "to split one sweep across n machines") + parser.add_argument("--hydra-mem-gb", type=float, default=200, + help="hydra: start a run only while the running runs' estimated " + "memory fits") + # Narrow the sweeps (smoke runs, reruns); each keeps only what it names. + parser.add_argument("--hydra-ns", help="hydra: these record counts instead") + parser.add_argument("--hydra-ws", help="hydra: only these grid widths") + parser.add_argument("--hydra-seeds", type=int, help="hydra: only seeds 1..this") + parser.add_argument("--hydra-variants", help="hydra: only these variants") + parser.add_argument("--hydra-datasets", help="hydra: only these yaml stems") + parser.add_argument("--hydra-groupings", + help="hydra: only these groupings, ';'-separated, labels by name " + "joined by ',' in schema order") args = parser.parse_args() + if args.phase == "hydra": + return hydra(args) families = args.families.split(",") thetas = [float(t) for t in args.thetas.split(",")] diff --git a/scripts/test_study_saturation.py b/scripts/test_study_saturation.py index c8166ba0..3cddb32c 100644 --- a/scripts/test_study_saturation.py +++ b/scripts/test_study_saturation.py @@ -16,7 +16,9 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from study_saturation import checkpoints, n_saturation, n_star, read_curve # noqa: E402 +from study_saturation import ( # noqa: E402 + HYDRA_COLUMNS, checkpoints, covered_max, hydra_groupings, hydra_labels, hydra_spec, + n_saturation, n_star, read_curve, read_per_group) SCRIPT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "study_saturation.py") @@ -565,5 +567,175 @@ def test_a_skipped_row_fails(self): self.assertIn(b"did not keep all 11 rows", result.stderr) +class CoveredMaxTest(unittest.TestCase): + # (n_q, error) of 100 records: shares 0.6, 0.3, 0.06, 0.04. + GROUPS = [(60, 0.1), (30, 0.2), (6, 0.5), (4, 0.9)] + + def test_groups_at_or_above_the_share_count(self): + self.assertEqual(covered_max(self.GROUPS, 100, 0.05), 0.5) + self.assertEqual(covered_max(self.GROUPS, 100, 0.04), 0.9) + self.assertEqual(covered_max(self.GROUPS, 100, 0.3), 0.2) + + def test_the_largest_group_is_always_covered(self): + self.assertEqual(covered_max(self.GROUPS, 100, 0.9), 0.1) + # Wherever it sits in the file. + self.assertEqual(covered_max(self.GROUPS[::-1], 100, 0.9), 0.1) + + def test_unscored_groups_have_no_error_but_can_be_the_largest(self): + groups = [(60, None), (30, 0.2), (6, 0.5)] + self.assertEqual(covered_max(groups, 100, 0.05), 0.5) + # The largest is unscored, so nothing covered has an error. + self.assertEqual(covered_max(groups, 100, 0.9), "") + # An unscored group does not hand "largest" to the next one. + self.assertEqual(covered_max([(60, None), (30, 0.2)], 100, 0.5), "") + + def test_no_groups(self): + self.assertEqual(covered_max([], 100, 0.01), "") + + +SPEC = textwrap.dedent("""\ + column_num: 3 + column_label: [region, service, user_id] + row_num: 200000 + column_spec: + - data_type: string + distribution: + kind: uniform + seed: 1 + - data_type: i64 + distribution: {kind: zipf, skewness: 0.8, population_size: 10, seed: 5} +""") + + +class HydraSpecTest(unittest.TestCase): + def test_rows_and_seeds(self): + self.assertEqual(hydra_spec(SPEC, 1000, 1).replace("row_num: 1000", "row_num: 200000"), + SPEC) + text = hydra_spec(SPEC, 1000, 3) + self.assertIn("row_num: 1000\n", text) + self.assertIn(" seed: 2001\n", text) + # A flow mapping's seed is not a line of its own; it stays. + self.assertIn("seed: 5}", text) + + def test_labels_and_groupings_in_schema_order(self): + self.assertEqual(hydra_labels(SPEC), ["region", "service"]) + self.assertEqual(hydra_groupings(["a", "b", "c"]), + [["a"], ["b"], ["c"], ["a", "b"], ["a", "c"], ["b", "c"], + ["a", "b", "c"]]) + + def test_the_eval_specs_match_their_schemas(self): + root = os.path.join(os.path.dirname(SCRIPT), "..", "configs", "datagen") + + def read(stem): + with open(os.path.join(root, f"{stem}.yaml")) as f: + return f.read() + http = ["region", "service", "endpoint", "status"] + self.assertEqual(hydra_labels(read("hydra_http")), http) + self.assertEqual(hydra_labels(read("hydra_http_latency")), http) + self.assertEqual(hydra_labels(read("hydra_flows")), ["dst_subnet", "dst_port", "proto"]) + self.assertEqual(len(hydra_groupings(http)), 15) + + +# A Hydra run: writes a fixed per-group file for the grouping asked, and a +# record of N = 100 records over 2 labels. Merged runs score 0.1 worse. +FAKE_HYDRA_APPROXBENCH = textwrap.dedent("""\ + import json, os, sys + a = sys.argv + value = lambda flag: a[a.index(flag) + 1] if flag in a else None + with open(os.environ["FAKE_LOG"], "a") as f: + f.write(json.dumps(a[1:]) + "\\n") + if value("--variant") == "hydra-cms" and value("--group-columns") == "1": + sys.exit("boom") + m = int(value("--merge-shards") or 1) + worse = 0.1 if m > 1 else 0.0 + groups = [(60, 0.1 + worse), (30, None), (6, 0.5 + worse), (4, 0.9 + worse)] + with open(value("--per-group-out"), "w") as f: + f.write("group_key,n_q,error\\n") + for i, (n, e) in enumerate(groups): + f.write(f'"label0:g{i},x",{n},{"" if e is None else e}\\n') + errs = sorted(e for _, e in groups if e is not None) + bench = {"accuracy": {"err_mean": sum(errs) / 3, "err_p50": errs[1], "err_p90": errs[2], + "err_max": errs[2], "groups_scored": 3.0, "schema_width": 2.0, + "records": 100.0, "fanned_mass": 300.0}} + if m > 1: + bench.update(merge_shards=m, merge_split=value("--merge-split")) + print(json.dumps({"bench": bench})) +""") + + +class HydraPhaseTest(unittest.TestCase): + def run_phase(self, d, *extra, out="out"): + binary = os.path.join(d, "approxbench") + with open(binary, "w") as f: + f.write("#!" + sys.executable + "\n" + FAKE_HYDRA_APPROXBENCH) + os.chmod(binary, 0o755) + datagen = os.path.join(d, "datagen") + os.makedirs(datagen, exist_ok=True) + for stem in ("hydra_hier_d2", "hydra_hier_d3", "hydra_hier"): + with open(os.path.join(datagen, f"{stem}.yaml"), "w") as f: + f.write(SPEC) + log = os.path.join(d, "log.jsonl") + result = subprocess.run([ + sys.executable, SCRIPT, "--binary", binary, "--out", os.path.join(d, out), + "--phase", "hydra", "--datagen", datagen, "--hydra-sweeps", "generic", + "--hydra-datasets", "hydra_hier_d2", "--hydra-variants", "hydra-hll,hydra-cms", + "--hydra-ws", "1024", "--hydra-ns", "100", "--hydra-seeds", "1", "--jobs", "2", + *extra, + ], capture_output=True, env={**os.environ, "FAKE_LOG": log}) + table = os.path.join(d, out, "hydra_saturation.csv") + with open(table, newline="") as f: + header = next(csv.reader(f)) + with open(table, newline="") as f: + rows = list(csv.DictReader(f)) + with open(log) as f: + return result, rows, header, [json.loads(line) for line in f] + + def test_one_row_per_run_with_coverage_columns(self): + with tempfile.TemporaryDirectory() as d: + result, rows, header, calls = self.run_phase(d) + # 2 variants x 3 groupings x shards {1, 4}; hydra-cms at {service} + # fails (2 runs), is reported, and the phase exits 1. + self.assertEqual(result.returncode, 1) + self.assertIn(b"2 run(s) failed", result.stderr) + self.assertEqual(len(calls), 12) + self.assertEqual(header, HYDRA_COLUMNS) + self.assertEqual(len(rows), 10) + row = {(r["variant"], r["group_columns"], r["merge_shards"]): r for r in rows} + query = row[("hydra-hll", "region,service", "1")] + self.assertEqual( + {k: query[k] for k in ("statistic", "config", "R", "W", "dataset", "schema_width", + "records", "fanned_mass", "merge_split", "seed", + "err_max", "groups_scored")}, + {"statistic": "cardinality", "config": "rows=3 cols=1024", "R": "3", + "W": "1024", "dataset": "hydra_hier_d2", "schema_width": "2", "records": "100", + "fanned_mass": "300", "merge_split": "", "seed": "1", "err_max": "0.9", + "groups_scored": "3"}) + # Shares 0.6, 0.3 (unscored), 0.06, 0.04. + self.assertEqual((query["err_max_cov_0.01"], query["err_max_cov_0.05"]), ("0.9", "0.5")) + merged = row[("hydra-hll", "region,service", "4")] + self.assertEqual((merged["merge_split"], merged["err_max_cov_0.05"]), + ("interleaved", "0.6")) + # Groupings are asked by column index, in schema order. + hll = [c for c in calls if c[c.index("--variant") + 1] == "hydra-hll"] + self.assertEqual(sorted({c[c.index("--group-columns") + 1] for c in hll}), + ["0", "0,1", "1"]) + spec = hll[0][hll[0].index("--spec") + 1] + self.assertTrue(spec.endswith("hydra_hier_d2_n100_s1.yaml")) + + def test_shards_split_the_runs_and_resume_reruns_only_the_missing(self): + with tempfile.TemporaryDirectory() as d: + _, first, _, _ = self.run_phase(d, "--hydra-shard", "0/2") + os.remove(os.path.join(d, "log.jsonl")) + _, other, _, _ = self.run_phase(d, "--hydra-shard", "1/2", out="other") + os.remove(os.path.join(d, "log.jsonl")) + _, rows, _, calls = self.run_phase(d, "--resume") + self.assertEqual(len(first) + len(other), 10) + key = lambda r: (r["variant"], r["group_columns"], r["merge_shards"]) + self.assertFalse({key(r) for r in first} & {key(r) for r in other}) + # The rerun runs the 6 runs shard 0 lacked; the 2 failures fail again. + self.assertEqual(len(calls), 12 - len(first)) + self.assertEqual(len(rows), 10) + + if __name__ == "__main__": unittest.main() From 00337aa870c37cd8aa8c8b459cbb3c08b4d5fe5d Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 10 Oct 2026 05:49:38 +0000 Subject: [PATCH 13/18] hydra-univmon-sum: the ground truth refuses negative values too The lib row inserts each value as a UnivMon count and refuses negatives; the truth (and so the polars baseline) accepted them, so a spec with negatives ran on polars and failed on lib. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FhcWJExZcmVqS6r6rEtjis --- aqpbm-core/src/accuracy/subpopulation/sum.rs | 28 ++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/aqpbm-core/src/accuracy/subpopulation/sum.rs b/aqpbm-core/src/accuracy/subpopulation/sum.rs index 4c997eba..152e122a 100644 --- a/aqpbm-core/src/accuracy/subpopulation/sum.rs +++ b/aqpbm-core/src/accuracy/subpopulation/sum.rs @@ -25,6 +25,13 @@ fn sums_per_group( ) -> Result { let labels = group_labels(table, group_columns)?; let values: Vec = match table.column(value_column)? { + // The lib row inserts each value as a UnivMon count, which can't be + // negative, so the truth refuses negatives too. + ColumnData::Int64(v) if v.iter().any(|&x| x < 0) => { + return Err(DataGenError::BadParam( + "a sum's values are UnivMon counts, so they can't be negative".into(), + )) + } ColumnData::Int64(v) => v.iter().map(|&x| x as f64).collect(), ColumnData::Unsigned64(v) => v.iter().map(|&x| x as f64).collect(), other => { @@ -128,6 +135,27 @@ mod tests { } /// A group whose values sum to zero has no relative error: it is listed + /// Negative values are refused, as the lib row refuses them. + #[test] + fn a_negative_value_is_refused() { + use crate::accuracy::table_of; + let table = table_of( + &["key1", "value"], + vec![ + ColumnData::String(vec!["a".into()]), + ColumnData::Int64(vec![-1]), + ], + ); + let gt = SubpopSumGT { + group_columns: vec![0], + value_column: 1, + }; + let Err(err) = gt.truth(&table) else { + panic!("a negative value must be refused"); + }; + assert!(err.to_string().contains("negative"), "{err}"); + } + /// unscored, as a group with no distinct value is for cardinality. #[test] fn a_zero_sum_group_is_listed_unscored() { From bf8e8743ebf7caf5406299b0a0b0a9b6d9fa2fea Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 10 Oct 2026 06:20:30 +0000 Subject: [PATCH 14/18] study: Hydra cost rows at 3 runs; their test; datagen path from the script Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FhcWJExZcmVqS6r6rEtjis --- scripts/study_saturation.py | 12 +++++++--- scripts/test_study_saturation.py | 40 +++++++++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/scripts/study_saturation.py b/scripts/study_saturation.py index b5954bcb..7ef5cf1e 100644 --- a/scripts/study_saturation.py +++ b/scripts/study_saturation.py @@ -180,7 +180,8 @@ for stem in ("hydra_hier_d2", "hydra_hier_d3", "hydra_hier")], (1024, 4096), (1_000_000,), (1, 4), 2), } -HYDRA_DATAGEN = "configs/datagen" +HYDRA_DATAGEN = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "configs", + "datagen") HYDRA_TABLE = "hydra_saturation.csv" # Coverage thresholds τ: err_max_cov_τ is the worst error over the groups of # share >= τ plus the largest group (the eval's covers_share rule). @@ -197,6 +198,9 @@ # at each eval W. Their accuracy column is the default grouping's err_max; the # optimizer reads Hydra accuracy from hydra_saturation.csv. HYDRA_COST_METRIC = "err_max" +# Fewer passes than COST_RUNS: one pass inserts COST_N records into up to +# 2^4 - 1 subsets each, and a UnivMon grid's merge pass rebuilds 16 of them. +HYDRA_COST_RUNS = (3, 1) SUMMARY_COLUMNS = [ "family", "sketch", "config", "dist", "param", "cardinality", "n_sat", @@ -445,10 +449,12 @@ def sketchbench(variant, library, config, data, passes): for variant, library, config, comparator, data in rows: print(f" {variant} {' '.join(config[1:])}", file=sys.stderr) # Cost first: flatten keeps the first pass's query timings. + runs, warmup = HYDRA_COST_RUNS if variant.startswith("hydra-") else \ + (COST_RUNS, COST_WARMUP) sketchbench(variant, library, config, data, [ "--operations", "insert,query,merge", "--metrics", "throughput,cpu,memory", - "--merge-shards", str(MERGE_SHARDS), "--runs", str(COST_RUNS), - "--warmup-runs", str(COST_WARMUP)]) + "--merge-shards", str(MERGE_SHARDS), "--runs", str(runs), + "--warmup-runs", str(warmup)]) sketchbench(variant, library, config, data, [ "--operations", "query", "--metrics", "accuracy", "--runs", "1", "--warmup-runs", "0"] + (["--comparator", comparator] if comparator else [])) diff --git a/scripts/test_study_saturation.py b/scripts/test_study_saturation.py index 3cddb32c..2ce149dd 100644 --- a/scripts/test_study_saturation.py +++ b/scripts/test_study_saturation.py @@ -486,8 +486,14 @@ def test_a_grid_that_already_has_the_cost_shape_adds_nothing(self): v = c[c.index("--variant") + 1] cfg = c[c.index("--config") + 1] if "--config" in c else "" params = {k: float(x) for k, x in (kv.split("=") for kv in cfg.split())} + # The reducer reads a Hydra row's width off its scores. + at = {} + if v.startswith("hydra-"): + labels = open(c[c.index("--spec") + 1]).read().split("column_label: [")[1] + at = {"schema_width": labels.split("]")[0].count(",")} rows.append({"sketch": v, "sketch_config": {"params": params}, - "query_accuracy": {metrics[v]: 0.5}, "accuracy_metric": metrics[v]}) + "query_accuracy": {metrics[v]: 0.5}, "accuracy_metric": metrics[v], + "measured_at": at}) json.dump(rows, open(value("--output"), "w")) kept = int(os.environ["FAKE_KEPT"]) sys.stderr.write(f"approxbench atomic-costs: {kept} row(s), 0 skipped\\n") @@ -500,7 +506,7 @@ def test_a_grid_that_already_has_the_cost_shape_adds_nothing(self): class OptimizerCostTest(unittest.TestCase): - def run_phase(self, d, kept): + def run_phase(self, d, kept, families="topk,quantile"): binary = os.path.join(d, "approxbench") with open(binary, "w") as f: f.write("#!" + sys.executable + "\n" + FAKE_COST_APPROXBENCH) @@ -508,7 +514,7 @@ def run_phase(self, d, kept): log = os.path.join(d, "log.jsonl") result = subprocess.run([ sys.executable, SCRIPT, "--binary", binary, "--out", os.path.join(d, "out"), - "--phase", "optimizer-cost", "--families", "topk,quantile", "--one-config", + "--phase", "optimizer-cost", "--families", families, "--one-config", "--seeds", "3", ], capture_output=True, env={**os.environ, "FAKE_LOG": log, "FAKE_KEPT": str(kept)}) with open(log) as f: @@ -560,6 +566,34 @@ def test_sketch_rows_take_the_seed_mean_accuracy(self): if row["sketch"].startswith("exact-"): self.assertEqual(row["query_accuracy"][row["accuracy_metric"]], 0.5) + def test_hydra_rows_on_the_eval_datasets_name_their_dataset(self): + with tempfile.TemporaryDirectory() as d: + # hydra-hll and hydra-univmon-cardinality on http and flows, + # hydra-kll on http_latency, at the first W; plus 5 exact. + result, calls = self.run_phase(d, 10, families="hydra") + with open(os.path.join(d, "out", "rqe_atomic_costs.json")) as f: + rows = json.load(f) + self.assertEqual(result.returncode, 0, result.stderr) + value = lambda c, flag: c[c.index(flag) + 1] + hydra = [c for c in calls if c[0] == "sketchbench" and "--report" in c + and value(c, "--variant").startswith("hydra-")] + self.assertEqual(len(hydra), 10) + cost = hydra[0] + self.assertEqual((value(cost, "--runs"), value(cost, "--warmup-runs"), + value(cost, "--merge-shards")), ("3", "1", "16")) + self.assertEqual(value(cost, "--config"), "rows=3 cols=1024") + self.assertTrue(value(cost, "--spec").endswith("hydra_http_n1000000.yaml")) + self.assertNotIn("--comparator", hydra[1]) + # No seed runs: Hydra accuracy comes from hydra_saturation.csv. + self.assertFalse([c for c in calls if c[0] == "sketchbench" and "--report" not in c]) + at = {(r["sketch"], r["measured_at"]["schema_width"]): r["measured_at"]["dataset"] + for r in rows if r["sketch"].startswith("hydra-")} + self.assertEqual(at, { + ("hydra-hll", 4): "hydra_http", ("hydra-hll", 3): "hydra_flows", + ("hydra-univmon-cardinality", 4): "hydra_http", + ("hydra-univmon-cardinality", 3): "hydra_flows", + ("hydra-kll", 4): "hydra_http_latency"}) + def test_a_skipped_row_fails(self): with tempfile.TemporaryDirectory() as d: result, _ = self.run_phase(d, 10) From 003eb3194a37918cc20ed324286fe0dad71cce4e Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 10 Oct 2026 06:36:37 +0000 Subject: [PATCH 15/18] docs: Hydra cost rows' run count Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FhcWJExZcmVqS6r6rEtjis --- docs/saturation_study.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/saturation_study.md b/docs/saturation_study.md index f60b69de..33a7029b 100644 --- a/docs/saturation_study.md +++ b/docs/saturation_study.md @@ -315,7 +315,8 @@ awk 'FNR > 1 || NR == 1' out_hydra_*/hydra_saturation.csv > hydra_saturation.csv ``` `--phase optimizer-cost` also measures the eval variants at each eval `W` on -their datasets (full schema, `COST_N` records, `--families` includes `hydra`): +their datasets (full schema, `COST_N` records, 3 runs after 1 warm-up, `--families` +includes `hydra`): `insert_cpu_secs` per record at that schema width, `merge_cpu_secs` per grid merge, `query_cpu_secs` per probe (a group, or a group and quantile), memory per grid; `measured_at` names the `dataset` and `schema_width`. Their accuracy From ef9c0244e8b50dfdcbd0f56a94a8728b7bc35a4e Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 10 Oct 2026 07:00:29 +0000 Subject: [PATCH 16/18] study_saturation: atomic hydra table rewrite, stricter resume, schema checks --phase hydra --resume now writes the kept rows to hydra_saturation.csv.tmp and os.replace()s it over the table, and keeps a row only when it ends in a line end and every column parses (integers, numbers or "" for the error columns, non-empty text). The CSV format is unchanged, so tables written by 003eb31 resume as before. The eval-spec test also checks each label's cardinality, skew and child_of against PR #196's SCHEMAS. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FhcWJExZcmVqS6r6rEtjis --- scripts/study_saturation.py | 39 +++++++++++++++++++++-- scripts/test_study_saturation.py | 53 ++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 3 deletions(-) diff --git a/scripts/study_saturation.py b/scripts/study_saturation.py index 7ef5cf1e..c4ec3a80 100644 --- a/scripts/study_saturation.py +++ b/scripts/study_saturation.py @@ -54,6 +54,7 @@ import argparse import csv +import io import itertools import json import math @@ -612,6 +613,31 @@ def hydra_row(job, labels, record, groups): } +# hydra_saturation.csv's integer columns; the error columns are numbers or +# "" (no scored group), merge_split is "" for a query run, the rest are text. +HYDRA_INT_COLUMNS = ["R", "W", "schema_width", "records", "fanned_mass", "merge_shards", + "seed", "groups_scored"] + + +def hydra_row_parses(row): + """Whether every column of a hydra_saturation.csv row holds a value of + its type (a row cut off mid-write does not).""" + try: + for c in HYDRA_COLUMNS: + if row.get(c) is None: + return False + if c in HYDRA_INT_COLUMNS: + int(row[c]) + elif c.startswith("err_"): + if row[c] != "": + float(row[c]) + elif c != "merge_split" and not row[c]: + return False + except ValueError: + return False + return True + + def hydra_key(row): """A hydra_saturation.csv row's identity, from a row dict (values as the CSV holds them).""" @@ -668,8 +694,11 @@ def hydra(args): kept = [] if args.resume and os.path.exists(table): with open(table, newline="") as f: - # A row cut off mid-write has empty trailing fields; it reruns. - kept = [r for r in csv.DictReader(f) if r.get(HYDRA_COLUMNS[-1]) is not None] + text = f.read() + # A row cut off mid-write (no line end, or a field that does not + # parse) reruns. + kept = [r for r in csv.DictReader(io.StringIO(text[:text.rfind("\n") + 1])) + if hydra_row_parses(r)] done = {hydra_key(r) for r in kept} todo = [j for j in jobs if hydra_key({ "variant": j[2], "config": j[3], "dataset": j[0], "group_columns": ",".join(j[5]), @@ -710,12 +739,16 @@ def measure(i, job): os.remove(per_group) return record, hydra_row(job, labels, record, groups) - with open(table, "w", newline="") as f, \ + # The kept rows go to a new file that then replaces the table, so a + # crash mid-rewrite cannot lose them; new rows append to it. + with open(table + ".tmp", "w", newline="") as f, \ open(os.path.join(args.out, "hydra_accuracy.jsonl"), "a") as raw: writer = csv.DictWriter(f, HYDRA_COLUMNS) writer.writeheader() writer.writerows(kept) f.flush() + os.fsync(f.fileno()) + os.replace(table + ".tmp", table) failed = 0 with ThreadPoolExecutor(max_workers=args.jobs) as pool: futures = {pool.submit(measure, i, j): j for i, j in enumerate(todo)} diff --git a/scripts/test_study_saturation.py b/scripts/test_study_saturation.py index 2ce149dd..fdc5e226 100644 --- a/scripts/test_study_saturation.py +++ b/scripts/test_study_saturation.py @@ -8,6 +8,7 @@ import csv import json import os +import re import subprocess import sys import tempfile @@ -669,6 +670,33 @@ def read(stem): self.assertEqual(hydra_labels(read("hydra_flows")), ["dst_subnet", "dst_port", "proto"]) self.assertEqual(len(hydra_groupings(http)), 15) + def labels(stem): + """Each label column's (cardinality, skew, child_of), as datagen + reads them; a uniform column's cardinality is its upper bound.""" + text = read(stem).split("column_spec:")[1] + out = [] + for column in text.split("\n - ")[1:]: + field = dict(re.findall(r"(\w+): ([\w.]+)\n", column + "\n")) + if field["data_type"] != "string": + continue + if field["kind"] == "uniform": + out.append((int(float(field["upper_bound"])), 0.0, None)) + else: + child_of = field.get("child_of") + out.append((int(field["population_size"]), float(field["skewness"]), + int(child_of) if child_of else None)) + if child_of: + self.assertEqual(field["fan_out"], field["population_size"]) + return out + # PR #196's SCHEMAS (origin/eval/hydra-workload, + # scripts/export_autosketch_eval_table.py): (cardinality, skew, + # child_of as a column index) per label, in schema order. + http_schema = [(4, 0.5, None), (25, 1.1, None), (25, 1.1, 1), (4, 2.0, None)] + self.assertEqual(labels("hydra_http"), http_schema) + self.assertEqual(labels("hydra_http_latency"), http_schema) + self.assertEqual(labels("hydra_flows"), [(1000, 1.1, None), (1000, 1.2, None), + (3, 0.0, None)]) + # A Hydra run: writes a fixed per-group file for the grouping asked, and a # record of N = 100 records over 2 labels. Merged runs score 0.1 worse. @@ -770,6 +798,31 @@ def test_shards_split_the_runs_and_resume_reruns_only_the_missing(self): self.assertEqual(len(calls), 12 - len(first)) self.assertEqual(len(rows), 10) + def test_resume_reruns_a_cut_off_row_and_replaces_the_table(self): + with tempfile.TemporaryDirectory() as d: + self.run_phase(d) + os.remove(os.path.join(d, "log.jsonl")) + table = os.path.join(d, "out", "hydra_saturation.csv") + with open(table, newline="") as f: + lines = f.read().splitlines(keepends=True) + # Two rows cut off mid-write: one with every field there but an + # empty groups_scored, and the last one with no line end. + fields = lines[1].split(",") + fields[HYDRA_COLUMNS.index("groups_scored")] = "" + lines[1] = ",".join(fields) + lines[-1] = lines[-1].rstrip("\r\n") + with open(table, "w", newline="") as f: + f.writelines(lines) + # The rewrite goes to a new file, so the old one survives. + os.link(table, table + ".old") + _, rows, _, calls = self.run_phase(d, "--resume") + with open(table + ".old", newline="") as f: + self.assertEqual(f.read(), "".join(lines)) + self.assertFalse(os.path.exists(table + ".tmp")) + # Those 2 rerun, and the 2 failures again. + self.assertEqual(len(calls), 4) + self.assertEqual(len(rows), 10) + if __name__ == "__main__": unittest.main() From fed6f4d529c006bf2a46914a7350342f9cc35bcd Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 10 Oct 2026 11:04:50 +0000 Subject: [PATCH 17/18] study_saturation: --hydra-variants/--hydra-ws narrow the Hydra cost rows too A hydra-univmon-cardinality grid at W=16384 merged from 16 shards outgrows a 251 GB node, and the cost phase can't resume; these flags let the cost phase leave a variant or width out, as they do the hydra phase. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FhcWJExZcmVqS6r6rEtjis --- scripts/study_saturation.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/study_saturation.py b/scripts/study_saturation.py index c4ec3a80..a37e3dad 100644 --- a/scripts/study_saturation.py +++ b/scripts/study_saturation.py @@ -428,8 +428,16 @@ def optimizer_cost(args, families): with open(spec, "w") as f: f.write(hydra_spec(text, COST_N, 1)) for variant in variants: + # --hydra-variants / --hydra-ws narrow these rows as they do + # the hydra phase (e.g. to leave out a variant whose 16-shard + # merge at the largest W outgrows the machine). + if args.hydra_variants and variant not in args.hydra_variants.split(","): + continue metrics[variant] = HYDRA_COST_METRIC + hydra_ws = {int(x) for x in args.hydra_ws.split(",")} if args.hydra_ws else None for w in ws[:1] if args.one_config else ws: + if hydra_ws and w not in hydra_ws: + continue config = hydra_config(variant, w) rows.append((variant, "lib", ["--config", config], None, ["--spec", spec, "--dtype", dtype])) From bb46dfd49de319412fd56727765db62ba4fd2f5e Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 10 Oct 2026 11:45:21 +0000 Subject: [PATCH 18/18] aqpbm-datagen: scale_by/scale_range; hydra_http_latency scales latency per service A f64 column with `scale_by: